[svn] r3531 - in trunk/tools/pydar2: . dries dries/pydar2 pydar
scripts
packagers at lists.rpmforge.net
packagers at lists.rpmforge.net
Wed Aug 24 23:40:07 CEST 2005
Author: dries
Date: 2005-08-24 23:40:06 +0200 (Wed, 24 Aug 2005)
New Revision: 3531
Modified:
trunk/tools/pydar2/Makefile
trunk/tools/pydar2/dries/mirrors-part.html
trunk/tools/pydar2/dries/pydar2/planning-part.html
trunk/tools/pydar2/pydar/buildroot.py
trunk/tools/pydar2/pydar/config.py
trunk/tools/pydar2/pydar/postgresqlstorage.py
trunk/tools/pydar2/pydar/smartbasedbuildroot.py
trunk/tools/pydar2/pydar/svnwrapper.py
trunk/tools/pydar2/pydar/yumbasedbuildroot.py
trunk/tools/pydar2/scripts/norpmforgetemplatesfilter.py
trunk/tools/pydar2/scripts/rpmforgesite.py
Log:
update
Modified: trunk/tools/pydar2/Makefile
===================================================================
--- trunk/tools/pydar2/Makefile 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/Makefile 2005-08-24 21:40:06 UTC (rev 3531)
@@ -17,6 +17,7 @@
echo cleaning up..
rm -f *~ */*~
echo nothing to build
+ touch emptyfile
install:
install -m0755 -d \
@@ -32,7 +33,8 @@
$(DESTDIR)$(datadir)/pydar2/sql \
$(DESTDIR)$(datadir)/pydar2/dries \
$(DESTDIR)$(datadir)/pydar2/dries/pydar2 \
- $(DESTDIR)$(datadir)/pydar2/scripts
+ $(DESTDIR)$(datadir)/pydar2/scripts \
+ $(DESTDIR)$(sysconfdir)/pydar2/rpmforgeacceptcommandconfig
install -m0755 $(DIST_PYSCRIPTS) log4py.conf $(DESTDIR)$(datadir)/pydar2/
install -m0755 $(DIST_ETCFILES) $(DESTDIR)$(sysconfdir)/pydar2/
install -m0755 $(DIST_BASHSCRIPTS) $(DESTDIR)$(bindir)
@@ -43,8 +45,9 @@
install sql/master.sql $(DESTDIR)$(datadir)/pydar2/sql
install dries/*.txt dries/*.html $(DESTDIR)$(datadir)/pydar2/dries/
install dries/pydar2/*.html dries/pydar2/*.png dries/pydar2/*.xmi $(DESTDIR)$(datadir)/pydar2/dries/pydar2/
-
-
+ install -m0755 emptyfile $(DESTDIR)$(sysconfdir)/pydar2/rpmforgeacceptcommandconfig/excludedauthoritytags
+ install -m0755 emptyfile $(DESTDIR)$(sysconfdir)/pydar2/rpmforgeacceptcommandconfig/excluded
+ install -m0755 emptyfile $(DESTDIR)$(sysconfdir)/pydar2/rpmforgeacceptcommandconfig/temporarilyexcluded
#install -m0755 scripts/* $(DESTDIR)$(sysconfdir)/rc.d/init.d
#install -m0755 defaults.conf $(DESTDIR)$(datadir)/pydar/
#install -m0755 pydar.conf $(DESTDIR)$(sysconfdir)/pydar/
Modified: trunk/tools/pydar2/dries/mirrors-part.html
===================================================================
--- trunk/tools/pydar2/dries/mirrors-part.html 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/dries/mirrors-part.html 2005-08-24 21:40:06 UTC (rev 3531)
@@ -23,6 +23,9 @@
<li><a href="http://rh-mirror.linux.iastate.edu/pub/dag/dries/">http://rh-mirror.linux.iastate.edu/pub/dag/dries/</a></li>
<li><a href="http://ftp.riken.jp/Linux/dag/dries/">http://ftp.riken.jp/Linux/dag/dries/</a></li>
<li><a href="ftp://ftp.cs.uu.nl/mirror/dag.wieers/dries">ftp://ftp.cs.uu.nl/mirror/dag.wieers/dries</a></li>
+ <li><a href="http://www.mirrorservice.org/sites/apt.sw.be/dries/">http://www.mirrorservice.org/sites/apt.sw.be/dries/</a></li>
+ <li><a href="ftp://ftp.mirrorservice.org/sites/apt.sw.be/dries/">ftp://ftp.mirrorservice.org/sites/apt.sw.be/dries/</a></li>
+ <li><a href="rsync://rsync.mirrorservice.org/sites/apt.sw.be/dries/">rsync://rsync.mirrorservice.org/sites/apt.sw.be/dries/</a></li>
</ul>
</p>
Modified: trunk/tools/pydar2/dries/pydar2/planning-part.html
===================================================================
--- trunk/tools/pydar2/dries/pydar2/planning-part.html 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/dries/pydar2/planning-part.html 2005-08-24 21:40:06 UTC (rev 3531)
@@ -1,4 +1,12 @@
<a href="index.html">Back</a><br />
+<h2>Recent changes:</h2>
+<p>
+<ul>
+ <li>New option in master config: checkmissingtagsperdistro. When this option is set, the master checks if the most recent version of each spec file has tags for each distro/arch combination in
+ the database. This slows down the startup a lot and it shouldn't be needed with normal use.</li>
+ <li>Fix: the slave now creates the cache dirs for yum if needed.</li>
+</ul>
+</p>
<h2>Short term:</h2>
<p>
<ul>
@@ -12,7 +20,6 @@
<li>finish the spec file of pydar2 and build the rpms + put them online: not yet done.. </li>
<li>Basic version of autoqueueing: done</li>
<li>RsyncYumBasedBuildRoot: first version done</li>
-
</ul>
</p>
<h2>Near future:</h2>
Modified: trunk/tools/pydar2/pydar/buildroot.py
===================================================================
--- trunk/tools/pydar2/pydar/buildroot.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/pydar/buildroot.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -31,13 +31,15 @@
# normally you should check the result each time
from log4py import Logger
+import buildrootresult
+import commands, re, os, string, shutil
#from specfile import SpecFile
class BuildRoot:
# this will already set certain variables
def __init__(self,buildRootTag):
- self.__cat = Logger().get_instance(self)
- self.__cat.debug("start")
+ self._cat = Logger().get_instance(self)
+ self._cat.debug("start")
self._setBuildRootTag(buildRootTag)
def _setBuildRootTag(self,buildRootTag):
@@ -47,23 +49,14 @@
def getBuildRootTag(self):
return self.__buildRootTag
- # this cleans up this root
- # depends on the implementation
- # the result is a BuildRootResult object
- def cleanRoot(self):
- self.__cat.debug("start")
# this prepares a buildroot so it can be used for building an rpm
# this calls cleanRoot if needed
# this creates the buildRoot from scratch if needed
# the result is a BuildRootResult object
def prepareRoot(self):
- self.__cat.debug("start")
+ self._cat.debug("start")
- # the argument is a SlaveCommand object
- # the result is a BuildRootResult object
- def buildRpm(self,command):
- self.__cat.debug("start")
# rpms is an array of strings (names of rpms)
# this is used for installing the buildrequirements
@@ -71,5 +64,167 @@
# otherwise you get a lot of errors (so use prepareRoot first)
# the result is a BuildRootResult object
def installRpms(self,rpms):
- self.__cat.debug("start")
+ self._cat.debug("start")
+ raise Exception("you should override installRpms")
+ def setUseCacheOnly(self,bVar):
+ self.__useCacheOnly = bVar
+
+ def _umounts(self, brr):
+ self._cat.debug("start")
+ cmd = "sudo umount " + self.rootdir + "/proc"
+ self.logCommandWithoutErrors(cmd,brr)
+ cmd = "sudo umount " + self.rootdir + "/sys"
+ self.logCommandWithoutErrors(cmd,brr)
+
+ def cleanRoot(self):
+ self._cat.debug("cleaning this root..")
+ brr = buildrootresult.BuildRootResult(self)
+ self._umounts(brr)
+ cmd = "sudo rm -Rf " + self.rootdir
+ self.logCommand(cmd,brr)
+ return brr
+
+ def logCommands(self,cmdArr,brr):
+ for cmd in cmdArr:
+ self.logCommand(cmd,brr)
+ # todo: if error => stop
+
+ def logCommand(self,cmd,brr):
+ brr.logDebugLine("CMD: Command line:" + cmd)
+ (status,output) = commands.getstatusoutput(cmd)
+ brr.logDebugLine("CMD: Status:" + str(status))
+ # first remove all the '\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*
+ output = re.sub('\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*',"",output)
+ output = re.sub('\r',"",output)
+ brr.logDebugLine("CMD: output:" + str(output))
+ if status != 0:
+ brr.logErrorLine("CMD: status (" + str(status) + ") is not 0 !")
+
+ def logCommandWithoutErrors(self,cmd,brr):
+ brr.logDebugLine("CMD: Command line:" + cmd)
+ (status,output) = commands.getstatusoutput(cmd)
+ brr.logDebugLine("CMD: Status:" + str(status))
+ # first remove all the '\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*
+ output = re.sub('\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*',"",output)
+ output = re.sub('\r',"",output)
+ brr.logDebugLine("CMD: output:" + str(output))
+
+ def __getLogsAndRpmsAndSrpm(self,command,brr):
+ self._cat.debug("start")
+ shutil.copy(os.path.join(self.rootdir, "buildlog.txt"), command.getBuildDir())
+ cmd = "gzip " + command.getBuildDir() + "/buildlog.txt"
+ self.logCommand(cmd,brr)
+ # search in the /usr/src/redhat/RPMS and /usr/src/redhat/SRPMS dirs..
+ rpmsdir = os.path.join(self.rootdir,'usr/src/redhat/RPMS')
+ ld = os.listdir(rpmsdir)
+ for entry in ld:
+ self.__getRpmsOrSrpms(command,brr,os.path.join(rpmsdir,entry))
+ srpmsdir = os.path.join(self.rootdir,'usr/src/redhat/SRPMS')
+ self.__getRpmsOrSrpms(command,brr,srpmsdir)
+
+ def __getRpmsOrSrpms(self,command,brr,path):
+ self._cat.debug('start, path=' + path)
+ ld = os.listdir(path)
+ for entry in ld:
+ rindex = string.rfind(entry,'.rpm')
+ if os.path.isfile(os.path.join(path,entry)) and (rindex > 0 and rindex == len(entry) - len('.rpm')):
+ # let's copy it
+ shutil.copy(os.path.join(path,entry),command.getBuildDir())
+ rindex = string.rfind(entry,'.src.rpm')
+ if rindex > 0 and rindex == len(entry) - len('.src.rpm'):
+ brr.setSrpm(entry)
+ else:
+ brr.addRpm(entry)
+
+ def __createScript(self,command,brr):
+ self._cat.debug("start")
+ script = "#!/bin/bash\n"
+ script = script + 'for i in ' + self.rootdir + '/etc/profile.d/*.sh; do' + "\n"
+ script = script + ' if [ -r "$i" ]; then' + "\n"
+ script = script + ' . $i' + "\n"
+ script = script + " fi\n"
+ script = script + "done\n"
+ script = script + "chown -R root.root " + self.rootdir + "/usr/src/redhat\n"
+ # the following command first checks if the dependencies are ok
+ # ex: --define '_topdir /var/lib/pydar2/roots/yum-fc3-i386/usr/src/redhat'
+ script = script + "(echo file: " + command.getSpecFileShortFileName() + ";/usr/bin/rpmbuild --define '_topdir " + self.rootdir + "/usr/src/redhat' " + command.getDefines() + " --root=" + self.rootdir + " --nobuild --bs " + self.rootdir + "/usr/src/redhat/SPECS/" + command.getSpecFileShortFileName() + " && echo \"deps check ok\" && "
+ # do the real build
+ # todo: some 'post mortem analysis'
+ # if contains error: Installed (but unpackaged) file(s) found: -> print output of a find in the buildroot
+ # if it contains a config.log: See 'Config.log' for more details.
+ script = script + "/usr/sbin/chroot " + self.rootdir + " /usr/bin/rpmbuild --nodeps -ba /usr/src/redhat/SPECS/" + command.getSpecFileShortFileName() + " " + command.getDefines() + " || "
+ script = script + "(echo ; echo ; echo CONFIG.LOG; echo ; cat " + self.rootdir + "/usr/src/redhat/BUILD/*/*onfig.log; echo ; echo ; echo FILES; echo ; echo ; (cd " + self.rootdir + "/var/tmp/*-root*/ && find .))) > " + self.rootdir + "/buildlog.txt 2>&1"
+ self._cat.debug( "script contains: " + script)
+ return script
+
+ def __callBuildScript(self, scriptcontents, command, brr):
+ self._cat.debug("start")
+ # first write it to a temp file
+ filepath = self.rootdir + "/tmp/pydar2build.sh"
+ fd = open(filepath,"w")
+ fd.write(scriptcontents)
+ fd.close()
+ cmd = "sudo /bin/bash " + self.rootdir + "/tmp/pydar2build.sh"
+ self.logCommand(cmd,brr)
+
+ def __parseLog(self,command,brr):
+ self._cat.debug("start, not ready")
+
+ def buildRpm(self,command):
+ brr = buildrootresult.BuildRootResult(self)
+ self._cat.debug("start")
+ # copy the sources
+ self.__copySources(command,brr)
+ # create a script which does the actual building
+ scriptcontents = self.__createScript(command,brr)
+ # then we call the script
+ self.__callBuildScript(scriptcontents, command, brr)
+ # get all the results
+ self.__getLogsAndRpmsAndSrpm(command,brr)
+ # then we parse the log: get additional usefull files if need, for example config.log if the build has failed
+ self.__parseLog(command,brr)
+ return brr
+
+ def __copySources(self,command,brr):
+ self._cat.debug("start")
+ startdir = command.getBuildDir()
+ listdir = os.listdir(startdir)
+ specdestdir = os.path.join(self.rootdir,'usr/src/redhat/SPECS')
+ sourcesdestdir = os.path.join(self.rootdir,'usr/src/redhat/SOURCES')
+ self._cat.debug('specdestdir: ' + specdestdir)
+ self._cat.debug('sourcesdestdir: ' + sourcesdestdir)
+ for entry in listdir:
+ rindex = string.rfind(entry,'.spec')
+ if os.path.isfile(os.path.join(startdir,entry)) and not (rindex > 0 and rindex == len(entry) - len('.spec')):
+ # its not the spec files
+ shutil.copy(os.path.join(startdir,entry), sourcesdestdir)
+ else:
+ # its the spec file
+ shutil.copy(os.path.join(startdir,entry),specdestdir)
+ self._cat.debug('copy done')
+
+ def _setGeneralVars(self):
+ # this definetely has to move to a config file
+ self.minimalRpmsPart1 = "basesystem filesystem setup bash glibc"
+ self.minimalRpmsPart2 = "coreutils findutils"
+ self.minimalRpmsPart3 = "rpm-devel rpm-build make gcc tar gzip patch unzip bzip2 diffutils cpio elfutils redhat-rpm-config perl"
+ self.minimalRpmsPart4 = ""
+
+ def _getGeneralPrepareRootCommands(self):
+ cmd = []
+ # create some directories first
+ cmd.append("sudo mkdir -p " + self.rootdir + "/var/lib/rpm " + self.rootdir + "/var/log " + self.rootdir + "/var/cache " + self.rootdir + "/etc " + self.rootdir + "/proc " + self.rootdir + "/sys " + self.rootdir + "/usr/src/redhat/BUILD " + self.rootdir + "/usr/src/redhat/RPMS " + self.rootdir + "/usr/src/redhat/SOURCES " + self.rootdir + "/usr/src/redhat/SPECS " + self.rootdir + "/usr/src/redhat/SRPMS " + self.rootdir + "/dev " + self.rootdir + "/home/pydar2slave ")
+ # copy fstab
+ cmd.append("sudo cp /etc/fstab " + self.rootdir + "/etc")
+ # create hosts
+ cmd.append("sudo echo -e \"127.0.0.1 localhost.localdomain localhost\" > " + self.rootdir + "/etc/hosts")
+ # mount proc
+ cmd.append("sudo mount -t proc none " + self.rootdir + "/proc")
+ # mount sys
+ cmd.append("sudo mount -t sysfs none " + self.rootdir + "/sys")
+ # create /dev/null
+ cmd.append("sudo mknod " + self.rootdir + "/dev/null c 1 3")
+ # rpm initdb
+ cmd.append("sudo rpm --root=" + self.rootdir + " --initdb")
+ return cmd
Modified: trunk/tools/pydar2/pydar/config.py
===================================================================
--- trunk/tools/pydar2/pydar/config.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/pydar/config.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -37,6 +37,7 @@
import yumbasedbuildroot
import rsyncyumbasedbuildroot
import storagefactory
+import smartbasedbuildroot
class Config:
__myInstance = None
@@ -130,7 +131,10 @@
if myParser.has_option('master','checksumtype'):
self.__checksumtype = myParser.get('master','checksumtype')
if myParser.has_option('master','checkmissingtagsperdistro'):
- self.__doCheckMissingTagsPerDistro = True
+ if myParser.get('master','checkmissingtagsperdistro') != "0":
+ self.__doCheckMissingTagsPerDistro = True
+ else:
+ self.__doCheckMissingTagsPerDistro = False
else:
self.__doCheckMissingTagsPerDistro = False
#
@@ -267,6 +271,8 @@
tmp = rsyncyumbasedbuildroot.RsyncYumBasedBuildRoot(sect)
if type == "YUM":
tmp = yumbasedbuildroot.YumBasedBuildRoot(sect)
+ if type == "SMART":
+ tmp = smartbasedbuildroot.SmartBasedBuildRoot(sect)
if tmp == None:
self.__cat.error("unknown type for buildarch: " + sect)
else:
Modified: trunk/tools/pydar2/pydar/postgresqlstorage.py
===================================================================
--- trunk/tools/pydar2/pydar/postgresqlstorage.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/pydar/postgresqlstorage.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -45,7 +45,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select id from pydar2_distroarch where id='" + id + "'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
retval = True
cursor.close()
@@ -57,7 +57,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select current.id, current.lastversionid, tname.value as name, tsumm.value as summary, tgroup.value as group from (select sf.id, max(sv.id) as lastversionid from pydar2_specfile sf, pydar2_specrepo repo, pydar2_specfile_version sv where repo.id=sf.specrepoid and repo.name='" + specRepoName + "' and sv.specfileid=sf.id group by sf.id) as current, pydar2_specfile_tags tname, pydar2_specfile_tags tsumm, pydar2_specfile_tags tgroup where tname.versionid=current.lastversionid and tsumm.versionid=current.lastversionid and tgroup.versionid=current.lastversionid and tname.name='NAME' and tsumm.name='SUMMARY' and tgroup.name='GROUP' and tname.distroid=1 and tsumm.distroid=1 and tgroup.distroid=1 order by tgroup.value asc, tname.value asc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
temp = {}
temp['id'] = row[0]
@@ -77,7 +77,7 @@
cursor = conn.cursor()
sql = "select t.value from pydar2_specfile_version sv, pydar2_specfile_tags t where t.versionid=sv.id and upper(t.name)='# AUTHORITY' and sv.specfileid=" + str(specFileId) + " order by sv.id desc limit 1"
self.__cat.debug("sql:" + sql)
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
retval = row[0]
cursor.close()
@@ -89,7 +89,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select min(v.addtimestamp)::date as mintimestamp, tname.value, f.filename, f.subdir from pydar2_specfile_version v, pydar2_specfile f, pydar2_specfile_tags tname, pydar2_specrepo sr where tname.name='NAME' and tname.versionid=v.id and f.id=v.specfileid and f.specrepoid=sr.id and sr.name='" + specRepoName + "' group by f.id, tname.value, f.filename, f.subdir having min(v.addtimestamp) > (current_date - '1 month'::interval) order by mintimestamp asc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
temphash = {}
temphash['timestamp'] = row[0] # mintimestamp, first occurrence of this rpm with this version/release for this distroarchtag
@@ -107,7 +107,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select min(tf.addtimestamp)::date as mintimestamp, tf.filename, tf.name, tf.distroarchtag from pydar2_target_files tf, pydar2_targets targ, pydar2_specfile_version v, pydar2_specfile_tags tname, pydar2_distroarch distro where tf.version = v.version and tname.versionid=v.id and tname.name='NAME' and tname.value=tf.name and tname.distroid=distro.id and distro.distroarchtag=tf.distroarchtag and tf.targetid=targ.id and targ.name='" + targetName + "' group by tf.filename, tf.name, tf.distroarchtag having (min(tf.addtimestamp) > (current_date - '1 month'::interval)) order by mintimestamp asc, tf.name asc, tf.filename asc, tf.distroarchtag asc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
temphash = {}
temphash['timestamp'] = row[0] # mintimestamp, first occurrence of this rpm with this version/release for this distroarchtag
@@ -125,7 +125,7 @@
cursor = conn.cursor()
sql = "select t.value from pydar2_specfile_version sv, pydar2_specfile_tags t where t.versionid=sv.id and upper(t.name)='NAME' and sv.specfileid=" + str(specFileId) + " order by sv.id desc limit 1"
self.__cat.debug("sql:" + sql)
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
retval = row[0]
cursor.close()
@@ -137,7 +137,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select sv.specfileid, max(version) as maxversion from pydar2_specfile_version sv, pydar2_specfile v where sv.specfileid=v.id and v.specrepoid=" + str(specRepository.getId()) + " group by sv.specfileid order by maxversion desc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
retval[row[0]] = row[1]
cursor.close()
@@ -149,7 +149,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select specfileid, max(version) from (select sv.specfileid, sv.version from pydar2_specfile_version sv, pydar2_distroarch pd, pydar2_specfile_tags t1, pydar2_target_files tf, pydar2_specfile s where tf.name=t1.value and t1.name='NAME' and t1.versionid=sv.id and s.id=sv.specfileid and tf.version=sv.version and t1.distroid=pd.id and pd.distroarchtag=tf.distroarchtag and s.specrepoid=" + str(specRepository.getId())+ " and tf.targetid=" + str(targetObj.getId() )+ " and tf.distroarchtag='" + datag + "') as foo1 group by specfileid"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
retval[row[0]] = row[1]
cursor.close()
@@ -162,7 +162,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "update pydar2_target_files set targetid=" + str(targetObj.getId()) + ", version=" + str(version )+ ",name='"+ str(name) + "',checksum='"+ str(checksum) + "',distroarchtag='"+distroarchtag +"' where fileName='" + fileName + "'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -172,7 +172,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "insert into pydar2_target_files (targetid, filename, name, version, checksum, distroarchtag) values (" + str(targetObj.getId()) + ",'"+ fileName + "','"+ name + "'," + str(version) + ",'" + str(checksum) + "','"+ distroarchtag+ "')"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -182,7 +182,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select distroarchtag from pydar2_distroarch"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
retval.append(row[0])
cursor.close()
@@ -194,7 +194,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select subdir, filename, id from pydar2_specfile where specrepoid=" + str(specRepo.getId())
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
arr = cursor.fetchall()
for row in arr:
@@ -215,7 +215,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select version from pydar2_specfile_version where specfileid=" + str(specId)
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
row= cursor.fetchone()
retval = row[0]
@@ -234,7 +234,7 @@
sql = sql + " specrepoid=" + str(specRepo.getId()) + ", "
sql = sql + " checksum='" + str(specRepoSpecFile.getCheckSum()) + "' "
sql = sql + " where id=" + str(specRepoSpecFile.getId())
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
else:
# new spec file, insert it
@@ -243,10 +243,11 @@
sql = sql + "'" + str(specRepoSpecFile.getFileName()) + "',"
sql = sql + "'" + str(specRepoSpecFile.getSubDir()) + "',"
sql = sql + "'" + str(specRepoSpecFile.getCheckSum()) + "')"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
# now get the id so we can return it at the end
- cursor.execute("select id from pydar2_specfile where filename='" + str(specRepoSpecFile.getFileName()) + "' and subdir='" + str(specRepoSpecFile.getSubDir()) + "' and specrepoid=" + str(specRepo.getId()))
+ sql="select id from pydar2_specfile where filename='" + str(specRepoSpecFile.getFileName()) + "' and subdir='" + str(specRepoSpecFile.getSubDir()) + "' and specrepoid=" + str(specRepo.getId())
+ self.__execute(cursor,sql)
retval = None
if cursor.rowcount > 0:
arr = cursor.fetchall()
@@ -265,7 +266,7 @@
cursor = conn.cursor()
sql = "select version from pydar2_specfile_version where specfileid=" + str(specfileid) + " and version=" + str(currentversion)
self.__cat.debug(sql)
- cursor.execute(sql)
+ self.__execute(cursor,sql)
retval = 0
if cursor.rowcount > 0:
retval = 1
@@ -276,7 +277,8 @@
def __insertSpecRepositorySpecfileVersion(self,specfileid,currentversion):
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("insert into pydar2_specfile_version (specfileid,version) values (" + str(specfileid) + "," + str(currentversion) + ")")
+ sql="insert into pydar2_specfile_version (specfileid,version) values (" + str(specfileid) + "," + str(currentversion) + ")"
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -310,7 +312,7 @@
sql = "select pd.id, pd.distroarchtag, defines, (select count(*) from pydar2_specfile_tags t, pydar2_specfile_version v where t.distroid=pd.id and t.versionid=v.id and v.specfileid=" + str(specRepoSpecFile.getId()) + " and v.version=" + str(currentversion) + ") as cnt from pydar2_distroarch pd"
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute(sql)
+ self.__execute(cursor,sql)
rsdistros = cursor.fetchall()
for row in rsdistros:
distroid=row[0]
@@ -332,7 +334,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select distroarchtag, defines from pydar2_distroarch where disabled='0'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
rsdistros = cursor.fetchall()
for row in rsdistros:
code = row[0]
@@ -351,7 +353,7 @@
# first get the versionid
sql = "select id from pydar2_specfile_version where specfileid=" + str(specRepoSpecFile.getId()) + " and version=" + str(currentversion)
versionid = None
- cursor.execute(sql)
+ self.__execute(cursor,sql)
versionid=""
rstmp = cursor.fetchall()
for row in rstmp:
@@ -384,34 +386,38 @@
#print 'value: ' + sf.rpmforgetags[tag]
value = string.replace(sf.rpmforgetags[tag],"'","''")
value = string.rstrip(value,"\\")
- cursor.execute("insert into pydar2_specfile_tags (versionid,distroid,name,value) values (" + str(versionid) + ",(select id from pydar2_distroarch where distroarchtag='" + str(distrocode) + "'),'"+ str(name) + "','" + str(value) + "')")
+ sql="insert into pydar2_specfile_tags (versionid,distroid,name,value) values (" + str(versionid) + ",(select id from pydar2_distroarch where distroarchtag='" + str(distrocode) + "'),'"+ str(name) + "','" + str(value) + "')"
+ self.__execute(cursor,sql)
for tag in sf.standardtags:
print 'standard tag: ' + tag
name = tag
# print 'value: ' + string.replace(sf.standardtags[tag],"'","''")
value = string.replace(sf.standardtags[tag],"'","''")
value = string.rstrip(value,"\\")
- cursor.execute("insert into pydar2_specfile_tags (versionid,distroid,name,value) values (" + str(versionid) + ",(select id from pydar2_distroarch where distroarchtag='" + str(distrocode) + "'),'"+ str(name) + "','" + str(value) + "')")
+ sql="insert into pydar2_specfile_tags (versionid,distroid,name,value) values (" + str(versionid) + ",(select id from pydar2_distroarch where distroarchtag='" + str(distrocode) + "'),'"+ str(name) + "','" + str(value) + "')"
+ self.__execute(cursor,sql)
def saveSpecRepository(self,specRepo):
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("select * from pydar2_specrepo where name='" + str(specRepo.getName()) + "'")
+ sql="select * from pydar2_specrepo where name='" + str(specRepo.getName()) + "'"
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
# specrepo exists, update it
sql = "update pydar2_specrepo set rootdirectory='" + str(specRepo.getRootDirectory()) + "', type='"
sql = sql + str(specRepo.getType()) + "', deleted='0' where name='" + str(specRepo.getName()) + "'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
else:
# specrepo does not exist, add it
sql = "insert into pydar2_specrepo (name, rootdirectory, type) values ('" + str(specRepo.getName()) + "','" + str(specRepo.getRootDirectory()) + "','" + str(specRepo.getType()) + "')"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
# now get the id again
- cursor.execute("select id from pydar2_specrepo where name='" + str(specRepo.getName()) + "'")
+ sql="select id from pydar2_specrepo where name='" + str(specRepo.getName()) + "'"
+ self.__execute(cursor,sql)
retval = None
arr = cursor.fetchall()
for row in arr:
@@ -421,12 +427,21 @@
return retval
# @todo add some type of connection pooling
+ myConnection = None
+ #currentConnectionUser = None
+
def __getConnection(self):
- conn = pgdb.connect(config.Config.getInstance().getPostgresqlConnectString())
- return conn
+ if PostgresqlStorage.myConnection == None:
+ PostgresqlStorage.myConnection = pgdb.connect(config.Config.getInstance().getPostgresqlConnectString())
+ #PostgresqlStorage.currentConnectionUser = name
+ return PostgresqlStorage.myConnection
+ else:
+ #if PostgresqlStorage.currentConnectionUser
+ return PostgresqlStorage.myConnection
def __releaseConnection(self,conn):
- conn.close()
+ closes = 0
+ #conn.close()
def getTargetFileListToChecksums(self, targetObj):
self.__cat.debug("start")
@@ -434,7 +449,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select filename, checksum from pydar2_target_files where targetid=" + str(targetObj.getId())
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
arr = cursor.fetchall()
for row in arr:
@@ -453,7 +468,7 @@
cursor = conn.cursor()
sql = "select subdir, filename, checksum from pydar2_specfile where specrepoid "
sql = sql + "in (select id from pydar2_specrepo where name='" + specRepo.getName() + "')"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
arr = cursor.fetchall()
for row in arr:
@@ -470,13 +485,14 @@
self.__cat.debug("start")
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("select nextval('pydar2_commands_id_seq')")
+ sql="select nextval('pydar2_commands_id_seq')"
+ self.__execute(cursor,sql)
id = -1
temparr = cursor.fetchall()
for row in temparr:
id = row[0]
#commandName,userId,specrepoid,specfileid,toEmail,distroArchTag,priority,targetid
- cursor.execute("insert into pydar2_commands (id,commandname,userid,specrepoid,specfileid,toemail,distroarchtag,priority,targetid,version) values ("
+ sql="insert into pydar2_commands (id,commandname,userid,specrepoid,specfileid,toemail,distroarchtag,priority,targetid,version) values ("
+ str(id) + ",'"
+ aCommand.getCommandName() + "','"
+ str(aCommand.getUserId()) + "',"
@@ -486,7 +502,8 @@
+ str(aCommand.getDistroArchTag()) + "',"
+ str(aCommand.getPriority()) + ","
+ str(aCommand.getTargetId()) + ","
- + "(select max(version) from pydar2_specfile_version sv where sv.specfileid="+str(aCommand.getSpecFileId())+"))")
+ + "(select max(version) from pydar2_specfile_version sv where sv.specfileid="+str(aCommand.getSpecFileId())+"))"
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -498,7 +515,8 @@
retval = 0
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("select * from pydar2_users u, pydar2_rights r, pydar2_users_to_rights ur where u.id=ur.userid and ur.rightid=r.id and r.abbrev='"+ str(right) + "' and u.id='" + str(userId) + "' and u.password='" + str(password) + "'")
+ sql="select * from pydar2_users u, pydar2_rights r, pydar2_users_to_rights ur where u.id=ur.userid and ur.rightid=r.id and r.abbrev='"+ str(right) + "' and u.id='" + str(userId) + "' and u.password='" + str(password) + "'"
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
retval = 1
cursor.close()
@@ -511,7 +529,8 @@
conn = self.__getConnection()
retval = 0
cursor = conn.cursor()
- cursor.execute("select * from pydar2_buildmachines u, pydar2_rights r, pydar2_buildmachines_to_rights ur where u.id=ur.buildmachineid and ur.rightid=r.id and r.abbrev='"+ str(right) + "' and u.id='" + str(buildmachineid) + "' and u.password='"+ str(password) + "'")
+ sql="select * from pydar2_buildmachines u, pydar2_rights r, pydar2_buildmachines_to_rights ur where u.id=ur.buildmachineid and ur.rightid=r.id and r.abbrev='"+ str(right) + "' and u.id='" + str(buildmachineid) + "' and u.password='"+ str(password) + "'"
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
retval = 1
cursor.close()
@@ -525,15 +544,15 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select * from pydar2_targets where name='" + newTarget.getName() + "'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount == 0:
sql = "insert into pydar2_targets (name) values ('" + newTarget.getName() + "')"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
# else:
# nothing to update yet..
sql = "select id from pydar2_targets where name='" + newTarget.getName() + "'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
newId = None
if cursor.rowcount > 0:
rs = cursor.fetchone()
@@ -546,8 +565,10 @@
self.__cat.debug("start, buildmachineid=" + buildmachineid)
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("insert into pydar2_buildmachine_registrations (id) values ('" + buildmachineid + "')")
- cursor.execute("delete from pydar2_buildmachine_distroarchtags where buildmachineid='" + buildmachineid + "'")
+ sql="insert into pydar2_buildmachine_registrations (id) values ('" + buildmachineid + "')"
+ self.__execute(cursor,sql)
+ sql="delete from pydar2_buildmachine_distroarchtags where buildmachineid='" + buildmachineid + "'"
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -556,7 +577,8 @@
self.__cat.debug("start, id=" + buildmachineid + ",distroArchTag=" + distroArchTag)
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("insert into pydar2_buildmachine_distroarchtags (buildmachineid,distroarchtag) values ('" + buildmachineid + "','" + distroArchTag + "')")
+ sql="insert into pydar2_buildmachine_distroarchtags (buildmachineid,distroarchtag) values ('" + buildmachineid + "','" + distroArchTag + "')"
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -565,7 +587,8 @@
# no check on buildmachineid yet...
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("insert into pydar2_actions_files (commandid,filename) values (" + str(commandId) + ",'" + fileName + "')")
+ sql="insert into pydar2_actions_files (commandid,filename) values (" + str(commandId) + ",'" + fileName + "')"
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -573,8 +596,10 @@
def setBuildResult(self,buildmachineid,commandId,buildResult):
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("insert into pydar2_actions (buildmachineid,commandid,statusid) values ('" + buildmachineid + "'," + str(commandId) + "," + str(buildResult+1) + ")")
- cursor.execute("update pydar2_commands set handled=true,inprogress=false where id=" + str(commandId))
+ sql="insert into pydar2_actions (buildmachineid,commandid,statusid) values ('" + buildmachineid + "'," + str(commandId) + "," + str(buildResult+1) + ")"
+ self.__execute(cursor,sql)
+ sql="update pydar2_commands set handled=true,inprogress=false where id=" + str(commandId)
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -583,7 +608,8 @@
retval = None
conn = self.__getConnection()
cursor = conn.cursor()
- cursor.execute("select count(*) as total from pydar2_commands where not pc.handled and not pc.inprogress")
+ sql="select count(*) as total from pydar2_commands where not pc.handled and not pc.inprogress"
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
row = cursor.fetchone()
retval = row[0]
@@ -598,7 +624,7 @@
cursor = conn.cursor()
sql = "select pc.id, pc.specfileid, pc.specrepoid, pc.toemail, pc.distroarchtag, pc.targetid, pc.commandname, pc.userid, pc.priority, pc.version from pydar2_commands pc where pc.commandname='BUILDSPECBYPATH' and not pc.handled and not pc.inprogress and pc.distroarchtag in (select distroarchtag from pydar2_buildmachine_distroarchtags where buildmachineid='" + buildmachineid + "') order by pc.priority desc, pc.distroarchtag desc, pc.id asc limit 1"
self.__cat.debug("sql 1: " + sql)
- cursor.execute(sql)
+ self.__execute(cursor,sql)
retval = ""
if cursor.rowcount > 0:
rs = cursor.fetchone()
@@ -618,8 +644,10 @@
self.__cat.debug("command id: " + str(rs[0]))
aCommand.setCommandId(rs[0])
self.__cat.debug("id according to command object: " + str(aCommand.getId()))
- cursor.execute("update pydar2_commands set inprogress=true where id=" + str(aCommand.getId()))
- cursor.execute("insert into pydar2_actions (commandid, statusid, buildmachineid) values (" + str(aCommand.getId()) + ",0,'" + str(buildmachineid) + "')")
+ sql="update pydar2_commands set inprogress=true where id=" + str(aCommand.getId())
+ self.__execute(cursor,sql)
+ sql="insert into pydar2_actions (commandid, statusid, buildmachineid) values (" + str(aCommand.getId()) + ",0,'" + str(buildmachineid) + "')"
+ self.__execute(cursor,sql)
retval = aCommand
else:
self.__cat.debug("nothing to do for buildmachineid=" + buildmachineid)
@@ -635,7 +663,7 @@
cursor = conn.cursor()
sql = "select pc.id, pc.specfileid, pc.specrepoid, pc.toemail, pc.distroarchtag, pc.targetid, pc.commandname, pc.userid, pc.priority, pc.version from pydar2_commands pc where pc.id=" + str(commandId)
self.__cat.debug("sql 1: " + sql)
- cursor.execute(sql)
+ self.__execute(cursor,sql)
retval = ""
if cursor.rowcount > 0:
self.__cat.debug("rowcount > 0")
@@ -668,7 +696,7 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select buildmachineid, count(buildmachineid) as cnt from pydar2_actions where commandid=" + str(commandId) + " group by buildmachineid order by cnt desc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
row = cursor.fetchone()
retval = row[0]
@@ -681,7 +709,7 @@
cursor = conn.cursor()
retval = []
sql = "select id from pydar2_specfile_version where specfileid=" + str(specRepoSpecFile.getId()) + " order by id desc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
all = cursor.fetchall()
for row in all:
retval.append(row[0])
@@ -695,7 +723,7 @@
cursor = conn.cursor()
retval = []
sql = "select t1.name, t1.value, d.distroarchtag from pydar2_specfile_tags t1, pydar2_distroarch d where t1.distroid=d.id and versionid=" + str(versionId) + " order by t1.name asc, t1.distroid asc"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
for row in cursor.fetchall():
h = {}
h["name"] = row[0]
@@ -706,13 +734,36 @@
self.__releaseConnection(conn)
return retval
+ # returns a hash of hashes with first key distroarchtag with as value a hash with key = name and val= value
+ def getTagsOfSpecFileVersionNew(self, versionId):
+ conn = self.__getConnection()
+ cursor = conn.cursor()
+ retval = {}
+ sql = "select t1.name, t1.value, d.distroarchtag from pydar2_specfile_tags t1, pydar2_distroarch d where t1.distroid=d.id and versionid=" + str(versionId) + " order by t1.distroid asc, t1.name asc"
+ self.__execute(cursor,sql)
+ for row in cursor.fetchall():
+ name = row[0]
+ value = row[1]
+ distroarchtag = row[2]
+ if distroarchtag not in retval.keys():
+ retval[distroarchtag] = {}
+ # autofill certain values
+ retval[distroarchtag]['# Screenshot'] = ""
+ retval[distroarchtag]['# Authority'] = ""
+ retval[distroarchtag][name] = value
+ cursor.close()
+ self.__releaseConnection(conn)
+ return retval
+
+
+
# get a mapping from something like fc3-i386 to --define fc3 1 --define dist fc3 for example
def getDefinesByDistroArchTag(self,distroArchTag):
conn = self.__getConnection()
cursor = conn.cursor()
sql = "select defines from pydar2_distroarch where distroarchtag='" + distroArchTag + "'"
retval = ""
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
rs = cursor.fetchone()
retval = rs[0]
@@ -724,9 +775,9 @@
conn = self.__getConnection()
cursor = conn.cursor()
sql = "delete from pydar2_users_to_rights"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
sql = "delete from pydar2_buildmachines_to_rights"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
@@ -739,13 +790,13 @@
if type=="BUILDMACHINE":
tablename="pydar2_buildmachines"
sql = "select * from " + tablename + " where id='" + name + "'"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
if cursor.rowcount > 0:
# update
sql = "update " + tablename + " set name='" + fullname + "', password='" +str(passw)+ "' where id='" + name + "'"
else:
sql = "insert into " + tablename + " (id,name,password) values ('" + str(name) + "','" + str(fullname) + "','" + str(passw) + "')"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
# add the rights
tablename = "pydar2_users_to_rights"
firstfield = "userid"
@@ -754,7 +805,24 @@
firstfield="buildmachineid"
for r in rights:
sql = "insert into " + str(tablename) + " (" + str(firstfield) + ",rightid) values ('" + str(name) + "',(select id from pydar2_rights where abbrev='" + str(r) + "'))"
- cursor.execute(sql)
+ self.__execute(cursor,sql)
conn.commit()
cursor.close()
self.__releaseConnection(conn)
+
+
+ def printCounters(self):
+ queryKeys = PostgresqlStorage.queryCounter.keys()
+ queryKeys.sort(lambda x,y : PostgresqlStorage.queryCounter[x] > PostgresqlStorage.queryCounter[y])
+ for k in queryKeys:
+ print "k:" + k + ",val:" + str(PostgresqlStorage.queryCounter[k])
+
+ def __execute(self,cursor,sql):
+ if sql in PostgresqlStorage.queryCounter.keys():
+ PostgresqlStorage.queryCounter[sql] = PostgresqlStorage.queryCounter[sql] + 1
+ else:
+ PostgresqlStorage.queryCounter[sql] = 1
+ cursor.execute(sql)
+
+ queryCounter = {}
+
\ No newline at end of file
Modified: trunk/tools/pydar2/pydar/smartbasedbuildroot.py
===================================================================
--- trunk/tools/pydar2/pydar/smartbasedbuildroot.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/pydar/smartbasedbuildroot.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -29,53 +29,30 @@
# -orpm-root=/var/lib/pydar2/roots/ <buildmachineid> / <buildroottag>-smart /
# todo: how to change the dir /etc/smart/channels ?
-# not finished, todo
+# still not finished, todo
class SmartBasedBuildRoot(BuildRoot):
def __init__(self,buildRootTag):
- self.__cat = Logger().get_instance(self)
+ self._cat = Logger().get_instance(self)
brr = BuildRootResult(self)
self.__config = config.Config.getInstance()
self._setBuildRootTag( buildRootTag)
self.__setVars(brr)
- self.__cat.debug("initialized")
+ self._cat.debug("initialized")
self.__useCacheOnly = False
#return brr
- # todo possible with smart?
- def setUseCacheOnly(self,bVar):
- self.__useCacheOnly = bVar
-
- def _umounts(self, brr):
- self.__cat.debug("start")
- cmd = "sudo umount " + self.rootdir + "/proc"
- self.logCommandWithoutErrors(cmd,brr)
- cmd = "sudo umount " + self.rootdir + "/sys"
- self.logCommandWithoutErrors(cmd,brr)
-
- def cleanRoot(self):
- self.__cat.debug("cleaning this root..")
- brr = BuildRootResult(self)
- self._umounts(brr)
- cmd = "sudo rm -Rf " + self.rootdir
- self.logCommand(cmd,brr)
- return brr
-
def __setVars(self,brr):
- self.yumConfFile = "/etc/pydar2/smart/" + self.__config.getBuildMachineId() + "/" + self.getBuildRootTag() + ".conf"
- if os.path.exists(self.yumConfFile):
- self.__cat.debug("using userdefined yum conf: " + self.yumConfFile)
+ self.smartChannelsDir = "/etc/pydar2/smart/" + self.__config.getBuildMachineId() + "/" + self.getBuildRootTag() + ""
+ if os.path.exists(self.smartChannelsDir):
+ self._cat.debug("using userdefined smart channels dir: " + self.smartChannelsDir)
else:
- self.yumConfFile = "/etc/pydar2/yum/" + self.getBuildRootTag() + ".conf"
- self.__cat.debug("using normal yum conf: " + self.yumConfFile)
- self.yumCacheDir = "/var/lib/pydar2/yum/cache/" + self.getBuildRootTag()
- self.rootdir = "/var/lib/pydar2/roots/yum-" + self.getBuildRootTag()
+ self.smartChannelsDir = "/etc/pydar2/smart/" + self.getBuildRootTag() + ""
+ self._cat.debug("using normal smart channels dir: " + self.smartChannelsDir)
+ self.smartCacheDir = "/var/lib/pydar2/smart/cache/" + self.getBuildRootTag()
+ self.rootdir = "/var/lib/pydar2/roots/smart-" + self.getBuildRootTag()
- # this definetely has to move to a config file
- self.minimalRpmsPart1 = "basesystem filesystem setup bash glibc"
- self.minimalRpmsPart2 = "coreutils findutils"
- self.minimalRpmsPart3 = "rpm-devel rpm-build make gcc tar gzip patch unzip bzip2 diffutils cpio elfutils redhat-rpm-config perl"
- self.minimalRpmsPart4 = ""
+ self._setGeneralVars()
brr.logDebugLine("rootdir:" + self.rootdir)
def installRpms(self,builddepsarr):
@@ -88,7 +65,7 @@
cacheParam = ""
if self.__useCacheOnly:
cacheParam = " -C "
- cmd = "sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + commandpart
+ cmd = "sudo smart " + cacheParam + " --data-dir=" + self.smartCacheDir + " -orpm-root=" + self.rootdir + " install " + commandpart
self.logCommand(cmd,brr)
return brr
@@ -97,7 +74,7 @@
brr = BuildRootResult(self)
cmd = []
# create some directories first
- cmd.append("sudo mkdir -p " + self.rootdir + "/var/lib/rpm " + self.rootdir + "/var/log " + self.rootdir + "/var/cache " + self.rootdir + "/etc " + self.rootdir + "/proc " + self.rootdir + "/sys " + self.rootdir + "/usr/src/redhat/BUILD " + self.rootdir + "/usr/src/redhat/RPMS " + self.rootdir + "/usr/src/redhat/SOURCES " + self.rootdir + "/usr/src/redhat/SPECS " + self.rootdir + "/usr/src/redhat/SRPMS " + self.rootdir + "/dev/")
+ cmd.append("sudo mkdir -p " + self.rootdir + "/var/lib/rpm " + self.rootdir + "/var/log " + self.rootdir + "/var/cache " + self.rootdir + "/etc " + self.rootdir + "/proc " + self.rootdir + "/sys " + self.rootdir + "/usr/src/redhat/BUILD " + self.rootdir + "/usr/src/redhat/RPMS " + self.rootdir + "/usr/src/redhat/SOURCES " + self.rootdir + "/usr/src/redhat/SPECS " + self.rootdir + "/usr/src/redhat/SRPMS " + self.rootdir + "/dev/ " + self.rootdir + "/var/lib ")
# copy fstab
cmd.append("sudo cp /etc/fstab " + self.rootdir + "/etc")
# mount proc
@@ -109,143 +86,20 @@
# rpm initdb
cmd.append("sudo rpm --root=" + self.rootdir + " --initdb")
# use a central yum cache
- cmd.append("sudo ln -s " + self.yumCacheDir + " " + self.rootdir + "/var/cache/yum")
+ cmd.append("sudo ln -s " + self.smartCacheDir + " " + self.rootdir + "/var/lib/smart")
# install the rpms
cacheParam = ""
if self.__useCacheOnly:
cacheParam = " -C "
if self.minimalRpmsPart1 != "":
- cmd.append("sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + self.minimalRpmsPart1)
+ cmd.append("sudo smart " + cacheParam + " --data-dir=" + self.smartCacheDir + " -orpm-root=" + self.rootdir + " install " + self.minimalRpmsPart1)
if self.minimalRpmsPart2 != "":
- cmd.append("sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + self.minimalRpmsPart2)
+ cmd.append("sudo smart " + cacheParam + " --data-dir=" + self.smartCacheDir + " -orpm-root=" + self.rootdir + " install " + self.minimalRpmsPart2)
if self.minimalRpmsPart3 != "":
- cmd.append("sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + self.minimalRpmsPart3)
+ cmd.append("sudo smart " + cacheParam + " --data-dir=" + self.smartCacheDir + " -orpm-root=" + self.rootdir + " install " + self.minimalRpmsPart3)
if self.minimalRpmsPart4 != "":
- cmd.append("sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + self.minimalRpmsPart4)
- cmd.append("sudo chmod -R a+rwx " + self.rootdir + "/usr/src/redhat/")
+ cmd.append("sudo smart " + cacheParam + " --data-dir=" + self.smartCacheDir + " -orpm-root=" + self.rootdir + " install " + self.minimalRpmsPart4)
+ cmd.append("sudo chmod -R a+rwx " + self.rootdir + "/usr/src/redhat/ " + self.rootdir + "/home/pydar2slave ")
self.logCommands(cmd,brr)
return brr
- def logCommands(self,cmdArr,brr):
- for cmd in cmdArr:
- self.logCommand(cmd,brr)
- # todo: if error => stop
-
- def logCommand(self,cmd,brr):
- brr.logDebugLine("CMD: Command line:" + cmd)
- (status,output) = commands.getstatusoutput(cmd)
- brr.logDebugLine("CMD: Status:" + str(status))
- # first remove all the '\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*
- output = re.sub('\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*',"",output)
- output = re.sub('\r',"",output)
- brr.logDebugLine("CMD: output:" + str(output))
- if status != 0:
- brr.logErrorLine("CMD: status (" + str(status) + ") is not 0 !")
-
- def logCommandWithoutErrors(self,cmd,brr):
- brr.logDebugLine("CMD: Command line:" + cmd)
- (status,output) = commands.getstatusoutput(cmd)
- brr.logDebugLine("CMD: Status:" + str(status))
- # first remove all the '\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*
- output = re.sub('\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*',"",output)
- output = re.sub('\r',"",output)
- brr.logDebugLine("CMD: output:" + str(output))
-
- def buildRpm(self,command):
- brr = BuildRootResult(self)
- self.__cat.debug("start")
- # copy the sources
- self.__copySources(command,brr)
- # create a script which does the actual building
- scriptcontents = self.__createScript(command,brr)
- # then we call the script
- self.__callBuildScript(scriptcontents, command, brr)
- # get all the results
- self.__getLogsAndRpmsAndSrpm(command,brr)
- # then we parse the log: get additional usefull files if need, for example config.log if the build has failed
- self.__parseLog(command,brr)
- return brr
-
- def __copySources(self,command,brr):
- self.__cat.debug("start")
- startdir = command.getBuildDir()
- listdir = os.listdir(startdir)
- specdestdir = os.path.join(self.rootdir,'usr/src/redhat/SPECS')
- sourcesdestdir = os.path.join(self.rootdir,'usr/src/redhat/SOURCES')
- self.__cat.debug('specdestdir: ' + specdestdir)
- self.__cat.debug('sourcesdestdir: ' + sourcesdestdir)
- for entry in listdir:
- rindex = string.rfind(entry,'.spec')
- if os.path.isfile(os.path.join(startdir,entry)) and not (rindex > 0 and rindex == len(entry) - len('.spec')):
- # its not the spec files
- shutil.copy(os.path.join(startdir,entry), sourcesdestdir)
- else:
- # its the spec file
- shutil.copy(os.path.join(startdir,entry),specdestdir)
- self.__cat.debug('copy done')
-
- def __getLogsAndRpmsAndSrpm(self,command,brr):
- self.__cat.debug("start")
- shutil.copy(os.path.join(self.rootdir, "buildlog.txt"), command.getBuildDir())
- cmd = "gzip " + command.getBuildDir() + "/buildlog.txt"
- self.logCommand(cmd,brr)
- # search in the /usr/src/redhat/RPMS and /usr/src/redhat/SRPMS dirs..
- rpmsdir = os.path.join(self.rootdir,'usr/src/redhat/RPMS')
- ld = os.listdir(rpmsdir)
- for entry in ld:
- self.__getRpmsOrSrpms(command,brr,os.path.join(rpmsdir,entry))
- srpmsdir = os.path.join(self.rootdir,'usr/src/redhat/SRPMS')
- self.__getRpmsOrSrpms(command,brr,srpmsdir)
-
- def __getRpmsOrSrpms(self,command,brr,path):
- self.__cat.debug('start, path=' + path)
- ld = os.listdir(path)
- for entry in ld:
- rindex = string.rfind(entry,'.rpm')
- if os.path.isfile(os.path.join(path,entry)) and (rindex > 0 and rindex == len(entry) - len('.rpm')):
- # let's copy it
- shutil.copy(os.path.join(path,entry),command.getBuildDir())
- rindex = string.rfind(entry,'.src.rpm')
- if rindex > 0 and rindex == len(entry) - len('.src.rpm'):
- brr.setSrpm(entry)
- else:
- brr.addRpm(entry)
-
- def __createScript(self,command,brr):
- self.__cat.debug("start")
- script = "#!/bin/bash\n"
- script = script + 'for i in ' + self.rootdir + '/etc/profile.d/*.sh; do' + "\n"
- script = script + ' if [ -r "$i" ]; then' + "\n"
- script = script + ' . $i' + "\n"
- script = script + " fi\n"
- script = script + "done\n"
- script = script + "chown -R root.root " + self.rootdir + "/usr/src/redhat\n"
- # the following command first checks if the dependencies are ok
- # ex: --define '_topdir /var/lib/pydar2/roots/yum-fc3-i386/usr/src/redhat'
- script = script + "(/usr/bin/rpmbuild --define '_topdir " + self.rootdir + "/usr/src/redhat' " + command.getDefines() + " --root=" + self.rootdir + " --nobuild --bs " + self.rootdir + "/usr/src/redhat/SPECS/" + command.getSpecFileShortFileName() + " && echo \"deps check ok\" && "
- # do the real build
- script = script + "/usr/sbin/chroot " + self.rootdir + " /usr/bin/rpmbuild --nodeps -ba /usr/src/redhat/SPECS/" + command.getSpecFileShortFileName() + " " + command.getDefines() + " ) > " + self.rootdir + "/buildlog.txt 2>&1"
- self.__cat.debug( "script contains: " + script)
- return script
-
- def __callBuildScript(self, scriptcontents, command, brr):
- self.__cat.debug("start")
- # first write it to a temp file
- filepath = self.rootdir + "/tmp/pydar2build.sh"
- fd = open(filepath,"w")
- fd.write(scriptcontents)
- fd.close()
- cmd = "sudo /bin/bash " + self.rootdir + "/tmp/pydar2build.sh"
- self.logCommand(cmd,brr)
-
- def __parseLog(self,command,brr):
- self.__cat.debug("start, not ready")
-
-## tests
-#myRoot = YumBasedBuildRoot("","fedora-3-i386-core")
-#myRoot.cleanRoot()
-#myRoot.prepareRoot()
-#spec = SpecFile('/home/dries/projects/rap-svn/svn/trunk/rpms/aldo/aldo.spec',' --define "fc3 1" ')
-#myRoot.installRpms(spec.getBuildRequires())
-##myRoot.
-#print myRoot.logText
Modified: trunk/tools/pydar2/pydar/svnwrapper.py
===================================================================
--- trunk/tools/pydar2/pydar/svnwrapper.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/pydar/svnwrapper.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -94,6 +94,9 @@
def update_callback(self):
self.cat.debug("start")
+ # changed!
+ # now -1 is returned if it is not within svn
+
def getLastCommitRev(self,dirpath,fullpath):
#if dirpath[len(dirpath)-1] == "/":
# self.cat.debug("removing last '/'")
@@ -101,12 +104,16 @@
#else:
# self.cat.debug("last char:" + dirpath[len(dirpath)-1])
self.cat.debug("start,dirpath=" + dirpath + ",fullpath="+fullpath)
- adm_baton = svn.wc.svn_wc_adm_open(None,dirpath,False,True,self.pool)
- entry = svn.wc.svn_wc_entry(fullpath,adm_baton,0,self.pool)
- self.cat.debug("entry: " + str(entry))
- self.cat.debug("cmtRev: " + str(entry.cmt_rev))
- svn.wc.svn_wc_adm_close(adm_baton)
- return entry.cmt_rev
+ try:
+ adm_baton = svn.wc.svn_wc_adm_open(None,dirpath,False,True,self.pool)
+ entry = svn.wc.svn_wc_entry(fullpath,adm_baton,0,self.pool)
+ self.cat.debug("entry: " + str(entry))
+ self.cat.debug("cmtRev: " + str(entry.cmt_rev))
+ svn.wc.svn_wc_adm_close(adm_baton)
+ return entry.cmt_rev
+ except:
+ self.cat.debug("problem, probably not in svn")
+ return -1
# tocheck: http://svn.slog.dk/repos/+svn/py/util.py
# http://www.contactor.se/~dast/svn/archive-2002-12/0234.shtml
Modified: trunk/tools/pydar2/pydar/yumbasedbuildroot.py
===================================================================
--- trunk/tools/pydar2/pydar/yumbasedbuildroot.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/pydar/yumbasedbuildroot.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -26,48 +26,27 @@
class YumBasedBuildRoot(BuildRoot):
def __init__(self,buildRootTag):
- self.__cat = Logger().get_instance(self)
+ self._cat = Logger().get_instance(self)
brr = BuildRootResult(self)
self.__config = config.Config.getInstance()
self._setBuildRootTag( buildRootTag)
self.__setVars(brr)
- self.__cat.debug("initialized")
+ self._cat.debug("initialized")
self.__useCacheOnly = False
#return brr
- def setUseCacheOnly(self,bVar):
- self.__useCacheOnly = bVar
-
- def _umounts(self, brr):
- self.__cat.debug("start")
- cmd = "sudo umount " + self.rootdir + "/proc"
- self.logCommandWithoutErrors(cmd,brr)
- cmd = "sudo umount " + self.rootdir + "/sys"
- self.logCommandWithoutErrors(cmd,brr)
- def cleanRoot(self):
- self.__cat.debug("cleaning this root..")
- brr = BuildRootResult(self)
- self._umounts(brr)
- cmd = "sudo rm -Rf " + self.rootdir
- self.logCommand(cmd,brr)
- return brr
-
def __setVars(self,brr):
self.yumConfFile = "/etc/pydar2/yum/" + self.__config.getBuildMachineId() + "/" + self.getBuildRootTag() + ".conf"
if os.path.exists(self.yumConfFile):
- self.__cat.debug("using userdefined yum conf: " + self.yumConfFile)
+ self._cat.debug("using userdefined yum conf: " + self.yumConfFile)
else:
self.yumConfFile = "/etc/pydar2/yum/" + self.getBuildRootTag() + ".conf"
- self.__cat.debug("using normal yum conf: " + self.yumConfFile)
+ self._cat.debug("using normal yum conf: " + self.yumConfFile)
self.yumCacheDir = "/var/lib/pydar2/yum/cache/" + self.getBuildRootTag()
self.rootdir = "/var/lib/pydar2/roots/yum-" + self.getBuildRootTag()
- # this definetely has to move to a config file
- self.minimalRpmsPart1 = "basesystem filesystem setup bash glibc"
- self.minimalRpmsPart2 = "coreutils findutils"
- self.minimalRpmsPart3 = "rpm-devel rpm-build make gcc tar gzip patch unzip bzip2 diffutils cpio elfutils redhat-rpm-config perl"
- self.minimalRpmsPart4 = ""
+ self._setGeneralVars()
brr.logDebugLine("rootdir:" + self.rootdir)
def installRpms(self,builddepsarr):
@@ -87,21 +66,7 @@
# makes a new root
def prepareRoot(self):
brr = BuildRootResult(self)
- cmd = []
- # create some directories first
- cmd.append("sudo mkdir -p " + self.rootdir + "/var/lib/rpm " + self.rootdir + "/var/log " + self.rootdir + "/var/cache " + self.rootdir + "/etc " + self.rootdir + "/proc " + self.rootdir + "/sys " + self.rootdir + "/usr/src/redhat/BUILD " + self.rootdir + "/usr/src/redhat/RPMS " + self.rootdir + "/usr/src/redhat/SOURCES " + self.rootdir + "/usr/src/redhat/SPECS " + self.rootdir + "/usr/src/redhat/SRPMS " + self.rootdir + "/dev " + self.rootdir + "/home/pydar2slave ")
- # copy fstab
- cmd.append("sudo cp /etc/fstab " + self.rootdir + "/etc")
- # create hosts
- cmd.append("sudo echo -e \"127.0.0.1 localhost.localdomain localhost\" > " + self.rootdir + "/etc/hosts")
- # mount proc
- cmd.append("sudo mount -t proc none " + self.rootdir + "/proc")
- # mount sys
- cmd.append("sudo mount -t sysfs none " + self.rootdir + "/sys")
- # create /dev/null
- cmd.append("sudo mknod " + self.rootdir + "/dev/null c 1 3")
- # rpm initdb
- cmd.append("sudo rpm --root=" + self.rootdir + " --initdb")
+ cmd = self._getGeneralPrepareRootCommands()
# use a central yum cache
cmd.append("sudo mkdir -p " + self.yumCacheDir)
cmd.append("sudo ln -s " + self.yumCacheDir + " " + self.rootdir + "/var/cache/yum")
@@ -117,129 +82,12 @@
cmd.append("sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + self.minimalRpmsPart3)
if self.minimalRpmsPart4 != "":
cmd.append("sudo yum " + cacheParam + " -c " + self.yumConfFile + " -y -t --installroot=" + self.rootdir + " install " + self.minimalRpmsPart4)
- cmd.append("sudo chmod -R a+rwx " + self.rootdir + "/usr/src/redhat/")
+ cmd.append("sudo chmod -R a+rwx " + self.rootdir + "/usr/src/redhat/ " + self.rootdir + "/home/pydar2slave ")
self.logCommands(cmd,brr)
return brr
- def logCommands(self,cmdArr,brr):
- for cmd in cmdArr:
- self.logCommand(cmd,brr)
- # todo: if error => stop
- def logCommand(self,cmd,brr):
- brr.logDebugLine("CMD: Command line:" + cmd)
- (status,output) = commands.getstatusoutput(cmd)
- brr.logDebugLine("CMD: Status:" + str(status))
- # first remove all the '\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*
- output = re.sub('\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*',"",output)
- output = re.sub('\r',"",output)
- brr.logDebugLine("CMD: output:" + str(output))
- if status != 0:
- brr.logErrorLine("CMD: status (" + str(status) + ") is not 0 !")
-
- def logCommandWithoutErrors(self,cmd,brr):
- brr.logDebugLine("CMD: Command line:" + cmd)
- (status,output) = commands.getstatusoutput(cmd)
- brr.logDebugLine("CMD: Status:" + str(status))
- # first remove all the '\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*
- output = re.sub('\rInstalling: .*? [0-9][0-9]? % done [0-9]*\/[0-9]*',"",output)
- output = re.sub('\r',"",output)
- brr.logDebugLine("CMD: output:" + str(output))
-
- def buildRpm(self,command):
- brr = BuildRootResult(self)
- self.__cat.debug("start")
- # copy the sources
- self.__copySources(command,brr)
- # create a script which does the actual building
- scriptcontents = self.__createScript(command,brr)
- # then we call the script
- self.__callBuildScript(scriptcontents, command, brr)
- # get all the results
- self.__getLogsAndRpmsAndSrpm(command,brr)
- # then we parse the log: get additional usefull files if need, for example config.log if the build has failed
- self.__parseLog(command,brr)
- return brr
- def __copySources(self,command,brr):
- self.__cat.debug("start")
- startdir = command.getBuildDir()
- listdir = os.listdir(startdir)
- specdestdir = os.path.join(self.rootdir,'usr/src/redhat/SPECS')
- sourcesdestdir = os.path.join(self.rootdir,'usr/src/redhat/SOURCES')
- self.__cat.debug('specdestdir: ' + specdestdir)
- self.__cat.debug('sourcesdestdir: ' + sourcesdestdir)
- for entry in listdir:
- rindex = string.rfind(entry,'.spec')
- if os.path.isfile(os.path.join(startdir,entry)) and not (rindex > 0 and rindex == len(entry) - len('.spec')):
- # its not the spec files
- shutil.copy(os.path.join(startdir,entry), sourcesdestdir)
- else:
- # its the spec file
- shutil.copy(os.path.join(startdir,entry),specdestdir)
- self.__cat.debug('copy done')
-
- def __getLogsAndRpmsAndSrpm(self,command,brr):
- self.__cat.debug("start")
- shutil.copy(os.path.join(self.rootdir, "buildlog.txt"), command.getBuildDir())
- cmd = "gzip " + command.getBuildDir() + "/buildlog.txt"
- self.logCommand(cmd,brr)
- # search in the /usr/src/redhat/RPMS and /usr/src/redhat/SRPMS dirs..
- rpmsdir = os.path.join(self.rootdir,'usr/src/redhat/RPMS')
- ld = os.listdir(rpmsdir)
- for entry in ld:
- self.__getRpmsOrSrpms(command,brr,os.path.join(rpmsdir,entry))
- srpmsdir = os.path.join(self.rootdir,'usr/src/redhat/SRPMS')
- self.__getRpmsOrSrpms(command,brr,srpmsdir)
-
- def __getRpmsOrSrpms(self,command,brr,path):
- self.__cat.debug('start, path=' + path)
- ld = os.listdir(path)
- for entry in ld:
- rindex = string.rfind(entry,'.rpm')
- if os.path.isfile(os.path.join(path,entry)) and (rindex > 0 and rindex == len(entry) - len('.rpm')):
- # let's copy it
- shutil.copy(os.path.join(path,entry),command.getBuildDir())
- rindex = string.rfind(entry,'.src.rpm')
- if rindex > 0 and rindex == len(entry) - len('.src.rpm'):
- brr.setSrpm(entry)
- else:
- brr.addRpm(entry)
-
- def __createScript(self,command,brr):
- self.__cat.debug("start")
- script = "#!/bin/bash\n"
- script = script + 'for i in ' + self.rootdir + '/etc/profile.d/*.sh; do' + "\n"
- script = script + ' if [ -r "$i" ]; then' + "\n"
- script = script + ' . $i' + "\n"
- script = script + " fi\n"
- script = script + "done\n"
- script = script + "chown -R root.root " + self.rootdir + "/usr/src/redhat\n"
- # the following command first checks if the dependencies are ok
- # ex: --define '_topdir /var/lib/pydar2/roots/yum-fc3-i386/usr/src/redhat'
- script = script + "(echo file: " + command.getSpecFileShortFileName() + ";/usr/bin/rpmbuild --define '_topdir " + self.rootdir + "/usr/src/redhat' " + command.getDefines() + " --root=" + self.rootdir + " --nobuild --bs " + self.rootdir + "/usr/src/redhat/SPECS/" + command.getSpecFileShortFileName() + " && echo \"deps check ok\" && "
- # do the real build
- # todo: some 'post mortem analysis'
- # if contains error: Installed (but unpackaged) file(s) found: -> print output of a find in the buildroot
- # if it contains a config.log: See 'Config.log' for more details.
- script = script + "/usr/sbin/chroot " + self.rootdir + " /usr/bin/rpmbuild --nodeps -ba /usr/src/redhat/SPECS/" + command.getSpecFileShortFileName() + " " + command.getDefines() + " || "
- script = script + "(echo ; echo ; echo CONFIG.LOG; echo ; cat " + self.rootdir + "/usr/src/redhat/BUILD/*/*onfig.log; echo ; echo ; echo FILES; echo ; echo ; (cd " + self.rootdir + "/var/tmp/*-root*/ && find .))) > " + self.rootdir + "/buildlog.txt 2>&1"
- self.__cat.debug( "script contains: " + script)
- return script
-
- def __callBuildScript(self, scriptcontents, command, brr):
- self.__cat.debug("start")
- # first write it to a temp file
- filepath = self.rootdir + "/tmp/pydar2build.sh"
- fd = open(filepath,"w")
- fd.write(scriptcontents)
- fd.close()
- cmd = "sudo /bin/bash " + self.rootdir + "/tmp/pydar2build.sh"
- self.logCommand(cmd,brr)
-
- def __parseLog(self,command,brr):
- self.__cat.debug("start, not ready")
-
## tests
#myRoot = YumBasedBuildRoot("","fedora-3-i386-core")
#myRoot.cleanRoot()
Modified: trunk/tools/pydar2/scripts/norpmforgetemplatesfilter.py
===================================================================
--- trunk/tools/pydar2/scripts/norpmforgetemplatesfilter.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/scripts/norpmforgetemplatesfilter.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -23,7 +23,13 @@
def NoRpmforgeTemplatesFilter(specRepoSpecFile):
#print "called with obj " + str(specRepoSpecFile)
+
+ # needs to be converted to 1 encoding
+ otherencodings = ["a52dec.spec","acroread.spec","akfquiz.spec","avifile.spec","bind.spec","bluez-bluefw.spec","dd2.spec","directfb.spec","easytag2.spec","easytag.spec","echoping.spec","fslint.spec","gambas-dag.spec","gambas.spec","gammu.spec","gcombust.spec","gkrellm.spec","gkrellm-plugins.spec","gnomeradio.spec","gpm.spec","grip.spec","Gtk-Perl.spec","gtoaster.spec","id3lib.spec","ifstat.spec","ipw2100-firmware.spec","ipw2200-firmware.spec","lftp.spec","libbraille.spec","libdvdread.spec","libsnack.spec","libtar.spec","libtc.spec","libXvMCW.spec","lumiere.spec","mc.spec","memcached.spec","micq.spec","mod_gzip.spec","netscape4.spec","netspeed_applet.spec","ogle_gui.spec","ogle.spec","ogmtools.spec","oidentd.spec","partimage-0.6.spec","partimage.spec","perl-RPM-Specfile.spec","perl-Text-Unaccent.spec","perl-XML-SAX.spec","pine.spec","pointless.spec","qemu.spec","rte.spec","sendmail-old.spec","sendmail.spec","spit.spec","sqlitemanager.spec","tcpreen.spec","transcode.spec","unrar.spec","uudeview.spec","xine-skins.spec","xmame-roms.spec","xmms.spec","xmule.spec","xvattr.spec","zapping.spec"]
+
fn = specRepoSpecFile.getFileName()
+ if fn in otherencodings:
+ return False
if fn == "_template.spec" or fn == "_template-fr.spec" or fn == "_perl-template.spec":
return False
else:
Modified: trunk/tools/pydar2/scripts/rpmforgesite.py
===================================================================
--- trunk/tools/pydar2/scripts/rpmforgesite.py 2005-08-24 07:32:17 UTC (rev 3530)
+++ trunk/tools/pydar2/scripts/rpmforgesite.py 2005-08-24 21:40:06 UTC (rev 3531)
@@ -19,195 +19,260 @@
import sys, string, os, urllib,re ,shutil
from datetime import datetime
+sys.path.append("pydar")
sys.path.append("/usr/share/pydar2/pydar")
-sys.path.append("pydar")
+
sys.path.append(".")
from log4py import Logger
from config import Config
-import storagefactory
+import storagefactory, time
# currently this builds the site at dries.ulyssis.org
# certain parts are added for building the general rpmforge site: these functions start with RF
-def RpmforgeSite(specRepo,useLocalFilesOnly):
- print "start, test"
- so = SiteObject(specRepo,useLocalFilesOnly)
+defaultQueryDistroArch = "el4-i386"
+
+def RpmforgeSite(specRepo,useLocalFilesOnly,buildSiteDries,buildSiteRpmforge):
+ so = SiteObject(specRepo,useLocalFilesOnly,buildSiteDries,buildSiteRpmforge)
so.generate()
+
+stripDistRepoTagRegExpressions = ["0\.el[234]\.dag\..*\.rpm",
+ "0\.el[234]\.1\.dag\..*\.rpm",
+ "rhel[234]\.1\.dag\..*\.rpm",
+ "0\.el[234]\.rf\..*\.rpm",
+ "1\.el3\.dag\..*\.rpm",
+ "dag\.rhel3\..*\.rpm",
+ "rhel3\.dag\..*\.rpm",
+ "1\.el3\.rf\..*\.rpm",
+ "2\.el4\.rf\..*\.rpm",
+ "1\.fc[12]\.dag\..*\.rpm",
+ "rhfc1\.dag\..*\.rpm",
+ "rh80\.dag\..*\.rpm",
+ "dag\.rh80\..*\.rpm",
+ "rh90\.dag\..*\.rpm",
+ "dag\.rh90\..*\.rpm",
+ "0\.au1\.9[12]\..*\.rpm",
+ "1\.fc[123]\.rf\..*\.rpm",
+ "dag\.rhfc1\..*\.rpm",
+ "1\.fc[12]\.dries\..*\.rpm",
+ "2\.fc4\.rf\..*\.rpm",
+ "dag\.rh62\..*\.rpm",
+ "1\.el3\.dries\..*\.rpm",
+ "rh6.\.dag\..*\.rpm",
+ "0\.rh[6789]\.dag\..*\.rpm",
+ "0\.dag\.noarch\.rpm",
+ "rh73\.dag\..*\.rpm",
+ "0\.rh[789]\.rf\..*\.rpm",
+ "dag\.rh73\..*\.rpm",
+ "0\.rf\.noarch\.rpm"]
-class BinaryRpm:
- def __init__(self,distroArchObject,name):
- self.distroArchObject = distroArchObject
- self.name = name
- # let's extract the version-release
- self.verrel = ""
- tempname = name
+
+stripDistRepoTagREs = []
+for exp in stripDistRepoTagRegExpressions:
+ stripDistRepoTagREs.append(re.compile(exp))
+
+
+# First some lists of rpms are constructed. These lists are represented by DistArchPackagerList objects. Such an object
+# represents a list of rpms made by 1 packager for a certain dist/arch combination. For example: el4-i386-dries.
+# The list of rpms of such a list is made automatically from the index page with all the rpms, for example
+# http://apt.sw.be/dries/redhat/el4/i386/en/RPMS.dries
+
+# Each rpm is represented with a RpmPackage object.
+# It contains a link to the distArchPackagerList object where the rpm is found
+
+
+
+class RpmPackage:
+ def setSpecRepoSpecFile(self,specRepoSpecFileObj):
+ self.__specRepoSpecFile = specRepoSpecFileObj
+
+ # only returns the filename without the url
+ def getName(self):
+ return self.__name
+
+ # returns the full url which can be used in a webpage
+ def getFullUrl(self):
+ return self.__distArchPackagerList.rpmsUrl + self.__name
- regex1 = re.compile("0\.el[234]\.dag\..*\.rpm")
- regex2 = re.compile("0\.el[234]\.1\.dag\..*\.rpm")
- regex3 = re.compile("rhel[234]\.1\.dag\..*\.rpm")
- regex4 = re.compile("0\.el[234]\.rf\..*\.rpm")
- regex5 = re.compile("1\.el3\.dag\..*\.rpm")
- regex6 = re.compile("dag\.rhel3\..*\.rpm")
- regex7 = re.compile("rhel3\.dag\..*\.rpm")
- regex8 = re.compile("1\.el3\.rf\..*\.rpm")
- regex9 = re.compile("2\.el4\.rf\..*\.rpm")
- regex10 = re.compile("1\.fc[12]\.dag\..*\.rpm")
- regex11 = re.compile("rhfc1\.dag\..*\.rpm")
- regex12 = re.compile("rh80\.dag\..*\.rpm")
- regex13 = re.compile("dag\.rh80\..*\.rpm")
- regex14 = re.compile("rh90\.dag\..*\.rpm")
- regex15 = re.compile("dag\.rh90\..*\.rpm")
- regex16 = re.compile("0\.au1\.9[12]\..*\.rpm")
- regex17 = re.compile("1\.fc[123]\.rf\..*\.rpm")
- regex18 = re.compile("dag\.rhfc1\..*\.rpm")
- regex19 = re.compile("1\.fc[12]\.dries\..*\.rpm")
- regex20 = re.compile("2\.fc4\.rf\..*\.rpm")
- regex21 = re.compile("dag\.rh62\..*\.rpm")
- regex22 = re.compile("1\.el3\.dries\..*\.rpm")
- regex23 = re.compile("rh6.\.dag\..*\.rpm")
- regex24 = re.compile("0\.rh[6789]\.dag\..*\.rpm")
- regex25 = re.compile("0\.dag\.noarch\.rpm")
- regex26 = re.compile("rh73\.dag\..*\.rpm")
- regex27 = re.compile("0\.rh[789]\.rf\..*\.rpm")
- regex28 = re.compile("dag\.rh73\..*\.rpm")
- regex29 = re.compile("0\.rf\.noarch\.rpm")
- tempname = re.sub(regex1,"",tempname)
- tempname = re.sub(regex2,"",tempname)
- tempname = re.sub(regex3,"",tempname)
- tempname = re.sub(regex4,"",tempname)
- tempname = re.sub(regex5,"",tempname)
- tempname = re.sub(regex6,"",tempname)
- tempname = re.sub(regex7,"",tempname)
- tempname = re.sub(regex8,"",tempname)
- tempname = re.sub(regex9,"",tempname)
- tempname = re.sub(regex10,"",tempname)
- tempname = re.sub(regex11,"",tempname)
- tempname = re.sub(regex12,"",tempname)
- tempname = re.sub(regex13,"",tempname)
- tempname = re.sub(regex14,"",tempname)
- tempname = re.sub(regex15,"",tempname)
- tempname = re.sub(regex16,"",tempname)
- tempname = re.sub(regex17,"",tempname)
- tempname = re.sub(regex18,"",tempname)
- tempname = re.sub(regex19,"",tempname)
- tempname = re.sub(regex20,"",tempname)
- tempname = re.sub(regex21,"",tempname)
- tempname = re.sub(regex22,"",tempname)
- tempname = re.sub(regex23,"",tempname)
- tempname = re.sub(regex24,"",tempname)
- tempname = re.sub(regex25,"",tempname)
- tempname = re.sub(regex26,"",tempname)
- tempname = re.sub(regex27,"",tempname)
- tempname = re.sub(regex28,"",tempname)
- tempname = re.sub(regex29,"",tempname)
+ def getDistArchPackagerList(self):
+ return self.__distArchPackagerList
+ def getVersion(self):
+ return self.__version
+
+ def getRelease(self):
+ return self.__release
+
+ #return version-release
+ def getRelVer(self):
+ return self.__version + "-" + self.__release
+ def __stripDistRepoTag(self,name):
+ for reObj in stripDistRepoTagREs:
+ name = re.sub(reObj,"",name)
+ if name.find(".rpm") > 0:
+ raise Exception("suffix not known: " + name + ", " + name)
+ return name
+
+ def __init__(self,distArchPackagerList,name):
+ self.__distArchPackagerList = distArchPackagerList
+ self.__name = name
+ # let's extract the version-release
+ self.__verrel = ""
+ tempname = self.__stripDistRepoTag(name)
-
- if tempname.find(".rpm") > 0:
- print "suffix not known in BinaryRpm::__init__ : " + name + ", " + tempname
- exit(1)
regexp = "^(.*)-(.*?)-(.*?)\.$"
regex = re.compile(regexp)
result = regex.match(tempname)
if result:
- self.version = result.group(2)
- self.release = result.group(3)
- self.verrel = self.version + "-" + self.release
+ self.__version = result.group(2)
+ self.__release = result.group(3)
+ self.__verrel = self.__version + "-" + self.__release
else:
- print "could not get version and release, name" + name + ", tempname: "+ tempname
- exit(1)
+ raise Exception("could not get version and release, name:" + name + ", tempname:"+ tempname)
+
+
+
-# actually not an object for each distro/arch combination, but an object for each distro/arch/packager combination!!
+# an object for each distro/arch/packager combination
# it contains all the rpms on a certain url
-class DistroArchObject:
- def __init__(self,packager, shortName,displayName, rpmsUrl, suffixes, fullName, additionalPaths, localFile, distroArch):
- self.distroArch = distroArch
- self.shortName = shortName
- self.displayName = displayName
- self.rpmsUrl = rpmsUrl
- self.rpms = []
- self.binaryRpms = []
- self.srpms = []
- self.binarySrpms = []
- self.suffixes = suffixes
- self.fullName = fullName
- self.additionalPaths = additionalPaths
- self.packager = packager
- self.localFile = localFile
+class DistArchPackagerList:
+ def __init__(self, packager, displayShortName, rpmsUrl, displayFullName, additionalPaths, localFile, distroArch):
+ self.__distroArch = distroArch
+ #self.shortName = shortName
+ self.__displayShortName = displayShortName
+ self.__rpmsUrl = rpmsUrl
+ #self.rpms = []
+ self.__binaryRpms = []
+ #self.srpms = []
+ self.__binarySrpms = []
+ #self.__suffixes = suffixes
+ self.__displayFullName = displayFullName
+ self.__additionalPaths = additionalPaths
+ self.__packager = packager
+ self.__localFile = localFile
+ def getNumberOfBinaryRpms(self):
+ return len(self.__binaryRpms)
+
+ def getNumberOfSourceRpms(self):
+ return len(self.__binarySrpms)
+
+ # get the local file. the local file contains the page at the url rpmsUrl. this can be used for testing/debugging
+ # this local file is not automatically updated
+ def getLocalFile(self):
+ return self.__localFile
+
+ # rpmsUrl is an url where a page can be found containing all those rpms
+ def getRpmsUrl(self):
+ return self.__rpmsUrl
+
+ # short name of the packager like 'dries'
+ def getPackager(self):
+ return self.__packager
+
+ # distroarch, for example: 'el4-i386'
+ def getDistArch(self):
+ return self.__distroArch
+
+ # displayShortName, for example 'el4 i386 rpms' (was: displayName)
+ def getDisplayShortName(self):
+ return self.__displayShortName
+
+ # displayFullName, for example 'Red Hat Enterprise Linux 4 for i386' (was: fullName)
+ def getDisplayFullName(self):
+ return self.__displayFullName
+
+ # distroarchpackager, for example 'el4-i386-dries' (was: shortName)
+ def getDistArchPackager(self):
+ return self.__distroArch + "-" + self.__packager
+
+ def addBinaryRpm(self, bo):
+ self.__binaryRpms.append(bo)
+
+
class SiteObject:
- def __init__(self,specRepo,useLocalFilesOnly):
+ def __init__(self,specRepo,useLocalFilesOnly,buildDriesSite,buildRpmforgeSite):
self.__cat = Logger().get_instance(self)
self.__cat.debug("start")
+ self.__timer("SiteObject init")
self.__useLocalFilesOnly = useLocalFilesOnly
+ self.__buildDriesSite = buildDriesSite
+ self.__buildRpmforgeSite = buildRpmforgeSite
+
# this is a hash which contains values which can be used in templates
# for example FC3-I386-DRIES-RPMS -> 533 (the number of rpms)
# LASTUPDATE -> 4 jan 2010 5:07
self.__siteValues = {}
self.__specRepo = specRepo
- self.__fullcontentstemplate = None
+ self.__Driesfullcontentstemplate = None
self.__RFfullcontentstemplate = None
- self.__siterootdir = "/tmp/packages/"
+ self.__DriesSiterootdir = "/tmp/packages/"
self.__RFsiterootdir = "/tmp/rpmforgesite/"
self.__storage = storagefactory.StorageFactory.getStorage()
- self.__rpmlists = []
+ self.__timer("distArchPackagerLists init")
+ self.__distArchPackagerLists = []
addPaths = ['redhat/el4/en/i386']
- self.__rpmlists.append(DistroArchObject("dries", "el4-i386-dries", "el4 i386 rpms","http://ftp.belnet.be/packages/dries.ulyssis.org/redhat/el4/en/i386/RPMS.dries/", ["el4.rf"], "Red Hat Enterprise Linux 4 for i386", addPaths, "scripts/cache/el4-i386-dries.html", "el4-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "el4 i386 rpms","http://ftp.belnet.be/packages/dries.ulyssis.org/redhat/el4/en/i386/RPMS.dries/", "Red Hat Enterprise Linux 4 for i386", addPaths, "scripts/cache/el4-i386-dries.html", "el4-i386"))
addPaths = ['redhat/el3/en/i386']
- self.__rpmlists.append(DistroArchObject("dries", "el3-i386-dries", "el3 i386 rpms","http://ftp.belnet.be/packages/dries.ulyssis.org/redhat/el3/en/i386/RPMS.dries/", ["el3.rf"], "Red Hat Enterprise Linux 3 for i386", addPaths, "scripts/cache/el3-i386-dries.html", "el3-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "el3 i386 rpms","http://ftp.belnet.be/packages/dries.ulyssis.org/redhat/el3/en/i386/RPMS.dries/", "Red Hat Enterprise Linux 3 for i386", addPaths, "scripts/cache/el3-i386-dries.html", "el3-i386"))
addPaths = ['fedora/fc3/i386']
- self.__rpmlists.append(DistroArchObject("dries", "fc3-i386-dries", "fc3 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc3/i386/RPMS.dries/", ["fc3.rf", "fc3.dries"], "Fedora Core 3 for i386", addPaths, "scripts/cache/fc3-i386-dries.html", "fc3-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "fc3 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc3/i386/RPMS.dries/", "Fedora Core 3 for i386", addPaths, "scripts/cache/fc3-i386-dries.html", "fc3-i386"))
addPaths = ['fedora/fc2/i386']
- self.__rpmlists.append(DistroArchObject("dries", "fc2-i386-dries", "fc2 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc2/i386/RPMS.dries/", ["fc2.rf", "fc2.dries"], "Fedora Core 2 for i386", addPaths, "scripts/cache/fc2-i386-dries.html", "fc2-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "fc2 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc2/i386/RPMS.dries/", "Fedora Core 2 for i386", addPaths, "scripts/cache/fc2-i386-dries.html", "fc2-i386"))
addPaths = ['fedora/fc1/i386']
- self.__rpmlists.append(DistroArchObject("dries", "fc1-i386-dries", "fc1 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc1/i386/RPMS.dries/", ["fc1.rf", "fc1.dries"], "Fedora Core 1 for i386", addPaths, "scripts/cache/fc1-i386-dries.html", "fc1-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "fc1 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc1/i386/RPMS.dries/", "Fedora Core 1 for i386", addPaths, "scripts/cache/fc1-i386-dries.html", "fc1-i386"))
addPaths = ['fedora/fc4/i386']
- self.__rpmlists.append(DistroArchObject("dries", "fc4-i386-dries", "fc4 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc4/i386/RPMS.dries/", ["fc4.rf", "fc4.dries"], "Fedora Core 4 for i386", addPaths, "scripts/cache/fc4-i386-dries.html", "fc4-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "fc4 i386 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc4/i386/RPMS.dries/", "Fedora Core 4 for i386", addPaths, "scripts/cache/fc4-i386-dries.html", "fc4-i386"))
addPaths = ['fedora/fc4/x86_64']
- self.__rpmlists.append(DistroArchObject("dries", "fc4-x86_64-dries", "fc4 x86_64 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc4/x86_64/RPMS.dries/", ["fc4.rf", "fc4.dries"], "Fedora Core 4 for x86_64", addPaths, "scripts/cache/fc4-x86_64-dries.html", "fc4-x86_64"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "fc4 x86_64 rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/fedora/fc4/x86_64/RPMS.dries/", "Fedora Core 4 for x86_64", addPaths, "scripts/cache/fc4-x86_64-dries.html", "fc4-x86_64"))
addPaths = ['aurora/1.92/sparc']
- self.__rpmlists.append(DistroArchObject("dries", "au1.92-sparc-dries", "au1.92 sparc rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/aurora/1.92/sparc/RPMS.dries/", ["au1.92.rf", "au1.92.dries"], "Aurora 1.92 for sparc", addPaths, "scripts/cache/au1.92-sparc-dries.html", "au1.92-sparc"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "au1.92 sparc rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/aurora/1.92/sparc/RPMS.dries/", "Aurora 1.92 for sparc", addPaths, "scripts/cache/au1.92-sparc-dries.html", "au1.92-sparc"))
addPaths = ['aurora/1.91/sparc']
- self.__rpmlists.append(DistroArchObject("dries", "au1.91-sparc-dries", "au1.91 sparc rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/aurora/1.91/sparc/RPMS.dries/", ["au1.91.rf", "au1.91.dries"],"Aurora 1.91 for sparc", addPaths, "scripts/cache/au1.91-sparc-dries.html", "au1.91-sparc"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dries", "au1.91 sparc rpms", "http://ftp.belnet.be/packages/dries.ulyssis.org/aurora/1.91/sparc/RPMS.dries/", "Aurora 1.91 for sparc", addPaths, "scripts/cache/au1.91-sparc-dries.html", "au1.91-sparc"))
#dag:
#http://apt.sw.be/fedora/3/en/i386/RPMS.dag/
#http://apt.sw.be/fedora/3/en/x86_64/RPMS.dag/
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "rh6-i386-dag", "rh6 i386 rpms", "http://apt.sw.be/redhat/6.2/en/i386/RPMS.dag/", ["rh6.rf", "rh6.dag"],"Red Hat 6.2 for i386", addPaths, "scripts/cache/rh6-i386-dag.html", "rh6-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "rh6 i386 rpms", "http://apt.sw.be/redhat/6.2/en/i386/RPMS.dag/", "Red Hat 6.2 for i386", addPaths, "scripts/cache/rh6-i386-dag.html", "rh6-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "rh7-i386-dag", "rh7 i386 rpms", "http://apt.sw.be/redhat/7.3/en/i386/RPMS.dag/", ["rh7.rf", "rh7.dag"],"Red Hat 7.3 for i386", addPaths, "scripts/cache/rh7-i386-dag.html", "rh7-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "rh7 i386 rpms", "http://apt.sw.be/redhat/7.3/en/i386/RPMS.dag/", "Red Hat 7.3 for i386", addPaths, "scripts/cache/rh7-i386-dag.html", "rh7-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "rh8-i386-dag", "rh8 i386 rpms", "http://apt.sw.be/redhat/8.0/en/i386/RPMS.dag/", ["rh8.rf", "rh8.dag"],"Red Hat 8.0 for i386", addPaths, "scripts/cache/rh8-i386-dag.html", "rh8-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "rh8 i386 rpms", "http://apt.sw.be/redhat/8.0/en/i386/RPMS.dag/", "Red Hat 8.0 for i386", addPaths, "scripts/cache/rh8-i386-dag.html", "rh8-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "rh9-i386-dag", "rh8 i386 rpms", "http://apt.sw.be/redhat/9/en/i386/RPMS.dag/", ["rh9.rf", "rh9.dag"],"Red Hat 9 for i386", addPaths, "scripts/cache/rh9-i386-dag.html", "rh9-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "rh9 i386 rpms", "http://apt.sw.be/redhat/9/en/i386/RPMS.dag/","Red Hat 9 for i386", addPaths, "scripts/cache/rh9-i386-dag.html", "rh9-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "el2-i386-dag", "el2 i386 rpms", "http://apt.sw.be/redhat/el2.1/en/i386/RPMS.dag/", ["el2.rf", "el.dag"],"Red Hat Enterprise Linux 2.1 for i386", addPaths, "scripts/cache/el2-i386-dag.html", "el2-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "el2 i386 rpms", "http://apt.sw.be/redhat/el2.1/en/i386/RPMS.dag/", "Red Hat Enterprise Linux 2.1 for i386", addPaths, "scripts/cache/el2-i386-dag.html", "el2-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "el3-i386-dag", "el3 i386 rpms", "http://apt.sw.be/redhat/el3/en/i386/RPMS.dag/", ["el3.rf", "el3.dag"],"Red Hat Enterprise Linux 3 for i386", addPaths, "scripts/cache/el3-i386-dag.html", "el3-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "el3 i386 rpms", "http://apt.sw.be/redhat/el3/en/i386/RPMS.dag/", "Red Hat Enterprise Linux 3 for i386", addPaths, "scripts/cache/el3-i386-dag.html", "el3-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "el3-x86_64-dag", "el3 x86_64 rpms", "http://apt.sw.be/redhat/el3/en/x86_64/RPMS.dag/", ["el3.rf", "el3.dag"],"Red Hat Enterprise Linux 3 for x86_64", addPaths, "scripts/cache/el3-x86_64-dag.html", "el3-x86_64"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "el3 x86_64 rpms", "http://apt.sw.be/redhat/el3/en/x86_64/RPMS.dag/","Red Hat Enterprise Linux 3 for x86_64", addPaths, "scripts/cache/el3-x86_64-dag.html", "el3-x86_64"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "el4-i386-dag", "el4 i386 rpms", "http://apt.sw.be/redhat/el4/en/i386/RPMS.dag/", ["el4.rf", "el4.dag"],"Red Hat Enterprise Linux 4 for i386", addPaths, "scripts/cache/el4-i386-dag.html", "el4-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "el4 i386 rpms", "http://apt.sw.be/redhat/el4/en/i386/RPMS.dag/", "Red Hat Enterprise Linux 4 for i386", addPaths, "scripts/cache/el4-i386-dag.html", "el4-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "el4-x86_64-dag", "el4 x86_64 rpms", "http://apt.sw.be/redhat/el4/en/x86_64/RPMS.dag/", ["el4.rf", "el4.dag"],"Red Hat Enterprise Linux 4 for x86_64", addPaths, "scripts/cache/el4-x86_64-dag.html", "el4-x86_64"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "el4 x86_64 rpms", "http://apt.sw.be/redhat/el4/en/x86_64/RPMS.dag/", "Red Hat Enterprise Linux 4 for x86_64", addPaths, "scripts/cache/el4-x86_64-dag.html", "el4-x86_64"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "fc1-i386-dag", "fc1 i386 rpms", "http://apt.sw.be/fedora/1/en/i386/RPMS.dag/", ["fc1.rf", "fc1.dag"],"Fedora Core 1 for i386", addPaths, "scripts/cache/fc1-i386-dag.html", "fc1-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "fc1 i386 rpms", "http://apt.sw.be/fedora/1/en/i386/RPMS.dag/", "Fedora Core 1 for i386", addPaths, "scripts/cache/fc1-i386-dag.html", "fc1-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "fc2-i386-dag", "fc2 i386 rpms", "http://apt.sw.be/fedora/2/en/i386/RPMS.dag/", ["fc2.rf", "fc2.dag"],"Fedora Core 2 for i386", addPaths, "scripts/cache/fc2-i386-dag.html", "fc2-i386"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "fc2 i386 rpms", "http://apt.sw.be/fedora/2/en/i386/RPMS.dag/", "Fedora Core 2 for i386", addPaths, "scripts/cache/fc2-i386-dag.html", "fc2-i386"))
addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "fc2-x86_64-dag", "fc2 x86_64 rpms", "http://apt.sw.be/fedora/2/en/x86_64/RPMS.dag/", ["fc2.rf", "fc2.dag"],"Fedora Core 2 for x86_64", addPaths, "scripts/cache/fc2-x86_64-dag.html", "fc2-x86_64"))
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "fc2 x86_64 rpms", "http://apt.sw.be/fedora/2/en/x86_64/RPMS.dag/", "Fedora Core 2 for x86_64", addPaths, "scripts/cache/fc2-x86_64-dag.html", "fc2-x86_64"))
addPaths = []
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "fc3 i386 rpms", "http://apt.sw.be/fedora/3/en/i386/RPMS.dag/", "Fedora Core 3 for i386", addPaths, "scripts/cache/fc3-i386-dag.html", "fc3-i386"))
+ addPaths = []
+ self.__distArchPackagerLists.append(DistArchPackagerList("dag", "fc3 x86_64 rpms", "http://apt.sw.be/fedora/3/en/x86_64/RPMS.dag/", "Fedora Core 3 for x86_64", addPaths, "scripts/cache/fc3-x86_64-dag.html", "fc3-x86_64"))
+ self.__timer("distArchPackagerLists init")
- self.__rpmlists.append(DistroArchObject("dag", "fc3-i386-dag", "fc3 i386 rpms", "http://apt.sw.be/fedora/3/en/i386/RPMS.dag/", ["fc3.rf", "fc3.dag"],"Fedora Core 3 for i386", addPaths, "scripts/cache/fc3-i386-dag.html", "fc3-i386"))
- addPaths = []
- self.__rpmlists.append(DistroArchObject("dag", "fc3-x86_64-dag", "fc3 x86_64 rpms", "http://apt.sw.be/fedora/3/en/x86_64/RPMS.dag/", ["fc3.rf", "fc3.dag"],"Fedora Core 3 for x86_64", addPaths, "scripts/cache/fc3-x86_64-dag.html", "fc3-x86_64"))
self.__rpmNameToSpecSubDirName = {}
+ self.__timer("completeRpmsLists")
self.__completeRpmsLists()
+ self.__timer("completeRpmsLists")
+ self.__timer("completeSiteValues")
self.__completeSiteValues()
+ self.__timer("completeSiteValues")
+ self.__timer("SiteObject init")
-
# counters is a hash which contains interesting values which can be used on pages
def __completeSiteValues(self):
# it should already contain
@@ -218,35 +283,67 @@
totsrpms = 0
totrpmsdries = 0
totrpmsdag = 0
- for d in self.__rpmlists:
- totrpms = totrpms + len(d.rpms)
- totsrpms = totsrpms + len(d.srpms)
- if d.packager == "dries":
- totrpmsdries = totrpmsdries + len(d.rpms)
- if d.packager == "dag":
- totrpmsdag = totrpmsdag + len(d.rpms)
+ for d in self.__distArchPackagerLists:
+ totrpms = totrpms + d.getNumberOfBinaryRpms()
+ totsrpms = totsrpms + d.getNumberOfSourceRpms()
+ if d.getPackager() == "dries":
+ totrpmsdries = totrpmsdries + d.getNumberOfBinaryRpms()
+ if d.getPackager() == "dag":
+ totrpmsdag = totrpmsdag + d.getNumberOfBinaryRpms()
self.__siteValues["TOTALRPMS"] = totrpms
self.__siteValues["TOTALSRPMS"] = totsrpms
+ self.__siteValues["TOTAL-RPMS-DRIES"] = totrpmsdries
+ self.__siteValues["TOTAL-RPMS-DAG"] = totrpmsdag
+ for k in self.__siteValues.keys():
+ print "siteval,k:" + k + ",v:" + str(self.__siteValues[k])
def __completeRpmsLists(self):
+ self.__completeBinaryRpmsLists()
+ #self.__completeSourceRpmsLists()
+
+ lastTimerTimestamp = {}
+ totalTimerTimestamp = {}
+ isRunningTimerTimestamp = {}
+
+ def __timer(self,msg):
+ if msg in SiteObject.lastTimerTimestamp.keys():
+ if SiteObject.isRunningTimerTimestamp[msg] == 1:
+ SiteObject.totalTimerTimestamp[msg] = SiteObject.totalTimerTimestamp[msg] + time.time() - SiteObject.lastTimerTimestamp[msg]
+ SiteObject.isRunningTimerTimestamp[msg] = 0
+ #print "[TIMER] " + msg + ": " + str(SiteObject.totalTimerTimestamp[msg])
+ else:
+ SiteObject.isRunningTimerTimestamp[msg] = 1
+ SiteObject.lastTimerTimestamp[msg] = time.time()
+ else:
+ SiteObject.lastTimerTimestamp[msg] = time.time()
+ SiteObject.totalTimerTimestamp[msg] = 0
+ SiteObject.isRunningTimerTimestamp[msg] = 1
+
+ def printTimers(self):
+ for msg in SiteObject.totalTimerTimestamp.keys():
+ print "TIMER: " + msg + ": " + str(SiteObject.totalTimerTimestamp[msg])
+
+
+ def __completeBinaryRpmsLists(self):
+ self.__timer("__completeBinaryRpmsLists")
# list of packagenames
- packagenames = self.__getPackageNameListSortedByLength()
+ extendedPackagenames = self.__getPackageNameListSortedByLength()
# make a hash packagename -> specrepospecfile
nametospecrepospecfile = {}
for srsf in self.__specRepo.getSpecRepositorySpecFiles():
nametospecrepospecfile[srsf.getSubDir()] = srsf
#firstre = re.compile('<A HREF=".*\.rpm<\/A>')
secondre = re.compile('<[Aa] [Hh][Rr][Ee][Ff]="(.*\.rpm)">.*\.rpm</[Aa]> *([0-9]*)-([A-Za-z]*)-([0-9]*) ')
- for listje in self.__rpmlists:
+ for listje in self.__distArchPackagerLists:
# ok get the list of rpms and srpms of this list
f = None
if not self.__useLocalFilesOnly:
- f = urllib.urlopen(listje.rpmsUrl)
+ f = urllib.urlopen(listje.getRpmsUrl())
else:
- print "localFile: " + listje.localFile
- f = open(listje.localFile)
+ print "localFile: " + listje.getLocalFile()
+ f = open(listje.getLocalFile())
#print f.info()
- print "start, " + listje.rpmsUrl
+ print "start, " + listje.getRpmsUrl()
lijn = f.readline()
while lijn != "":
#print l
@@ -255,25 +352,39 @@
mo = secondre.search(lijn)
if mo != None:
rpm = mo.group(1)
- listje.rpms.append(rpm)
- bo = BinaryRpm(listje,rpm)
- listje.binaryRpms.append(bo)
+ #listje.rpms.append(rpm)
+ bo = RpmPackage(listje,rpm)
+ listje.addBinaryRpm(bo)
i = 0
- while ((i < len(packagenames)) and (not rpm.startswith(packagenames[i]))):
- i = i + 1
- #print i, len(packagenames)
- if i<len(packagenames):
- #print "match: " + rpm + ", package=" + packagenames[i]
- nametospecrepospecfile[packagenames[i]].rpmfilenames.append(listje.rpmsUrl + rpm)
- print "adding: " + rpm
- nametospecrepospecfile[packagenames[i]].binaryrpms.append(bo)
- self.__rpmNameToSpecSubDirName[rpm] = packagenames[i]
+ #self.__timer("search in packagenames")
+ #print "rpm: " + rpm
+ start = rpm[0:2]
+ if start in extendedPackagenames.keys():
+ packagenames = extendedPackagenames[rpm[0:2]]
+ while ((i < len(packagenames)) and (not rpm.startswith(packagenames[i]))):
+ i = i + 1
+ #print i, len(packagenames), packagenames[i]
+ #self.__timer("search in packagenames")
+ if i<len(packagenames):
+ #print "match: " + rpm + ", package=" + packagenames[i]
+ #nametospecrepospecfile[packagenames[i]].rpmfilenames.append(listje.getRpmsUrl() + rpm)
+ #print "adding: " + rpm
+ #nametospecrepospecfile[packagenames[i]].binaryrpms.append(bo)
+ bo.setSpecRepoSpecFile(nametospecrepospecfile[packagenames[i]])
+ #self.__rpmNameToSpecSubDirName[rpm] = packagenames[i]
+ else:
+ print "not adding: " + rpm
+ else:
+ print "not in keys: "+ start
else:
- print "bad line: " + lijn
+ badLines = 1
+ #print "bad line: " + lijn
lijn = f.readline()
f.close()
- self.__siteValues[listje.shortName.upper() + "-RPMS"] = len(listje.rpms)
+ self.__siteValues[listje.getDisplayShortName().upper() + "-RPMS"] = listje.getNumberOfBinaryRpms()
+ self.__timer("__completeBinaryRpmsLists")
+ def completeSourceRpmsLists(self):
# the same for the sprms
for listje in self.__rpmlists:
# ok get the list of rpms and srpms of this list
@@ -307,96 +418,129 @@
def __getPackageNameListSortedByLength(self):
retval = []
+ #for srsf in self.__specRepo.getSpecRepositorySpecFiles():
+ # name = srsf.getSubDir()
+ # retval.append(name)
+ #retval.sort(lambda x, y : len(y)-len(x))
+ self.__timer("extra packagenamelists")
+ extendedPackageLists = {}
for srsf in self.__specRepo.getSpecRepositorySpecFiles():
name = srsf.getSubDir()
- retval.append(name)
- retval.sort(lambda x, y : len(y)-len(x))
- return retval
+ start = name[0:2]
+ if start in extendedPackageLists.keys():
+ extendedPackageLists[start].append(name)
+ else:
+ tmplist = [name]
+ extendedPackageLists[start] = tmplist
+ for k in extendedPackageLists.keys():
+ extendedPackageLists[k].sort(lambda x, y: len(y) - len(x))
+ self.__timer("extra packagenamelists")
+ return extendedPackageLists
def generate(self):
# generate the content
print "start, static pages"
- self.generateIndexPage()
- self.generateMirrorsPage()
- self.generatePackagesPage()
- self.generateRecentChangesPage()
- self.generateGPGKeyPage()
- self.clientConfigPage()
- self.pydar2Pages()
- self.setGPGKey()
+ self.__timer("generate")
+ if self.__buildDriesSite:
+ self.generateIndexPage()
+ self.generateMirrorsPage()
+ self.generatePackagesPage()
+ self.generateRecentChangesPage()
+ self.generateGPGKeyPage()
+ self.clientConfigPage()
+ self.pydar2Pages()
+ self.setGPGKey()
+
print "start, specrepospecfile pages"
- for i in self.__specRepo.getSpecRepositorySpecFiles():
- self.generateOnePackagePage(i)
- #self.RFgenerateUserPackagePage(i)
- self.RFgenerateSpecPages(i)
- self.RFgeneratePackagesPages()
+ for specRepoSpecFile in self.__specRepo.getSpecRepositorySpecFiles():
+ shortName = specRepoSpecFile.getSubDir()
+ versions = self.__storage.getVersionIdsOfSpecRepoSpecFile(specRepoSpecFile)
+ hash = self.__storage.getTagsOfSpecFileVersionNew(versions[0])
+ if defaultQueryDistroArch in hash.keys():
+ if self.__buildDriesSite:
+ self.generateOnePackagePage(specRepoSpecFile)
+ if self.__buildRpmforgeSite:
+ #self.RFgenerateSpecPages(specRepoSpecFile)
+ self.RFgenerateUserPackagePage(specRepoSpecFile, hash)
+ self.RFgenerateDeveloperPackagePage(specRepoSpecFile, hash)
+ self.RFgeneratePackagerPackagePage(specRepoSpecFile)
+ else:
+ print "ERROR: defaultQueryDistArch not in hash for " + specRepoSpecFile.getSubDir()
+ if self.__buildRpmforgeSite:
+ self.RFgeneratePackagesPages()
print "start, distropages"
- for i in self.__rpmlists:
- if i.packager == "dries":
- self.generateDistroArchPageRpm(i)
- self.generateDistroArchPageSrpm(i)
- # also create some directory indexes
- dirs = ["fedora", "redhat", "aurora"]
- for i in dirs:
- self.generateDirectoryView(i)
- # add some symbolic links
- self.createDirHierarchy("fedora/core")
- self.createDirHierarchy("fedora/linux")
- self.createSymbolicLink("fedora/core/1", "../fc1")
- self.createSymbolicLink("fedora/core/2", "../fc2")
- self.createSymbolicLink("fedora/core/3", "../fc3")
- self.createSymbolicLink("fedora/core/4", "../fc4")
- self.createSymbolicLink("fedora/linux/1", "../fc1")
- self.createSymbolicLink("fedora/linux/2", "../fc2")
- self.createSymbolicLink("fedora/l