[svn] r5928 - in trunk/tools/mrepo: . config/dists docs
packagers at lists.rpmforge.net
packagers at lists.rpmforge.net
Fri Oct 19 17:35:52 CEST 2007
Author: dag
Date: 2007-10-19 17:35:50 +0200 (Fri, 19 Oct 2007)
New Revision: 5928
Added:
trunk/tools/mrepo/docs/you-support.txt
trunk/tools/mrepo/youget
Modified:
trunk/tools/mrepo/ChangeLog
trunk/tools/mrepo/Makefile
trunk/tools/mrepo/config/dists/nld9.conf
trunk/tools/mrepo/mrepo
trunk/tools/mrepo/rhnget
Log:
Updates
Modified: trunk/tools/mrepo/ChangeLog
===================================================================
--- trunk/tools/mrepo/ChangeLog 2007-10-17 22:08:52 UTC (rev 5927)
+++ trunk/tools/mrepo/ChangeLog 2007-10-19 15:35:50 UTC (rev 5928)
@@ -1,4 +1,4 @@
-* 0.8.4svn - ... - released 30/12/2006
+* 0.8.4svn - ... - released 19/10/2007
- Fixed a few typos (Ian Forde)
- Added promoteepoch directive to disable it for older distributions (rhel2.1, rh7.3)
- Fixed a problem with the disabled directive (disabled = no was ignored)
@@ -19,6 +19,7 @@
- Get rid of variables named 'list' that override python's list() builtin (Alexander Bergolth)
- Synchronize symlinks instead of deleting and recreating them (Alexander Bergolth)
- Added unit tests (Alexander Bergolth)
+- Added youget tool to download packages from Yast Online Update
* 0.8.4 - Sint-Jacobsplein - released 13/12/2006
- Renamed Yam to mrepo (Matthew Hannigan)
Modified: trunk/tools/mrepo/Makefile
===================================================================
--- trunk/tools/mrepo/Makefile 2007-10-17 22:08:52 UTC (rev 5927)
+++ trunk/tools/mrepo/Makefile 2007-10-19 15:35:50 UTC (rev 5928)
@@ -23,8 +23,9 @@
install:
install -Dp -m0755 gensystemid $(DESTDIR)$(bindir)/gensystemid
+ install -Dp -m0755 mrepo $(DESTDIR)$(bindir)/mrepo
install -Dp -m0755 rhnget $(DESTDIR)$(bindir)/rhnget
- install -Dp -m0755 mrepo $(DESTDIR)$(bindir)/mrepo
+ install -Dp -m0755 youget $(DESTDIR)$(bindir)/youget
[ ! -f $(DESTDIR)$(sysconfdir)/mrepo.conf ] && install -D -m0600 config/mrepo.conf $(DESTDIR)$(sysconfdir)/mrepo.conf || :
install -d -m0755 $(DESTDIR)$(sysconfdir)/mrepo.conf.d/
install -Dp -m0644 config/httpd/mrepo.conf $(DESTDIR)$(httpddir)/mrepo.conf
Modified: trunk/tools/mrepo/config/dists/nld9.conf
===================================================================
--- trunk/tools/mrepo/config/dists/nld9.conf 2007-10-17 22:08:52 UTC (rev 5927)
+++ trunk/tools/mrepo/config/dists/nld9.conf 2007-10-19 15:35:50 UTC (rev 5928)
@@ -7,10 +7,12 @@
[nld9]
name = Novell Linux Desktop $release SP3 ($arch)
release = 9
-arch = i386 x86_64
+arch = i386
### ISO images
iso = NLD$release-SP3-$arch-CD?.iso
### Additional repositories
-you = file:///var/yast2/you/
+updates = you://update.novell.com/data/$RCE/nld9/getPackage/nld-9-i586/
+extras = you://update.novell.com/data/$RCE/nld9-extras/getPackage/nld-9-i586/
+sdk = you://update.novell.com/data/$RCE/nld9-sdk/getPackage/nld-9-i586/
Added: trunk/tools/mrepo/docs/you-support.txt
===================================================================
--- trunk/tools/mrepo/docs/you-support.txt (rev 0)
+++ trunk/tools/mrepo/docs/you-support.txt 2007-10-19 15:35:50 UTC (rev 5928)
@@ -0,0 +1,43 @@
+= Using YOU (Yast Online Update) support in mrepo
+Dag Wieers <dag at wieers.com>
+0.8.4svn, 19 october 2007
+
+// Please send me improvements to this document.
+
+This document describes how to set up a Yast Online Update mirror for
+various Novell/SUSE distributions and architectures.
+
+== Requirements
+The youget python script needs cElementTree support.
+
+
+== Novell update credentials
+To be able to download updates from update.novell.com (or another YOU server)
+you need to copy the files _/etc/ximian/mcookie_ and _/etc/ximian/partnernet_
+from a registered/activated system to your +$srcdir/$dist-$arch/+ directory.
+
+So for NLD9, you would have eg:
+
+ /var/mrepo/nld9-i586/mcookie
+ /var/mrepo/nld9-i586/partnernet
+
+These are used by youget to authenticate with the YOU server.
+
+
+== YOU configuration
+Having done that, you can enable multiple channels for these credentials by
+adding a configuration like the one below:
+
+----
+[nld9]
+name = Novell Linux Desktop $release SP3 ($arch)
+release = 9
+arch = i386
+iso = NLD$release-SP3-$arch-CD?.iso
+updates = you://update.novell.com/data/$RCE/nld9/getPackage/nld-9-i586/
+extras = you://update.novell.com/data/$RCE/nld9-extras/getPackage/nld-9-i586/
+sdk = you://update.novell.com/data/$RCE/nld9-sdk/getPackage/nld-9-i586/
+----
+
+After that, mrepo should be able to successfully log on and download
+all packages for the configured channels.
Modified: trunk/tools/mrepo/mrepo
===================================================================
--- trunk/tools/mrepo/mrepo 2007-10-17 22:08:52 UTC (rev 5927)
+++ trunk/tools/mrepo/mrepo 2007-10-19 15:35:50 UTC (rev 5928)
@@ -28,12 +28,13 @@
'alpha': ('alpha', 'alphaev5', 'alphaev56', 'alphaev6', 'alphaev67'),
'i386': ('i386', 'i486', 'i586', 'i686', 'athlon'),
'ia64': ('i386', 'i686', 'ia64'),
+ 'ppc': ('ppc', ),
'ppc64': ('ppc', 'ppc64', 'ppc64pseries', 'ppc64iseries'),
'x86_64': ('i386', 'i486', 'i586', 'i686', 'athlon', 'x86_64', 'amd64', 'ia32e'),
'sparc64': ('sparc', 'sparcv8', 'sparcv9', 'sparc64'),
'sparc64v': ('sparc', 'sparcv8', 'sparcv9', 'sparcv9v', 'sparc64', 'sparc64v'),
's390': ('s390', ),
- 's390x': ('s390', 's390x', ),
+ 's390x': ('s390', 's390x'),
}
variables = {}
@@ -42,7 +43,7 @@
disable = ('no', 'off', 'false', '0')
### Register rhn and rhns as a known schemes
-for scheme in ('rhn', 'rhns'):
+for scheme in ('rhn', 'rhns', 'you'):
urlparse.uses_netloc.insert(0, scheme)
urlparse.uses_query.insert(0, scheme)
@@ -109,7 +110,7 @@
self.update = True
if not self.types:
- self.types = ['file', 'fish', 'ftp', 'http', 'https', 'mc', 'rhn', 'rhns', 'rsync', 'sftp', 'mrepo']
+ self.types = ['file', 'fish', 'ftp', 'http', 'https', 'mc', 'rhn', 'rhns', 'rsync', 'sftp', 'mrepo', 'you']
for arg in args:
self.dists = self.dists + arg.split(',')
@@ -197,6 +198,7 @@
self.cmd['rhnget'] = self.getoption('main', 'rhngetcmd', '/usr/bin/rhnget')
self.cmd['rsync'] = self.getoption('main', 'rsynccmd', '/usr/bin/rsync')
self.cmd['umount'] = self.getoption('main', 'umountcmd', '/bin/umount')
+ self.cmd['youget'] = self.getoption('main', 'yougetcmd', '/usr/bin/youget')
self.cmd['yumarch'] = self.getoption('main', 'yumarchcmd', '/usr/bin/yum-arch')
self.createrepooptions = self.getoption('main', 'createrepo-options', '-p')
@@ -623,6 +625,8 @@
mirrormirrordir(url, self.srcdir)
elif s in ('rhn', 'rhns'):
mirrorrhnget(url, self.srcdir, self.dist)
+ elif s in ('you', ):
+ mirroryouget(url, self.srcdir, self.dist)
else:
error(2, 'Scheme %s:// not implemented yet (in %s)' % (s, url))
except mrepoMirrorException, e:
@@ -1310,6 +1314,50 @@
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
+def mirroryouget(url, path, dist):
+ "Mirror everything from a you:// URL"
+ if not cf.cmd['youget']:
+ error(1, 'youget was not found. YOU support is therefor disabled.')
+ return
+ mkdir(path)
+
+ url = url.replace('you://', 'https://')
+
+ opts = cf.rhngetoptions
+ if op.verbose >= 3:
+ opts = opts + ' -v' * (op.verbose - 3)
+ if op.dryrun:
+ opts = opts + ' --dry-run'
+ if cf.rhngetcleanup:
+ opts = opts + ' --delete'
+ if cf.rhngetdownloadall:
+ opts = opts + ' --download-all'
+
+ credpath = os.path.join(cf.srcdir, dist.nick)
+ if os.path.isdir(credpath):
+ opts = opts + ' --credpath="%s"' % credpath
+
+# if dist.rhnrelease:
+# opts = opts + ' --release="%s"' % dist.rhnrelease
+
+# if cf.rhnlogin:
+# rhnlogin = cf.rhnlogin.split(':')
+# if len(rhnlogin) > 0:
+# opts = opts + ' --username="%s"' % rhnlogin[0]
+# if len(rhnlogin) > 1:
+# opts = opts + ' --password="%s"' % rhnlogin[1]
+
+## opts = opts + ' -I \"*.rpm\"'
+# opts = opts + ' -G \"headers\" -G \"repodata\"'
+# if cf.mirrordirexclsrpm:
+# opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
+# if cf.mirrordirexcldebug:
+# opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'
+
+ ret = run("%s %s '%s' '%s'" % (cf.cmd['youget'], opts, url, path), dryrun=True)
+ if ret:
+ raise(mrepoMirrorException('Failed with return code: %s' % ret))
+
def hardlink(srcdir):
info(1, 'Hardlinking duplicate packages in %s.' % srcdir)
opts = ''
Modified: trunk/tools/mrepo/rhnget
===================================================================
--- trunk/tools/mrepo/rhnget 2007-10-17 22:08:52 UTC (rev 5927)
+++ trunk/tools/mrepo/rhnget 2007-10-19 15:35:50 UTC (rev 5928)
@@ -413,6 +413,7 @@
for pkg in package_list:
### FIXME: Check if not already on ISO-file or repository as well
filename = '%s-%s-%s.%s.rpm' % (pkg[0], pkg[1], pkg[2], pkg[4])
+ filesize = pkg[5]
### Filter packagelist
if op.filter and not fnmatch.fnmatch(filename, op.filter):
@@ -427,11 +428,11 @@
### If file (or symlink target) exists
if os.path.isfile(os.path.join(path, filename)):
stat = os.stat(os.path.join(path, filename))
- if stat.st_size == int(pkg[5]):
+ if stat.st_size == int(filesize):
info(3, 'File %s is already in %s' % (filename, path))
continue
else:
- info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, pkg[5]))
+ info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
remove(os.path.join(path, filename))
### If symlink target does not exist, remove symlink
@@ -477,22 +478,22 @@
for file in glob.glob(os.path.join(path, '*.rpm')):
if os.path.exists(file):
filename = os.path.basename(file)
- size = os.stat(file).st_size
- receiver.add( (filename, size) )
+ filesize = os.stat(file).st_size
+ receiver.add( (filename, filesize) )
receiver.sort()
### Collect sender side
sender = Set()
for pkg in package_list:
filename = '%s-%s-%s.%s.rpm' % (pkg[0], pkg[1], pkg[2], pkg[4])
- size = int(pkg[5])
- sender.add( (filename, size) )
+ filesize = int(pkg[5])
+ sender.add( (filename, filesize) )
sender.sort()
### Remove difference between receiver and sender
cleanse = receiver.difference(sender)
- for filename, size in cleanse.list:
- info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, size))
+ for filename, filesize in cleanse.list:
+ info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
remove(os.path.join(path, filename))
def main():
Added: trunk/tools/mrepo/youget
===================================================================
--- trunk/tools/mrepo/youget (rev 0)
+++ trunk/tools/mrepo/youget 2007-10-19 15:35:50 UTC (rev 5928)
@@ -0,0 +1,316 @@
+#!/usr/bin/python
+
+### This program is free software; you can redistribute it and/or modify
+### it under the terms of the GNU Library General Public License as published by
+### the Free Software Foundation; version 2 only
+###
+### This program is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+### GNU Library General Public License for more details.
+###
+### You should have received a copy of the GNU Library General Public License
+### along with this program; if not, write to the Free Software
+### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+### Copyright 2004-2006 Dag Wieers <dag at wieers.com>
+
+import os, sys, shutil, getopt, ConfigParser, urlparse, types
+import signal, xmlrpclib, getpass, glob, fnmatch, urllib2
+import gzip
+import cElementTree as ElementTree
+
+__version__ = "$Revision: 4786 $"
+# $Source$
+
+VERSION = '0.8.4svn'
+
+class Options:
+ def __init__(self, args):
+ self.cleanup = False
+ self.credpath = None
+ self.downloadall = False
+ self.dryrun = False
+ self.filter = None
+ self.list = None
+ self.quiet = False
+ self.password = None
+ self.username = None
+ self.source = False
+ self.verbose = 1
+
+ try:
+ opts, args = getopt.getopt (args, 'hlnqp:s:u:v',
+ ('credpath=', 'delete', 'download-all', 'dryrun', 'filter=', 'help', 'list',
+ 'password=', 'quiet', 'source', 'systemid=', 'username=', 'verbose', 'version' ))
+ except getopt.error, exc:
+ print 'youget: %s, try youget -h for a list of all the options' % str(exc)
+ sys.exit(1)
+
+ for opt, arg in opts:
+ if opt in ('--credpath', ):
+ self.credpath = arg
+ elif opt in ('--delete', ):
+ self.cleanup = True
+ elif opt in ('--download-all', ):
+ self.downloadall = True
+ elif opt in ('--filter', ):
+ self.filter = arg
+ self.downloadall = True
+ elif opt in ('-h', '--help'):
+ self.usage()
+ print
+ self.help()
+ sys.exit(0)
+ elif opt in ('-l', '--list'):
+ self.list = True
+ self.downloadall = True
+ elif opt in ('-n', '--dry-run'):
+ self.dryrun = True
+ elif opt in ['-p', '--password']:
+ self.password = arg
+ elif opt in ('-q', '--quiet'):
+ self.quiet = True
+ elif opt in ('--source'):
+ self.source = True
+ elif opt in ('-s', '--systemid'):
+ self.systemid = os.path.abspath(arg)
+ elif opt in ['-u', '--username']:
+ self.username = arg
+ elif opt in ('-v', '--verbose'):
+ self.verbose = self.verbose + 1
+ elif opt in ('--version', ):
+ self.version()
+ sys.exit(0)
+
+ if len(args) < 1:
+ self.usage()
+ print
+ self.help()
+ sys.exit(1)
+
+ self.uri = args[0]
+
+ if not self.username and not self.password:
+ if self.credpath:
+ try:
+ self.username = open(os.path.join(self.credpath, 'mcookie')).read().rstrip()
+ self.password = open(os.path.join(self.credpath, 'partnernet')).read().rstrip()
+ except:
+ die(1, 'Credentials directory %s does not contain mcookie and partnernet files.' % op.credpath)
+ elif os.path.isdir('/etc/ximian'):
+ try:
+ self.username = open('/etc/ximian/mcookie').read().rstrip()
+ self.password = open('/etc/ximian/partnernet').read().rstrip()
+ except:
+ die(1, 'Credentials directory /etc/ximian does not contain mcookie and partnernet files.')
+
+ if not self.username:
+ self.username = raw_input('YOU Username: ')
+
+ if self.username and not self.password:
+ self.password = getpass.getpass('YOU Password for user %s: ' % self.username)
+
+ if len(args) == 2:
+ self.destination = args[1]
+ else:
+ self.destination = os.getcwd()
+
+ if self.quiet:
+ self.verbose = 0
+
+ if self.verbose >= 3:
+ print 'Verbosity set to level %d' % (self.verbose - 1)
+
+ def version(self):
+ print 'youget %s' % VERSION
+ print 'Written by Dag Wieers <dag at wieers.com>'
+ print
+ print 'platform %s/%s' % (os.name, sys.platform)
+ print 'python %s' % sys.version
+ print
+ print 'build revision $Rev: 4786 $'
+
+ def usage(self):
+ print 'usage: youget [options] URL'
+
+ def help(self):
+ print '''Download packages from Yast Online Update (YOU)
+
+youget options:
+ -d, --credpath=dir credentials directory
+ --delete delete files that are not on the sender side
+ --download-all download all package versions available
+ --filter filter packages based on regexp
+ -l, --list list the available packages
+ -n, --dry-run show what would have been done
+ -q, --quiet minimal output
+ --source download source packages
+ -v, --verbose increase verbosity
+ -vv, -vvv, -vvvv.. increase verbosity more
+'''
+
+class Set:
+ def __init__(self):
+ self.list = []
+
+ def add(self, input):
+ if input not in self.list:
+ self.list.append(input)
+
+ def delete(self, input):
+ if input in self.list:
+ self.list.removed(input)
+
+ def difference(self, other):
+ newlist = Set()
+ for element in self.list:
+ if element not in other.list:
+ newlist.add(element)
+ return newlist
+
+ def sort(self):
+ return self.list.sort()
+
+ def __str__(self):
+ return '\n\t' + '\n\t'.join([element[0] for element in self.list])
+
+ def __len__(self):
+ return len(self.list)
+
+class MirrorException(Exception):
+ def __init__(self, value):
+ self.value = value
+ def __str__(self):
+ return repr(self.value)
+
+def error(level, str):
+ "Output error message"
+ if level <= op.verbose:
+ sys.stderr.write('youget: %s\n' % str)
+
+def info(level, str):
+ "Output info message"
+ if level <= op.verbose:
+ sys.stdout.write('%s\n' % str)
+
+def die(ret, str):
+ "Print error and exit with errorcode"
+ error(0, str)
+ sys.exit(ret)
+
+def mirroryou(url, path):
+ 'Mirror a channel from YOU'
+
+ ### Download packagelist for this channel
+ info(2, 'Downloading packagelist from %s' % url)
+ host = urlparse.urlparse(url)[1]
+ auth_handler = urllib2.HTTPDigestAuthHandler()
+ auth_handler.add_password('Express', host, op.username, op.password)
+ opener = urllib2.build_opener(auth_handler)
+ opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+ fdin = opener.open(os.path.join(url, 'packageinfo.xml.gz'))
+ fdout = open(os.path.join(path,'packageinfo.xml.gz'), 'w')
+ fdout.write(fdin.read())
+ fdin.close()
+ fdout.close()
+
+ ### Parse packagelist
+ fd = gzip.open(os.path.join(path,'packageinfo.xml.gz'), 'r')
+ tree = ElementTree.ElementTree(file=fd)
+ root = tree.getroot()
+ package_list = Set()
+ for elem in root.getiterator('package'):
+ pkgname = elem.findtext('history/update/filename')
+ pkgsize = elem.findtext('history/update/filesize')
+ package_list.add( (pkgname, pkgsize) )
+ fd.close()
+ package_list.sort()
+
+ ### Download packages from the packagelist
+ for filename, filesize in package_list.list:
+
+ ### Filter packagelist
+ if op.filter and not fnmatch.fnmatch(filename, op.filter):
+ info(4, 'Packages %s excluded by filter' % filename)
+ continue
+
+ ### List only files if requested
+ if op.list:
+ info(0, filename)
+ continue
+
+ ### If file (or symlink target) exists
+ if os.path.isfile(os.path.join(path, filename)):
+ stat = os.stat(os.path.join(path, filename))
+ if stat.st_size == int(filesize):
+ info(3, 'File %s is already in %s' % (filename, path))
+ continue
+ else:
+ info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
+ remove(os.path.join(path, filename))
+
+ ### If symlink target does not exist, remove symlink
+ elif os.path.islink(os.path.join(path, filename)):
+ remove(os.path.join(path, filename))
+
+ if op.dryrun:
+ info(1, 'Not downloading package %s' % filename)
+ continue
+
+ info(2, 'Download %s (%s)' % (filename, filesize))
+ fdin = opener.open(os.path.join(url, filename))
+ fdout = open(os.path.join(path, filename), 'w')
+ fdout.write(fdin.read())
+ fdin.close()
+ fdout.close()
+
+ ### Remove packages on the receiver side that are not on the sender side
+ if op.cleanup:
+
+ ### Collect receiver side
+ receiver = Set()
+ for file in glob.glob(os.path.join(path, '*.rpm')):
+ if os.path.exists(file):
+ filename = os.path.basename(file)
+ filesize = os.stat(file).st_size
+ receiver.add( (filename, filesize) )
+ receiver.sort()
+
+ ### Collect sender side
+ sender = package_list
+
+ ### Remove difference between receiver and sender
+ cleanse = receiver.difference(sender)
+ for filename, filesize in cleanse.list:
+ info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
+ remove(os.path.join(path, filename))
+
+def main():
+ try:
+ mirroryou(op.uri, op.destination)
+ except Exception, e:
+ die(1, e)
+
+### Unbuffered sys.stdout
+sys.stdout = os.fdopen(1, 'w', 0)
+sys.stderr = os.fdopen(2, 'w', 0)
+
+### Workaround for python <= 2.2.1
+try:
+ True, False
+except NameError:
+ True = 1
+ False = 0
+
+### Main entrance
+if __name__ == '__main__':
+ exitcode = 0
+
+ op = Options(sys.argv[1:])
+ try:
+ main()
+ except KeyboardInterrupt, e:
+ die(6, 'Exiting on user request')
+ sys.exit(exitcode)
+
+# vim:ts=4:sw=4:et
Property changes on: trunk/tools/mrepo/youget
___________________________________________________________________
Name: svn:executable
+ *
More information about the svn-commits
mailing list