[svn] r5940 - in trunk/tools/mrepo: . config/dists docs
packagers at lists.rpmforge.net
packagers at lists.rpmforge.net
Wed Oct 24 19:23:55 CEST 2007
Author: dag
Date: 2007-10-24 19:23:54 +0200 (Wed, 24 Oct 2007)
New Revision: 5940
Modified:
trunk/tools/mrepo/config/dists/sles10.conf
trunk/tools/mrepo/docs/you-support.txt
trunk/tools/mrepo/mrepo.spec
trunk/tools/mrepo/rhnget
trunk/tools/mrepo/youget
Log:
Updates
Modified: trunk/tools/mrepo/config/dists/sles10.conf
===================================================================
--- trunk/tools/mrepo/config/dists/sles10.conf 2007-10-23 07:26:59 UTC (rev 5939)
+++ trunk/tools/mrepo/config/dists/sles10.conf 2007-10-24 17:23:54 UTC (rev 5940)
@@ -7,10 +7,11 @@
[sles10]
name = SuSE Linux Enterprise Server $release SP1 ($arch)
release = 10
-arch = i386 ia64 ppc s390x x86_64
+arch = i586 ia64 ppc s390x x86_64
### ISO images
iso = SLES-$release-SP1-$arch-CD?.iso SLES-$release-SP1-$arch-DVD?.iso
### Additional repositories
-you = file:///var/yast2/you/
+updates = you://update.novell.com/repo/$RCE/SLES10-Updates/sles-10-$arch/
+sdk = you://update.novell.com/repo/$RCE/SLE10-SDK-Updates/sles-10-$arch/
Modified: trunk/tools/mrepo/docs/you-support.txt
===================================================================
--- trunk/tools/mrepo/docs/you-support.txt 2007-10-23 07:26:59 UTC (rev 5939)
+++ trunk/tools/mrepo/docs/you-support.txt 2007-10-24 17:23:54 UTC (rev 5940)
@@ -11,10 +11,10 @@
The youget python script needs cElementTree support.
-== Novell update credentials
-To be able to download updates from update.novell.com (or another YOU server)
-you need to copy the files _/etc/ximian/mcookie_ and _/etc/ximian/partnernet_
-from a registered/activated system to your +$srcdir/$dist-$arch/+ directory.
+== Novell update credentials for NLD9 (and alike)
+To be able to download updates from update.novell.com you need to copy the
+files _/etc/ximian/mcookie_ and _/etc/ximian/partnernet_ from a registered
+system to your +$srcdir/$dist-$arch/+ directory.
So for NLD9, you would have eg:
@@ -24,6 +24,19 @@
These are used by youget to authenticate with the YOU server.
+== Novell update credentials for SLES10 (and alike)
+To be able to download updates from nu.novell.com you need to copy the
+files _/etc/zmd/deviceid_ and _/etc/zmd/secret_ from a registered
+system to your +$srcdir/$dist-$arch/+ directory.
+
+So for SLES10, you would have eg:
+
+ /var/mrepo/sles10-x86_64/deviceid
+ /var/mrepo/sles10-x86_64/secret
+
+These are used by youget to authenticate with the YOU server.
+
+
== YOU configuration
Having done that, you can enable multiple channels for these credentials by
adding a configuration like the one below:
@@ -32,12 +45,24 @@
[nld9]
name = Novell Linux Desktop $release SP3 ($arch)
release = 9
-arch = i386
+arch = i586
iso = NLD$release-SP3-$arch-CD?.iso
-updates = you://update.novell.com/data/$RCE/nld9/getPackage/nld-9-i586/
-extras = you://update.novell.com/data/$RCE/nld9-extras/getPackage/nld-9-i586/
-sdk = you://update.novell.com/data/$RCE/nld9-sdk/getPackage/nld-9-i586/
+updates = you://update.novell.com/data/$RCE/nld9/getPackage/nld-9-$arch/
+extras = you://update.novell.com/data/$RCE/nld9-extras/getPackage/nld-9-$arch/
+sdk = you://update.novell.com/data/$RCE/nld9-sdk/getPackage/nld-9-$arch/
----
+or
+
+----
+[sles10]
+name = SuSE Linux Enterprise Server $release SP1 ($arch)
+release = 10
+arch = i586 ia64 ppc s390x x86_64
+iso = SLES-$release-SP1-$arch-CD?.iso SLES-$release-SP1-$arch-DVD?.iso
+updates = you://update.novell.com/repo/$RCE/SLES10-Updates/sles-10-$arch/
+sdk = you://update.novell.com/repo/$RCE/SLE10-SDK-Updates/sles-10-$arch/
+----
+
After that, mrepo should be able to successfully log on and download
all packages for the configured channels.
Modified: trunk/tools/mrepo/mrepo.spec
===================================================================
--- trunk/tools/mrepo/mrepo.spec 2007-10-23 07:26:59 UTC (rev 5939)
+++ trunk/tools/mrepo/mrepo.spec 2007-10-24 17:23:54 UTC (rev 5940)
@@ -101,8 +101,9 @@
%config(noreplace) %{_sysconfdir}/mrepo.conf.d/
%config %{_initrddir}/mrepo
%{_bindir}/gensystemid
+%{_bindir}/mrepo
%{_bindir}/rhnget
-%{_bindir}/mrepo
+%{_bindir}/youget
%{_datadir}/mrepo/
%{_localstatedir}/cache/mrepo/
%{_localstatedir}/www/mrepo/
Modified: trunk/tools/mrepo/rhnget
===================================================================
--- trunk/tools/mrepo/rhnget 2007-10-23 07:26:59 UTC (rev 5939)
+++ trunk/tools/mrepo/rhnget 2007-10-24 17:23:54 UTC (rev 5940)
@@ -413,7 +413,7 @@
for pkg in package_list:
### FIXME: Check if not already on ISO-file or repository as well
filename = '%s-%s-%s.%s.rpm' % (pkg[0], pkg[1], pkg[2], pkg[4])
- filesize = pkg[5]
+ filesize = int(pkg[5])
### Filter packagelist
if op.filter and not fnmatch.fnmatch(filename, op.filter):
@@ -428,7 +428,7 @@
### If file (or symlink target) exists
if os.path.isfile(os.path.join(path, filename)):
stat = os.stat(os.path.join(path, filename))
- if stat.st_size == int(filesize):
+ if stat.st_size == filesize:
info(3, 'File %s is already in %s' % (filename, path))
continue
else:
Modified: trunk/tools/mrepo/youget
===================================================================
--- trunk/tools/mrepo/youget 2007-10-23 07:26:59 UTC (rev 5939)
+++ trunk/tools/mrepo/youget 2007-10-24 17:23:54 UTC (rev 5940)
@@ -14,6 +14,9 @@
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
### Copyright 2004-2006 Dag Wieers <dag at wieers.com>
+### For SLES10 implementation, see:
+### http://lists.suse.com/archive/suse-sles-e/2006-Aug/0161.html
+
import os, sys, shutil, getopt, ConfigParser, urlparse, types
import signal, xmlrpclib, getpass, glob, fnmatch, urllib2
import gzip
@@ -35,13 +38,14 @@
self.quiet = False
self.password = None
self.username = None
+ self.style = 'sles10'
self.source = False
self.verbose = 1
try:
opts, args = getopt.getopt (args, 'd:hlnqp:s:u:v',
('credpath=', 'delete', 'download-all', 'dryrun', 'filter=', 'help', 'list',
- 'password=', 'quiet', 'source', 'systemid=', 'username=', 'verbose', 'version' ))
+ 'password=', 'quiet', 'source', 'style=', 'username=', 'verbose', 'version' ))
except getopt.error, exc:
print 'youget: %s, try youget -h for a list of all the options' % str(exc)
sys.exit(1)
@@ -72,8 +76,8 @@
self.quiet = True
elif opt in ('--source'):
self.source = True
- elif opt in ('-s', '--systemid'):
- self.systemid = os.path.abspath(arg)
+ elif opt in ('-s', '--style'):
+ self.style = arg
elif opt in ['-u', '--username']:
self.username = arg
elif opt in ('-v', '--verbose'):
@@ -90,32 +94,6 @@
self.uri = args[0]
- if not self.username and not self.password:
- if self.credpath:
- try:
- self.username = open(os.path.join(self.credpath, 'mcookie')).read().rstrip().rstrip('\0')
- self.password = open(os.path.join(self.credpath, 'partnernet')).read().rstrip().rstrip('\0')
- except:
- die(1, 'Credentials directory %s does not contain mcookie and partnernet files.' % op.credpath)
- elif os.path.isdir('/etc/ximian'):
- try:
- self.username = open('/etc/ximian/mcookie').read().rstrip().rstrip('\0')
- self.password = open('/etc/ximian/partnernet').read().rstrip().rstrip('\0')
- except:
- die(1, 'Credentials directory /etc/ximian does not contain mcookie and partnernet files.')
- elif os.path.isdir('/etc/zmd'):
- try:
- self.username = open('/etc/zmd/mcookie').read().rstrip().rstrip('\0')
- self.password = open('/etc/zmd/partnernet').read().rstrip().rstrip('\0')
- except:
- die(1, 'Credentials directory /etc/zmd does not contain mcookie and partnernet files.')
-
- if not self.username:
- self.username = raw_input('YOU Username: ')
-
- if self.username and not self.password:
- self.password = getpass.getpass('YOU Password for user %s: ' % self.username)
-
if len(args) == 2:
self.destination = args[1]
else:
@@ -204,6 +182,14 @@
error(0, str)
sys.exit(ret)
+def filelist(top):
+ flist = []
+ for root, dirs, files in os.walk(top):
+ for file in files:
+ flist.append(os.path.join(root, file))
+ flist.sort()
+ return flist
+
def remove(file):
"Remove files or directories"
if isinstance(file, types.StringType):
@@ -223,25 +209,171 @@
for f in file:
remove(f)
+def mkdir(path):
+ "Create a directory, and parents if needed"
+ if op.dryrun:
+ return
+ if os.path.islink(path):
+ os.unlink(path)
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+def httpget(opener, file, url, path=None):
+ "Download file from url to local path"
+ if path:
+ mkdir(path)
+ else:
+ path = os.getcwd()
+ if os.path.dirname(file):
+ mkdir(os.path.join(path, os.path.dirname(file)))
+ fdin = opener.open(os.path.join(url, file))
+ fdout = open(os.path.join(path, file), 'w')
+ fdout.write(fdin.read())
+ fdin.close()
+ fdout.close()
+
def mirroryou(url, path):
- 'Mirror a channel from YOU'
+ 'Check username/password and YOU mirror style'
+ ### See if we find mcookie/partnernet in credpath
+ if not op.username and not op.password:
+ try:
+ op.username = open(os.path.join(op.credpath, 'deviceid')).read().rstrip().rstrip('\0')
+ op.password = open(os.path.join(op.credpath, 'secret')).read().rstrip().rstrip('\0')
+ op.style = 'sles10'
+ except:
+ info(3, 'Credentials directory %s does not contain deviceid and secret files. (SLES10)' % op.credpath)
+
+ ### See if we find mcookie/partnernet in credpath
+ if not op.username and not op.password:
+ try:
+ op.username = open(os.path.join(op.credpath, 'mcookie')).read().rstrip().rstrip('\0')
+ op.password = open(os.path.join(op.credpath, 'partnernet')).read().rstrip().rstrip('\0')
+ op.style = 'nld9'
+ except:
+ info(3, 'Credentials directory %s does not contain mcookie and partnernet files. (NLD9)' % op.credpath)
+
+ if op.credpath and not op.username and not op.password:
+ die(2, 'No credentials found in %s.' % op.credpath)
+
+ if not op.username:
+ op.username = raw_input('YOU Username: ')
+
+ if op.username and not op.password:
+ op.password = getpass.getpass('YOU Password for user %s: ' % op.username)
+
+ if op.style == 'sles10':
+ mirroryou_sles10(url, path)
+ elif op.style == 'nld9':
+ mirroryou_nld9(url, path)
+ else:
+ mirroryou_sles10(url, path)
+
+def mirroryou_sles10(url, path):
+ 'Mirror a channel from YOU (SLES10 style)'
+
info(3, 'Using username %s with password %s.' % (op.username, op.password))
- ### Download packagelist for this channel
- info(2, 'Downloading packagelist from %s' % url)
+ ### Setting up connection
host = urlparse.urlparse(url)[1]
auth_handler = urllib2.HTTPDigestAuthHandler()
auth_handler.add_password('Express', host, op.username, op.password)
opener = urllib2.build_opener(auth_handler)
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
- fdin = opener.open(os.path.join(url, 'packageinfo.xml.gz'))
- fdout = open(os.path.join(path,'packageinfo.xml.gz'), 'w')
- fdout.write(fdin.read())
- fdin.close()
- fdout.close()
+ ### Download repodata for this channel
+ info(2, 'Downloading packagelist and metadata from %s' % url)
+ ### FIXME: check repomd.xml to see if any of the files have been updated
+# httpget(opener, 'repodata/repomd.xml.asc', url, path)
+# httpget(opener, 'repodata/repomd.xml.key', url, path)
+# httpget(opener, 'repodata/repomd.xml', url, path)
+# httpget(opener, 'repodata/filelists.xml.gz', url, path)
+ httpget(opener, 'repodata/primary.xml.gz', url, path)
+# httpget(opener, 'repodata/patches.xml', url, path)
+
### Parse packagelist
+ fd = gzip.open(os.path.join(path, 'repodata/primary.xml.gz'), 'r')
+ tree = ElementTree.ElementTree(file=fd)
+ root = tree.getroot()
+ package_list = Set()
+ for elem in root.getiterator('{http://linux.duke.edu/metadata/common}package'):
+ pkgname = elem.find('{http://linux.duke.edu/metadata/common}location').get('href')
+ pkgsize = int(elem.find('{http://linux.duke.edu/metadata/common}size').get('package'))
+ package_list.add( (pkgname, pkgsize) )
+ fd.close()
+ package_list.sort()
+
+ ### Download packages from the packagelist
+ for filename, filesize in package_list.list:
+
+ ### Filter packagelist
+ if op.filter and not fnmatch.fnmatch(filename, op.filter):
+ info(4, 'Packages %s excluded by filter' % filename)
+ continue
+
+ ### List only files if requested
+ if op.list:
+ info(0, filename)
+ continue
+
+ ### If file (or symlink target) exists
+ if os.path.isfile(os.path.join(path, filename)):
+ stat = os.stat(os.path.join(path, filename))
+ if stat.st_size == filesize:
+ info(3, 'File %s is already in %s' % (filename, path))
+ continue
+ else:
+ info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
+ remove(os.path.join(path, filename))
+
+ ### If symlink target does not exist, remove symlink
+ elif os.path.islink(os.path.join(path, filename)):
+ remove(os.path.join(path, filename))
+
+ if op.dryrun:
+ info(1, 'Not downloading package %s' % filename)
+ continue
+
+ info(2, 'Download %s (%s)' % (filename, filesize))
+ httpget(opener, filename, url, path)
+
+ ### Remove packages on the receiver side that are not on the sender side
+ if op.cleanup:
+
+ ### Collect receiver side
+ receiver = Set()
+ for file in filelist(os.path.join(path, 'rpm')):
+ filename = file.split(path+'/')[1]
+ filesize = os.stat(file).st_size
+ receiver.add( (filename, filesize) )
+ receiver.sort()
+
+ ### Collect sender side
+ sender = package_list
+
+ ### Remove difference between receiver and sender
+ cleanse = receiver.difference(sender)
+ for filename, filesize in cleanse.list:
+ info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
+ remove(os.path.join(path, filename))
+
+def mirroryou_nld9(url, path):
+ 'Mirror a channel from YOU (NLD9 style)'
+
+ info(3, 'Using username %s with password %s.' % (op.username, op.password))
+
+ ### Setting up connection
+ host = urlparse.urlparse(url)[1]
+ auth_handler = urllib2.HTTPDigestAuthHandler()
+ auth_handler.add_password('Express', host, op.username, op.password)
+ opener = urllib2.build_opener(auth_handler)
+ opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+
+ ### Download repodata for this channel
+ info(2, 'Downloading packagelist from %s' % url)
+ httpget(opener, 'packageinfo.xml.gz', url, path)
+
+ ### Parse packagelist
fd = gzip.open(os.path.join(path,'packageinfo.xml.gz'), 'r')
tree = ElementTree.ElementTree(file=fd)
root = tree.getroot()
@@ -285,22 +417,17 @@
continue
info(2, 'Download %s (%s)' % (filename, filesize))
- fdin = opener.open(os.path.join(url, filename))
- fdout = open(os.path.join(path, filename), 'w')
- fdout.write(fdin.read())
- fdin.close()
- fdout.close()
+ httpget(opener, filename, url, path)
### Remove packages on the receiver side that are not on the sender side
if op.cleanup:
### Collect receiver side
receiver = Set()
- for file in glob.glob(os.path.join(path, '*.rpm')):
- if os.path.exists(file):
- filename = os.path.basename(file)
- filesize = os.stat(file).st_size
- receiver.add( (filename, filesize) )
+ for file in filelist(os.path.join(path, 'rpm')):
+ filename = file.split(path+'/')[1]
+ filesize = os.stat(file).st_size
+ receiver.add( (filename, filesize) )
receiver.sort()
### Collect sender side
More information about the svn-commits
mailing list