createrepo/__init__.py worker.py

[s***@osuosl.org]

createrepo/__init__.py | 8 +++++++-
worker.py | 11 ++++++++++-
2 files changed, 17 insertions(+), 2 deletions(-)

New commits:
commit fcb7fa19a488fd60c27b912fe2bbc16b614decb8
Author: Seth Vidal <***@fedoraproject.org>
Date: Wed Aug 31 12:47:18 2011 -0400

add --pkglist option to worker.py - this allows us to pass
the list of pkgs from a tempfile made by the main process.

Apparently the shell doesn't like it when the worker is passed 30K pkgs
as arguments :)

this should resolve
https://bugzilla.redhat.com/show_bug.cgi?id=734838

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index e3c18aa..0fbbff3 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -608,6 +608,12 @@ class MetaDataGenerator:
base_worker_cmdline.append('--verbose')

for worker_num in range(self.conf.workers):
+ pkl = self._worker_tmp_path + '/pkglist-%s' % worker_num
+ print pkl
+ f = open(pkl, 'w')
+ f.write('\n'.join(worker_chunks[worker_num]))
+ f.close()
+
# make the worker directory
workercmdline = []
workercmdline.extend(base_worker_cmdline)
@@ -616,7 +622,7 @@ class MetaDataGenerator:
workercmdline.append('--tmpmdpath=%s' % thisdir)
else:
raise MDError, "Unable to create worker path: %s" % thisdir
- workercmdline.extend(worker_chunks[worker_num])
+ workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num))
worker_cmd_dict[worker_num] = workercmdline


diff --git a/worker.py b/worker.py
index ab78d90..23c87a3 100755
--- a/worker.py
+++ b/worker.py
@@ -5,6 +5,7 @@ import yum
import createrepo
import os
import rpmUtils
+import re
from optparse import OptionParser


@@ -23,6 +24,8 @@ def main(args):
parser = OptionParser()
parser.add_option('--tmpmdpath', default=None,
help="path where the outputs should be dumped for this worker")
+ parser.add_option('--pkglist', default=None,
+ help="file to read the pkglist from in lieu of all of them on the cli")
parser.add_option("--pkgoptions", default=[], action='append',
help="pkgoptions in the format of key=value")
parser.add_option("--quiet", default=False, action='store_true',
@@ -68,7 +71,13 @@ def main(args):
fl = open(opts.tmpmdpath + '/filelists.xml' , 'w')
other = open(opts.tmpmdpath + '/other.xml' , 'w')

-
+ if opts.pkglist:
+ for line in open(opts.pkglist,'r').readlines():
+ line = line.strip()
+ if re.match('^\s*\#.*', line) or re.match('^\s*$', line):
+ continue
+ pkgs.append(line)
+
for pkgfile in pkgs:
pkgpath = reldir + '/' + pkgfile
if not os.path.exists(pkgpath):