2 commits - createrepo/__init__.py genpkgmetadata.py

[s***@osuosl.org]

createrepo/__init__.py | 40 +++++++++++++++++++++++++++++++++-------
genpkgmetadata.py | 2 ++
2 files changed, 35 insertions(+), 7 deletions(-)

New commits:
commit 79b7871aaba432e267a8e15a4070118a9168f9e4
Author: Seth Vidal <***@fedoraproject.org>
Date: Wed Jul 27 18:07:35 2011 -0400

reset the worker cmd location, whoops

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 769defa..8cce31a 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -107,9 +107,9 @@ class MetaDataConfig(object):
# read in this run of createrepo
self.collapse_glibc_requires = True
self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
- #self.worker_cmd = '/usr/share/createrepo/worker.py'
+ self.worker_cmd = '/usr/share/createrepo/worker.py'

- self.worker_cmd = './worker.py' # helpful when testing
+ #self.worker_cmd = './worker.py' # helpful when testing
self.retain_old_md = 0

class SimpleMDCallBack(object):
commit 180e3042bf7337edba2b80dd867f3f7953622725
Author: Seth Vidal <***@fedoraproject.org>
Date: Wed Jul 27 18:05:17 2011 -0400

add retain_old_md config value and --retain-old-md cli option.

This adds the ability for createrepo to keep around the latest N old,
uniquely named, primary, filelist and otherdata xml and sqlite files
if they exist. This is handy for sites which have mirrors which are
not always in sync. It means if a user has an old repomd.xml they
can still get the xml or sqlite file referred to in that from
the normal mirrors

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 8549188..769defa 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -107,9 +107,10 @@ class MetaDataConfig(object):
# read in this run of createrepo
self.collapse_glibc_requires = True
self.workers = 1 # number of workers to fork off to grab metadata from the pkgs
- self.worker_cmd = '/usr/share/createrepo/worker.py'
+ #self.worker_cmd = '/usr/share/createrepo/worker.py'

- #self.worker_cmd = './worker.py' # helpful when testing
+ self.worker_cmd = './worker.py' # helpful when testing
+ self.retain_old_md = 0

class SimpleMDCallBack(object):
def errorlog(self, thing):
@@ -1094,17 +1095,42 @@ class MetaDataGenerator:
raise MDError, _(
'Could not remove old metadata file: %s: %s') % (oldfile, e)

+ old_to_remove = []
+ old_pr = []
+ old_fl = []
+ old_ot = []
+ old_pr_db = []
+ old_fl_db = []
+ old_ot_db = []
for f in os.listdir(output_old_dir):
oldfile = os.path.join(output_old_dir, f)
finalfile = os.path.join(output_final_dir, f)
- if f.find('-') != -1 and f.split('-')[1] in ('primary.sqlite.bz2',
- 'filelists.sqlite.bz2', 'primary.xml.gz','other.sqlite.bz2',
- 'other.xml.gz','filelists.xml.gz'):
- os.remove(oldfile) # kill off the old ones
- continue
+
+ for (end,lst) in (('-primary.sqlite.bz2', old_pr_db), ('-primary.xml.gz', old_pr),
+ ('-filelists.sqlite.bz2', old_fl_db), ('-filelists.xml.gz', old_fl),
+ ('-other.sqlite.bz2', old_ot_db), ('-other.xml.gz', old_ot)):
+ if f.endswith(end):
+ lst.append(oldfile)
+ break
+
+ # make a list of the old metadata files we don't want to remove.
+ for lst in (old_pr, old_fl, old_ot, old_pr_db, old_fl_db, old_ot_db):
+ sortlst = sorted(lst, key=lambda x: os.path.getmtime(x),
+ reverse=True)
+ for thisf in sortlst[self.conf.retain_old_md:]:
+ old_to_remove.append(thisf)
+
+ for f in os.listdir(output_old_dir):
+ oldfile = os.path.join(output_old_dir, f)
+ finalfile = os.path.join(output_final_dir, f)
+
if f in ('filelists.sqlite.bz2', 'other.sqlite.bz2',
- 'primary.sqlite.bz2'):
- os.remove(oldfile)
+ 'primary.sqlite.bz2') or oldfile in old_to_remove:
+ try:
+ os.remove(oldfile)
+ except (OSError, IOError), e:
+ raise MDError, _(
+ 'Could not remove old metadata file: %s: %s') % (oldfile, e)
continue

if os.path.exists(finalfile):
diff --git a/genpkgmetadata.py b/genpkgmetadata.py
index 8c98191..6c9217a 100755
--- a/genpkgmetadata.py
+++ b/genpkgmetadata.py
@@ -100,6 +100,8 @@ def parse_args(args, conf):
parser.add_option("--simple-md-filenames", dest="simple_md_filenames",
help="do not include the file's checksum in the filename, helps with proxies",
default=False, action="store_true")
+ parser.add_option("--retain-old-md", default=0, type='int', dest='retain_old_md',
+ help="keep around the latest (by timestamp) N copies of the old repodata")
parser.add_option("--distro", default=[], action="append",
help="distro tag and optional cpeid: --distro" "'cpeid,textname'")
parser.add_option("--content", default=[], dest='content_tags',