2 commits - createrepo/__init__.py createrepo/utils.py

[James Antill]

createrepo/__init__.py | 3 +++
createrepo/utils.py | 36 ++++++++++++++++++++++--------------
2 files changed, 25 insertions(+), 14 deletions(-)

New commits:
commit 26e5d78cd9106059d422c78b8e9448142f9bcaa4
Author: Elia Pinto <***@gmail.com>
Date: Tue Nov 29 04:58:31 2011 -0500

Exclude filename from .gz files

When you run createrepo, the original filenames,
including full path, of the files are stored
in the header of the .gz metadata

The path becomes part of the checksum of the gzip file
itself. So, if you gunzip the file and re-gzip it,
you get a different checksum.

Based on the original observation of Dennis Gregorovic

diff --git a/createrepo/utils.py b/createrepo/utils.py
index 16c57db..84a43ba 100644
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -40,22 +40,14 @@ def _(args):

class GzipFile(gzip.GzipFile):
def _write_gzip_header(self):
+ # Generate a header that is easily reproduced with gzip -9 -n on
+ # an unix-like system
self.fileobj.write('\037\213') # magic header
self.fileobj.write('\010') # compression method
- if hasattr(self, 'name'):
- fname = self.name[:-3]
- else:
- fname = self.filename[:-3]
- flags = 0
- if fname:
- flags = FNAME
- self.fileobj.write(chr(flags))
- write32u(self.fileobj, long(0))
- self.fileobj.write('\002')
- self.fileobj.write('\377')
- if fname:
- self.fileobj.write(fname + '\000')
-
+ self.fileobj.write('\000') # flags
+ write32u(self.fileobj, long(0)) # timestamp
+ self.fileobj.write('\002') # max compression
+ self.fileobj.write('\003') # UNIX

def _gzipOpen(filename, mode="rb", compresslevel=9):
return GzipFile(filename, mode, compresslevel)
commit b9080209ddf0faf38131ffcf9fd26a36c56d6795
Author: James Antill <***@and.org>
Date: Mon Nov 28 15:52:34 2011 -0500

Allow auto-setting of workers. via. sysconf SC_NPROCESSORS_ONLN.

diff --git a/createrepo/__init__.py b/createrepo/__init__.py
index 014765a..2d3514e 100644
--- a/createrepo/__init__.py
+++ b/createrepo/__init__.py
@@ -48,6 +48,7 @@ except ImportError:

from utils import _gzipOpen, compressFile, compressOpen, checkAndMakeDir, GzipFile, \
checksum_and_rename, split_list_into_equal_chunks
+from utils import num_cpus_online
import deltarpms

__version__ = '0.9.9'
@@ -606,6 +607,8 @@ class MetaDataGenerator:
# open the files they created and write them out to our metadata
# add up the total pkg counts and return that value
self._worker_tmp_path = tempfile.mkdtemp() # setting this in the base object so we can clean it up later
+ if self.conf.workers < 1:
+ self.conf.workers = num_cpus_online()
worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers)
worker_cmd_dict = {}
worker_jobs = {}
diff --git a/createrepo/utils.py b/createrepo/utils.py
index c5cec64..16c57db 100644
--- a/createrepo/utils.py
+++ b/createrepo/utils.py
@@ -200,6 +200,22 @@ def split_list_into_equal_chunks(seq, num_chunks):

return out

+def num_cpus_online(unknown=1):
+ if not hasattr(os, "sysconf"):
+ return unknown
+
+ if not os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
+ return unknown
+
+ ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
+ try:
+ if int(ncpus) > 0:
+ return ncpus
+ except:
+ pass
+
+ return unknown
+

class MDError(Exception):
def __init__(self, value=None):