From 8c00a1b1a90df99a7ff79f2a02292f75bf433486 Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Sat, 2 Mar 2013 16:31:41 +0000 Subject: [PATCH] Update Python debian_linux module from linux and use it in gencontrol.py svn path=/dists/trunk/linux-tools/; revision=19890 --- debian/bin/gencontrol.py | 62 +--- debian/lib/python/debian_linux/__init__.py | 1 + debian/lib/python/debian_linux/config.py | 298 +++++++++++-------- debian/lib/python/debian_linux/debian.py | 272 +++++++++++------ debian/lib/python/debian_linux/gencontrol.py | 196 ++++++------ debian/lib/python/debian_linux/utils.py | 131 ++++---- 6 files changed, 503 insertions(+), 457 deletions(-) diff --git a/debian/bin/gencontrol.py b/debian/bin/gencontrol.py index c7b9364ec51e..d67f7d56074a 100755 --- a/debian/bin/gencontrol.py +++ b/debian/bin/gencontrol.py @@ -4,10 +4,10 @@ import sys sys.path.append("debian/lib/python") from debian_linux.debian import * -from debian_linux.gencontrol import PackagesList, Makefile, MakeFlags +from debian_linux.gencontrol import PackagesList, Makefile, MakeFlags, Gencontrol from debian_linux.utils import * -class gencontrol(object): +class gencontrol(Gencontrol): makefile_targets = ('binary-arch', 'build') def __init__(self, underlay = None): @@ -60,63 +60,5 @@ class gencontrol(object): 'source_upstream': version.upstream, } - def process_relation(self, key, e, in_e, vars): - import copy - dep = copy.deepcopy(in_e[key]) - for groups in dep: - for item in groups: - item.name = self.substitute(item.name, vars) - e[key] = dep - - def process_description(self, e, in_e, vars): - in_desc = in_e['Description'] - desc = in_desc.__class__() - desc.short = self.substitute(in_desc.short, vars) - for i in in_desc.long: - desc.append(self.substitute(i, vars)) - e['Description'] = desc - - def process_package(self, in_entry, vars): - e = Package() - for key, value in in_entry.iteritems(): - if isinstance(value, PackageRelation): - self.process_relation(key, e, in_entry, vars) - elif key == 'Description': - self.process_description(e, in_entry, vars) - elif key[:2] == 'X-': - pass - else: - e[key] = self.substitute(value, vars) - return e - - def process_packages(self, in_entries, vars): - entries = [] - for i in in_entries: - entries.append(self.process_package(i, vars)) - return entries - - def substitute(self, s, vars): - if isinstance(s, (list, tuple)): - for i in xrange(len(s)): - s[i] = self.substitute(s[i], vars) - return s - def subst(match): - return vars[match.group(1)] - return re.sub(r'@([a-z_]+)@', subst, s) - - def write_control(self, list): - self.write_rfc822(file("debian/control", 'w'), list) - - def write_makefile(self, makefile): - f = file("debian/rules.gen", 'w') - makefile.write(f) - f.close() - - def write_rfc822(self, f, list): - for entry in list: - for key, value in entry.iteritems(): - f.write("%s: %s\n" % (key, value)) - f.write('\n') - if __name__ == '__main__': gencontrol()() diff --git a/debian/lib/python/debian_linux/__init__.py b/debian/lib/python/debian_linux/__init__.py index e69de29bb2d1..b785cebf7108 100644 --- a/debian/lib/python/debian_linux/__init__.py +++ b/debian/lib/python/debian_linux/__init__.py @@ -0,0 +1 @@ +# Module diff --git a/debian/lib/python/debian_linux/config.py b/debian/lib/python/debian_linux/config.py index 7e3e12ae8746..17211dddcf3b 100644 --- a/debian/lib/python/debian_linux/config.py +++ b/debian/lib/python/debian_linux/config.py @@ -1,10 +1,17 @@ -import os, os.path, re, sys, textwrap +import os +import os.path +import re +import sys +import textwrap +import cPickle __all__ = [ + 'ConfigCoreDump', + 'ConfigCoreHierarchy', 'ConfigParser', - 'ConfigReaderCore', ] + class SchemaItemBoolean(object): def __call__(self, i): i = i.strip().lower() @@ -14,8 +21,9 @@ class SchemaItemBoolean(object): return False raise Error + class SchemaItemList(object): - def __init__(self, type = "\s+"): + def __init__(self, type="\s+"): self.type = type def __call__(self, i): @@ -24,108 +32,38 @@ class SchemaItemList(object): return [] return [j.strip() for j in re.split(self.type, i)] -class ConfigReaderCore(dict): - config_name = "defines" - schemas = { - 'base': { - 'arches': SchemaItemList(), - 'enabled': SchemaItemBoolean(), - 'featuresets': SchemaItemList(), - 'flavours': SchemaItemList(), - 'modules': SchemaItemBoolean(), - }, - 'image': { - 'configs': SchemaItemList(), - 'initramfs': SchemaItemBoolean(), - 'initramfs-generators': SchemaItemList(), - }, - 'relations': { - }, - 'xen': { - 'dom0-support': SchemaItemBoolean(), - 'versions': SchemaItemList(), - } - } +class ConfigCore(dict): + def get_merge(self, section, arch, featureset, flavour, key, default=None): + temp = [] - def __init__(self, dirs = []): - self._dirs = dirs - self._read_base() + if arch and featureset and flavour: + temp.append(self.get((section, arch, featureset, flavour), {}).get(key)) + temp.append(self.get((section, arch, None, flavour), {}).get(key)) + if arch and featureset: + temp.append(self.get((section, arch, featureset), {}).get(key)) + if arch: + temp.append(self.get((section, arch), {}).get(key)) + if featureset: + temp.append(self.get((section, None, featureset), {}).get(key)) + temp.append(self.get((section,), {}).get(key)) - def _read_arch(self, arch): - config = ConfigParser(self.schemas) - config.read(self.get_files("%s/%s" % (arch, self.config_name))) + ret = [] - featuresets = config['base',].get('featuresets', []) - flavours = config['base',].get('flavours', []) - - for section in iter(config): - if section[0] in featuresets: - real = (section[-1], arch, section[0]) - elif len(section) > 1: - real = (section[-1], arch, None) + section[:-1] + for i in temp: + if i is None: + continue + elif isinstance(i, (list, tuple)): + ret.extend(i) + elif ret: + # TODO + return ret else: - real = (section[-1], arch) + section[:-1] - s = self.get(real, {}) - s.update(config[section]) - self[tuple(real)] = s + return i - for featureset in featuresets: - self._read_arch_featureset(arch, featureset) + return ret or default - if flavours: - base = self['base', arch] - featuresets.insert(0, 'none') - base['featuresets'] = featuresets - del base['flavours'] - self['base', arch] = base - self['base', arch, 'none'] = {'flavours': flavours, 'implicit-flavour': True} - - def _read_arch_featureset(self, arch, featureset): - config = ConfigParser(self.schemas) - config.read(self.get_files("%s/%s/%s" % (arch, featureset, self.config_name))) - - flavours = config['base',].get('flavours', []) - - for section in iter(config): - real = (section[-1], arch, featureset) + section[:-1] - s = self.get(real, {}) - s.update(config[section]) - self[tuple(real)] = s - - def _read_base(self): - config = ConfigParser(self.schemas) - config.read(self.get_files(self.config_name)) - - arches = config['base',]['arches'] - featuresets = config['base',]['featuresets'] - - for section in iter(config): - if section[0].startswith('featureset-'): - real = (section[-1], None, section[0].lstrip('featureset-')) - else: - real = (section[-1],) + section[1:] - self[real] = config[section] - - for arch in arches: - self._read_arch(arch) - for featureset in featuresets: - self._read_featureset(featureset) - - def _read_featureset(self, featureset): - config = ConfigParser(self.schemas) - config.read(self.get_files("featureset-%s/%s" % (featureset, self.config_name))) - - for section in iter(config): - real = (section[-1], None, featureset) - s = self.get(real, {}) - s.update(config[section]) - self[real] = s - - def get_files(self, name): - return [os.path.join(i, name) for i in self._dirs if i] - - def merge(self, section, arch = None, featureset = None, flavour = None): + def merge(self, section, arch=None, featureset=None, flavour=None): ret = {} ret.update(self.get((section,), {})) if featureset: @@ -139,6 +77,118 @@ class ConfigReaderCore(dict): ret.update(self.get((section, arch, featureset, flavour), {})) return ret + def dump(self, fp): + cPickle.dump(self, fp, 0) + + +class ConfigCoreDump(object): + def __new__(self, fp): + return cPickle.load(fp) + + +class ConfigCoreHierarchy(object): + schema_base = { + 'base': { + 'arches': SchemaItemList(), + 'enabled': SchemaItemBoolean(), + 'featuresets': SchemaItemList(), + 'flavours': SchemaItemList(), + }, + } + + def __new__(cls, schema, dirs=[]): + schema_complete = cls.schema_base.copy() + for key, value in schema.iteritems(): + schema_complete.setdefault(key, {}).update(value) + return cls.Reader(dirs, schema_complete)() + + class Reader(object): + config_name = "defines" + + def __init__(self, dirs, schema): + self.dirs, self.schema = dirs, schema + + def __call__(self): + ret = ConfigCore() + self.read(ret) + return ret + + def get_files(self, *dirs): + dirs = list(dirs) + dirs.append(self.config_name) + return (os.path.join(i, *dirs) for i in self.dirs if i) + + def read_arch(self, ret, arch): + config = ConfigParser(self.schema) + config.read(self.get_files(arch)) + + featuresets = config['base', ].get('featuresets', []) + flavours = config['base', ].get('flavours', []) + + for section in iter(config): + if section[0] in featuresets: + real = (section[-1], arch, section[0]) + elif len(section) > 1: + real = (section[-1], arch, None) + section[:-1] + else: + real = (section[-1], arch) + section[:-1] + s = ret.get(real, {}) + s.update(config[section]) + ret[tuple(real)] = s + + for featureset in featuresets: + self.read_arch_featureset(ret, arch, featureset) + + if flavours: + base = ret['base', arch] + featuresets.insert(0, 'none') + base['featuresets'] = featuresets + del base['flavours'] + ret['base', arch] = base + ret['base', arch, 'none'] = {'flavours': flavours, 'implicit-flavour': True} + + def read_arch_featureset(self, ret, arch, featureset): + config = ConfigParser(self.schema) + config.read(self.get_files(arch, featureset)) + + flavours = config['base', ].get('flavours', []) + + for section in iter(config): + real = (section[-1], arch, featureset) + section[:-1] + s = ret.get(real, {}) + s.update(config[section]) + ret[tuple(real)] = s + + def read(self, ret): + config = ConfigParser(self.schema) + config.read(self.get_files()) + + arches = config['base', ]['arches'] + featuresets = config['base', ].get('featuresets', []) + + for section in iter(config): + if section[0].startswith('featureset-'): + real = (section[-1], None, section[0][11:]) + else: + real = (section[-1],) + section[1:] + ret[real] = config[section] + + for arch in arches: + self.read_arch(ret, arch) + for featureset in featuresets: + self.read_featureset(ret, featureset) + + def read_featureset(self, ret, featureset): + config = ConfigParser(self.schema) + config.read(self.get_files('featureset-%s' % featureset)) + + for section in iter(config): + real = (section[-1], None, featureset) + s = ret.get(real, {}) + s.update(config[section]) + ret[real] = s + + class ConfigParser(object): __slots__ = '_config', 'schemas' @@ -163,47 +213,37 @@ class ConfigParser(object): data = {} for key, value in self._config.items(section): data[key] = value - s1 = section.split('_') - if s1[-1] in self.schemas: - ret[tuple(s1)] = self.SectionSchema(data, self.schemas[s1[-1]]) + section_list = section.split('_') + section_base = section_list[-1] + if section_base in self.schemas: + section_ret = tuple(section_list) + data = self._convert_one(self.schemas[section_base], data) else: - ret[(section,)] = self.Section(data) + section_ret = (section, ) + ret[section_ret] = data return ret + def _convert_one(self, schema, data): + ret = {} + for key, value in data.iteritems(): + if key in schema: + value = schema[key](value) + ret[key] = value + return ret + def keys(self): return self._convert().keys() def read(self, data): return self._config.read(data) - class Section(dict): - def __init__(self, data): - super(ConfigParser.Section, self).__init__(data) - - def __str__(self): - return '<%s(%s)>' % (self.__class__.__name__, self._data) - - class SectionSchema(Section): - __slots__ = () - - def __init__(self, data, schema): - for key in data.keys(): - try: - data[key] = schema[key](data[key]) - except KeyError: pass - super(ConfigParser.SectionSchema, self).__init__(data) if __name__ == '__main__': import sys - config = ConfigReaderCore(['debian/config']) - sections = config.keys() - sections.sort() - for section in sections: - print "[%s]" % (section,) - items = config[section] - items_keys = items.keys() - items_keys.sort() - for item in items: - print "%s: %s" % (item, items[item]) + sys.path.append('debian/lib/python') + config = ConfigCoreDump(open('debian/config.defines.dump')) + for section, items in sorted(config.iteritems()): + print u"[%s]" % (section,) + for item, value in sorted(items.iteritems()): + print u"%s: %s" % (item, value) print - diff --git a/debian/lib/python/debian_linux/debian.py b/debian/lib/python/debian_linux/debian.py index a949c90cd998..a577526582e5 100644 --- a/debian/lib/python/debian_linux/debian.py +++ b/debian/lib/python/debian_linux/debian.py @@ -1,4 +1,10 @@ -import itertools, os.path, re, utils +import collections +import itertools +import os.path +import re + +from . import utils + class Changelog(list): _rules = r""" @@ -26,7 +32,7 @@ class Changelog(list): def __init__(self, distribution, source, version): self.distribution, self.source, self.version = distribution, source, version - def __init__(self, dir = '', version = None): + def __init__(self, dir='', version=None): if version is None: version = Version f = file(os.path.join(dir, "debian/changelog")) @@ -45,6 +51,7 @@ class Changelog(list): v = Version(match.group('version')) self.append(self.Entry(match.group('distribution'), match.group('source'), v)) + class Version(object): _version_rules = ur""" ^ @@ -59,7 +66,7 @@ class Version(object): ) (?: - - (?P[^-]+) + (?P[^-]+) )? $ """ @@ -68,37 +75,45 @@ $ def __init__(self, version): match = self._version_re.match(version) if match is None: - raise RuntimeError, "Invalid debian version" + raise RuntimeError(u"Invalid debian version") self.epoch = None if match.group("epoch") is not None: self.epoch = int(match.group("epoch")) self.upstream = match.group("upstream") - self.debian = match.group("debian") + self.revision = match.group("revision") - def __str__(self): + def __unicode__(self): return self.complete @property def complete(self): if self.epoch is not None: - return "%d:%s" % (self.epoch, self.complete_noepoch) + return u"%d:%s" % (self.epoch, self.complete_noepoch) return self.complete_noepoch @property def complete_noepoch(self): - if self.debian is not None: - return "%s-%s" % (self.upstream, self.debian) + if self.revision is not None: + return u"%s-%s" % (self.upstream, self.revision) return self.upstream + @property + def debian(self): + from warnings import warn + warn(u"debian argument was replaced by revision", DeprecationWarning, stacklevel=2) + return self.revision + + class VersionLinux(Version): _version_linux_rules = ur""" ^ (?P \d+\.\d+ ) -(?: +(?P \.\d+ - | +)? +(?: ~ (?P .+? @@ -111,7 +126,17 @@ class VersionLinux(Version): ) )? - -(?:[^-]+) +\d+ +(\.\d+)? +(?: + (?P + ~experimental\.\d+ + ) + | + (?P + [^-]+ + ) +)? $ """ _version_linux_re = re.compile(_version_linux_rules, re.X) @@ -120,125 +145,159 @@ $ super(VersionLinux, self).__init__(version) match = self._version_linux_re.match(version) if match is None: - raise RuntimeError, "Invalid debian linux version" + raise RuntimeError(u"Invalid debian linux version") d = match.groupdict() self.linux_modifier = d['modifier'] self.linux_version = d['version'] if d['modifier'] is not None: - self.linux_upstream = '-'.join((d['version'], d['modifier'])) + assert not d['update'] + self.linux_upstream = u'-'.join((d['version'], d['modifier'])) else: self.linux_upstream = d['version'] + self.linux_upstream_full = self.linux_upstream + (d['update'] or u'') self.linux_dfsg = d['dfsg'] - -class PackageFieldList(list): - def __init__(self, value = None): - self.extend(value) + self.linux_revision_experimental = match.group('revision_experimental') and True + self.linux_revision_other = match.group('revision_other') and True - def __str__(self): - return ' '.join(self) - def _extend(self, value): - if value is not None: - self.extend([j.strip() for j in re.split('\s', value.strip())]) +class PackageArchitecture(collections.MutableSet): + __slots__ = '_data' + + def __init__(self, value=None): + self._data = set() + if value: + self.extend(value) + + def __contains__(self, value): + return self._data.__contains__(value) + + def __iter__(self): + return self._data.__iter__() + + def __len__(self): + return self._data.__len__() + + def __unicode__(self): + return u' '.join(sorted(self)) + + def add(self, value): + self._data.add(value) + + def discard(self, value): + self._data.discard(value) def extend(self, value): - if isinstance(value, str): - self._extend(value) + if isinstance(value, basestring): + for i in re.split('\s', value.strip()): + self.add(i) else: - super(PackageFieldList, self).extend(value) + raise RuntimeError + class PackageDescription(object): __slots__ = "short", "long" - def __init__(self, value = None): + def __init__(self, value=None): + self.short = [] self.long = [] if value is not None: - self.short, long = value.split("\n", 1) + short, long = value.split(u"\n", 1) self.append(long) - else: - self.short = None + self.append_short(short) - def __str__(self): - ret = self.short + '\n' - w = utils.TextWrapper(width = 74, fix_sentence_endings = True) - pars = [] + def __unicode__(self): + wrap = utils.TextWrapper(width=74, fix_sentence_endings=True).wrap + short = u', '.join(self.short) + long_pars = [] for i in self.long: - pars.append('\n '.join(w.wrap(i))) - return self.short + '\n ' + '\n .\n '.join(pars) + long_pars.append(wrap(i)) + long = u'\n .\n '.join([u'\n '.join(i) for i in long_pars]) + return short + u'\n ' + long def append(self, str): str = str.strip() if str: - self.long.extend(str.split("\n.\n")) + self.long.extend(str.split(u"\n.\n")) + + def append_short(self, str): + for i in [i.strip() for i in str.split(u",")]: + if i: + self.short.append(i) + + def extend(self, desc): + if isinstance(desc, PackageDescription): + self.short.extend(desc.short) + self.long.extend(desc.long) + else: + raise TypeError + class PackageRelation(list): - def __init__(self, value = None): - if value is not None: - self.extend(value) + def __init__(self, value=None, override_arches=None): + if value: + self.extend(value, override_arches) - def __str__(self): - return ', '.join([str(i) for i in self]) + def __unicode__(self): + return u', '.join((unicode(i) for i in self)) - def _match(self, value): + def _search_value(self, value): for i in self: - if i._match(value): + if i._search_value(value): return i return None - def append(self, value): + def append(self, value, override_arches=None): if isinstance(value, basestring): - value = PackageRelationGroup(value) + value = PackageRelationGroup(value, override_arches) elif not isinstance(value, PackageRelationGroup): - raise ValueError, "got %s" % type(value) - j = self._match(value) + raise ValueError(u"got %s" % type(value)) + j = self._search_value(value) if j: - j._updateArches(value) + j._update_arches(value) else: super(PackageRelation, self).append(value) - def extend(self, value): + def extend(self, value, override_arches=None): if isinstance(value, basestring): - value = [j.strip() for j in re.split(',', value.strip())] - elif not isinstance(value, (list, tuple)): - raise ValueError, "got %s" % type(value) + value = (j.strip() for j in re.split(u',', value.strip())) for i in value: - self.append(i) + self.append(i, override_arches) + class PackageRelationGroup(list): - def __init__(self, value = None): - if value is not None: - self.extend(value) + def __init__(self, value=None, override_arches=None): + if value: + self.extend(value, override_arches) - def __str__(self): - return ' | '.join([str(i) for i in self]) + def __unicode__(self): + return u' | '.join((unicode(i) for i in self)) - def _match(self, value): + def _search_value(self, value): for i, j in itertools.izip(self, value): if i.name != j.name or i.version != j.version: return None return self - def _updateArches(self, value): + def _update_arches(self, value): for i, j in itertools.izip(self, value): if i.arches: for arch in j.arches: if arch not in i.arches: i.arches.append(arch) - def append(self, value): + def append(self, value, override_arches=None): if isinstance(value, basestring): - value = PackageRelationEntry(value) + value = PackageRelationEntry(value, override_arches) elif not isinstance(value, PackageRelationEntry): raise ValueError super(PackageRelationGroup, self).append(value) - def extend(self, value): + def extend(self, value, override_arches=None): if isinstance(value, basestring): - value = [j.strip() for j in re.split('\|', value.strip())] - elif not isinstance(value, (list, tuple)): - raise ValueError + value = (j.strip() for j in re.split('\|', value.strip())) for i in value: - self.append(i) + self.append(i, override_arches) + class PackageRelationEntry(object): __slots__ = "name", "operator", "version", "arches" @@ -246,9 +305,31 @@ class PackageRelationEntry(object): _re = re.compile(r'^(\S+)(?: \((<<|<=|=|!=|>=|>>)\s*([^)]+)\))?(?: \[([^]]+)\])?$') class _operator(object): - OP_LT = 1; OP_LE = 2; OP_EQ = 3; OP_NE = 4; OP_GE = 5; OP_GT = 6 - operators = { '<<': OP_LT, '<=': OP_LE, '=': OP_EQ, '!=': OP_NE, '>=': OP_GE, '>>': OP_GT } - operators_neg = { OP_LT: OP_GE, OP_LE: OP_GT, OP_EQ: OP_NE, OP_NE: OP_EQ, OP_GE: OP_LT, OP_GT: OP_LE } + OP_LT = 1 + OP_LE = 2 + OP_EQ = 3 + OP_NE = 4 + OP_GE = 5 + OP_GT = 6 + + operators = { + u'<<': OP_LT, + u'<=': OP_LE, + u'=': OP_EQ, + u'!=': OP_NE, + u'>=': OP_GE, + u'>>': OP_GT, + } + + operators_neg = { + OP_LT: OP_GE, + OP_LE: OP_GT, + OP_EQ: OP_NE, + OP_NE: OP_EQ, + OP_GE: OP_LT, + OP_GT: OP_LE, + } + operators_text = dict([(b, a) for a, b in operators.iteritems()]) __slots__ = '_op', @@ -259,27 +340,30 @@ class PackageRelationEntry(object): def __neg__(self): return self.__class__(self.operators_text[self.operators_neg[self._op]]) - def __str__(self): + def __unicode__(self): return self.operators_text[self._op] - def __init__(self, value = None): - if isinstance(value, basestring): - self.parse(value) - else: + def __init__(self, value=None, override_arches=None): + if not isinstance(value, basestring): raise ValueError - def __str__(self): + self.parse(value) + + if override_arches: + self.arches = list(override_arches) + + def __unicode__(self): ret = [self.name] if self.operator is not None and self.version is not None: - ret.extend([' (', str(self.operator), ' ', self.version, ')']) + ret.extend((u' (', unicode(self.operator), u' ', self.version, u')')) if self.arches: - ret.extend([' [', ' '.join(self.arches), ']']) - return ''.join(ret) + ret.extend((u' [', u' '.join(self.arches), u']')) + return u''.join(ret) def parse(self, value): match = self._re.match(value) if match is None: - raise RuntimeError, "Can't parse dependency %s" % value + raise RuntimeError(u"Can't parse dependency %s" % value) match = match.groups() self.name = match[0] if match[1] is not None: @@ -292,16 +376,17 @@ class PackageRelationEntry(object): else: self.arches = [] + class Package(dict): - _fields = utils.SortedDict(( - ('Package', str), - ('Source', str), - ('Architecture', PackageFieldList), - ('Section', str), - ('Priority', str), - ('Maintainer', str), - ('Uploaders', str), - ('Standards-Version', str), + _fields = collections.OrderedDict(( + ('Package', unicode), + ('Source', unicode), + ('Architecture', PackageArchitecture), + ('Section', unicode), + ('Priority', unicode), + ('Maintainer', unicode), + ('Uploaders', unicode), + ('Standards-Version', unicode), ('Build-Depends', PackageRelation), ('Build-Depends-Indep', PackageRelation), ('Provides', PackageRelation), @@ -310,6 +395,7 @@ class Package(dict): ('Recommends', PackageRelation), ('Suggests', PackageRelation), ('Replaces', PackageRelation), + ('Breaks', PackageRelation), ('Conflicts', PackageRelation), ('Description', PackageDescription), )) @@ -319,13 +405,14 @@ class Package(dict): cls = self._fields[key] if not isinstance(value, cls): value = cls(value) - except KeyError: pass + except KeyError: + pass super(Package, self).__setitem__(key, value) def iterkeys(self): keys = set(self.keys()) for i in self._fields.iterkeys(): - if self.has_key(i): + if i in self: keys.remove(i) yield i for i in keys: @@ -338,4 +425,3 @@ class Package(dict): def itervalues(self): for i in self.iterkeys(): yield self[i] - diff --git a/debian/lib/python/debian_linux/gencontrol.py b/debian/lib/python/debian_linux/gencontrol.py index 4128ce58d200..aac564c420f6 100644 --- a/debian/lib/python/debian_linux/gencontrol.py +++ b/debian/lib/python/debian_linux/gencontrol.py @@ -1,8 +1,10 @@ -from config import * -from debian import * -from utils import * +import codecs +from collections import OrderedDict -class PackagesList(SortedDict): +from .debian import * + + +class PackagesList(OrderedDict): def append(self, package): self[package['Package']] = package @@ -10,16 +12,21 @@ class PackagesList(SortedDict): for package in packages: self[package['Package']] = package + class Makefile(object): def __init__(self): self.rules = {} self.add('.NOTPARALLEL') - def add(self, name, deps = None, cmds = None): + def add(self, name, deps=None, cmds=None): if name in self.rules: self.rules[name].add(deps, cmds) else: self.rules[name] = self.Rule(name, deps, cmds) + if deps is not None: + for i in deps: + if i not in self.rules: + self.rules[i] = self.Rule(i) def write(self, out): r = self.rules.keys() @@ -28,12 +35,12 @@ class Makefile(object): self.rules[i].write(out) class Rule(object): - def __init__(self, name, deps = None, cmds = None): + def __init__(self, name, deps=None, cmds=None): self.name = name self.deps, self.cmds = set(), [] self.add(deps, cmds) - def add(self, deps = None, cmds = None): + def add(self, deps=None, cmds=None): if deps is not None: self.deps.update(deps) if cmds is not None: @@ -56,22 +63,25 @@ class Makefile(object): else: out.write('%s:%s\n' % (self.name, deps_string)) + class MakeFlags(dict): def __repr__(self): repr = super(flags, self).__repr__() return "%s(%s)" % (self.__class__.__name__, repr) def __str__(self): - return ' '.join(["%s='%s'" % i for i in self.iteritems()]) + return ' '.join(["%s='%s'" % i for i in sorted(self.iteritems())]) def copy(self): return self.__class__(super(MakeFlags, self).copy()) -class Gencontrol(object): - makefile_targets = ('binary-arch', 'build', 'setup', 'source') - def __init__(self, config, templates): +class Gencontrol(object): + makefile_targets = ('binary-arch', 'build-arch', 'setup') + + def __init__(self, config, templates, version=Version): self.config, self.templates = config, templates + self.changelog = Changelog(version=version) def __call__(self): packages = PackagesList() @@ -81,45 +91,42 @@ class Gencontrol(object): self.do_main(packages, makefile) self.do_extra(packages, makefile) - self.write_control(packages.itervalues()) - self.write_makefile(makefile) + self.write(packages, makefile) def do_source(self, packages): - source = self.templates["control.source"] - packages['source'] = self.process_package(source[0], self.vars) + source = self.templates["control.source"][0] + source['Source'] = self.changelog[0].source + packages['source'] = self.process_package(source) def do_main(self, packages, makefile): - config_entry = self.config['base',] + config_entry = self.config['base', ] vars = self.vars.copy() - vars.update(config_entry) makeflags = MakeFlags() extra = {} self.do_main_setup(vars, makeflags, extra) - self.do_main_packages(packages, extra) self.do_main_makefile(makefile, makeflags, extra) - - for arch in iter(self.config['base',]['arches']): - self.do_arch(packages, makefile, arch, vars.copy(), makeflags.copy(), extra) + self.do_main_packages(packages, vars, makeflags, extra) + self.do_main_recurse(packages, makefile, vars, makeflags, extra) def do_main_setup(self, vars, makeflags, extra): - makeflags.update({ - 'VERSION': self.version.linux_version, - 'UPSTREAMVERSION': self.version.linux_upstream, - 'ABINAME': self.abiname, - }) - - def do_main_makefile(self, makefile, makeflags, extra): - makefile.add('binary-indep', cmds = ["$(MAKE) -f debian/rules.real binary-indep %s" % makeflags]) - - def do_main_packages(self, packages, extra): pass + def do_main_makefile(self, makefile, makeflags, extra): + makefile.add('build-indep', cmds=["$(MAKE) -f debian/rules.real build-indep %s" % makeflags]) + makefile.add('binary-indep', cmds=["$(MAKE) -f debian/rules.real binary-indep %s" % makeflags]) + + def do_main_packages(self, packages, vars, makeflags, extra): + pass + + def do_main_recurse(self, packages, makefile, vars, makeflags, extra): + for arch in iter(self.config['base', ]['arches']): + self.do_arch(packages, makefile, arch, vars.copy(), makeflags.copy(), extra) + def do_extra(self, packages, makefile): - try: - templates_extra = self.templates["control.extra"] - except IOError: + templates_extra = self.templates.get("control.extra", None) + if templates_extra is None: return packages.extend(self.process_packages(templates_extra, {})) @@ -136,19 +143,15 @@ class Gencontrol(object): cmds = [] for i in extra_arches[arch]: tmp = [] - if i.has_key('X-Version-Overwrite-Epoch'): + if 'X-Version-Overwrite-Epoch' in i: tmp.append("-v1:%s" % self.version['source']) cmds.append("$(MAKE) -f debian/rules.real install-dummy DH_OPTIONS='-p%s' GENCONTROL_ARGS='%s'" % (i['Package'], ' '.join(tmp))) - makefile.add('binary-arch_%s' % arch ['binary-arch_%s_extra' % arch]) - makefile.add("binary-arch_%s_extra" % arch, cmds = cmds) + makefile.add('binary-arch_%s' % arch['binary-arch_%s_extra' % arch]) + makefile.add("binary-arch_%s_extra" % arch, cmds=cmds) def do_arch(self, packages, makefile, arch, vars, makeflags, extra): - config_base = self.config['base', arch] - vars.update(config_base) vars['arch'] = arch - makeflags['ARCH'] = arch - self.do_arch_setup(vars, makeflags, arch, extra) self.do_arch_makefile(makefile, arch, makeflags, extra) self.do_arch_packages(packages, makefile, arch, vars, makeflags, extra) @@ -158,29 +161,27 @@ class Gencontrol(object): pass def do_arch_makefile(self, makefile, arch, makeflags, extra): + makeflags['ARCH'] = arch + for i in self.makefile_targets: target1 = i - target2 = "%s_%s" % (i, arch) + target2 = '_'.join((target1, arch)) + target3 = '_'.join((target2, 'real')) makefile.add(target1, [target2]) - makefile.add(target2, ['%s_real' % target2]) - makefile.add('%s_real' % target2) + makefile.add(target2, [target3]) def do_arch_packages(self, packages, makefile, arch, vars, makeflags, extra): pass def do_arch_recurse(self, packages, makefile, arch, vars, makeflags, extra): - for featureset in self.config['base', arch]['featuresets']: + for featureset in self.config['base', arch].get('featuresets', ()): self.do_featureset(packages, makefile, arch, featureset, vars.copy(), makeflags.copy(), extra) def do_featureset(self, packages, makefile, arch, featureset, vars, makeflags, extra): config_base = self.config.merge('base', arch, featureset) - vars.update(config_base) - if not config_base.get('enabled', True): return - makeflags['FEATURESET'] = featureset - vars['localversion'] = '' if featureset != 'none': vars['localversion'] = '-' + featureset @@ -194,12 +195,14 @@ class Gencontrol(object): pass def do_featureset_makefile(self, makefile, arch, featureset, makeflags, extra): + makeflags['FEATURESET'] = featureset + for i in self.makefile_targets: - target1 = "%s_%s" % (i, arch) - target2 = "%s_%s_%s" % (i, arch, featureset) + target1 = '_'.join((i, arch)) + target2 = '_'.join((target1, featureset)) + target3 = '_'.join((target2, 'real')) makefile.add(target1, [target2]) - makefile.add(target2, ['%s_real' % target2]) - makefile.add('%s_real' % target2) + makefile.add(target2, [target3]) def do_featureset_packages(self, packages, makefile, arch, featureset, vars, makeflags, extra): pass @@ -210,12 +213,7 @@ class Gencontrol(object): def do_flavour(self, packages, makefile, arch, featureset, flavour, vars, makeflags, extra): config_base = self.config.merge('base', arch, featureset, flavour) - vars.update(config_base) - if not vars.has_key('longclass'): - vars['longclass'] = vars['class'] - - makeflags['FLAVOUR'] = flavour vars['localversion'] += '-' + flavour self.do_flavour_setup(vars, makeflags, arch, featureset, flavour, extra) @@ -227,74 +225,74 @@ class Gencontrol(object): ('kernel-arch', 'KERNEL_ARCH'), ('localversion', 'LOCALVERSION'), ): - if vars.has_key(i[0]): + if i[0] in vars: makeflags[i[1]] = vars[i[0]] def do_flavour_makefile(self, makefile, arch, featureset, flavour, makeflags, extra): + makeflags['FLAVOUR'] = flavour + for i in self.makefile_targets: - target1 = "%s_%s_%s" % (i, arch, featureset) - target2 = "%s_%s_%s_%s" % (i, arch, featureset, flavour) + target1 = '_'.join((i, arch, featureset)) + target2 = '_'.join((target1, flavour)) + target3 = '_'.join((target2, 'real')) makefile.add(target1, [target2]) - makefile.add(target2, ['%s_real' % target2]) - makefile.add('%s_real' % target2) + makefile.add(target2, [target3]) def do_flavour_packages(self, packages, makefile, arch, featureset, flavour, vars, makeflags, extra): pass - def process_relation(self, key, e, in_e, vars): + def process_relation(self, dep, vars): import copy - dep = copy.deepcopy(in_e[key]) + dep = copy.deepcopy(dep) for groups in dep: for item in groups: item.name = self.substitute(item.name, vars) - e[key] = dep + if item.version: + item.version = self.substitute(item.version, vars) + return dep - def process_description(self, e, in_e, vars): - in_desc = in_e['Description'] + def process_description(self, in_desc, vars): desc = in_desc.__class__() desc.short = self.substitute(in_desc.short, vars) for i in in_desc.long: desc.append(self.substitute(i, vars)) - e['Description'] = desc + return desc - def process_package(self, in_entry, vars): - e = Package() + def process_package(self, in_entry, vars={}): + entry = in_entry.__class__() for key, value in in_entry.iteritems(): if isinstance(value, PackageRelation): - self.process_relation(key, e, in_entry, vars) - elif key == 'Description': - self.process_description(e, in_entry, vars) - elif key[:2] == 'X-': - pass + value = self.process_relation(value, vars) + elif isinstance(value, PackageDescription): + value = self.process_description(value, vars) else: - e[key] = self.substitute(value, vars) - return e + value = self.substitute(value, vars) + entry[key] = value + return entry - def process_packages(self, in_entries, vars): - entries = [] - for i in in_entries: - entries.append(self.process_package(i, vars)) - return entries - - def process_version_linux(self, version, abiname): - return { - 'upstreamversion': version.linux_upstream, - 'version': version.linux_version, - 'source_upstream': version.upstream, - 'abiname': abiname, - } + def process_packages(self, entries, vars): + return [self.process_package(i, vars) for i in entries] def substitute(self, s, vars): if isinstance(s, (list, tuple)): - for i in xrange(len(s)): - s[i] = self.substitute(s[i], vars) - return s + return [self.substitute(i, vars) for i in s] + def subst(match): return vars[match.group(1)] - return re.sub(r'@([-_a-z]+)@', subst, s) + + return re.sub(r'@([-_a-z]+)@', subst, unicode(s)) + + def write(self, packages, makefile): + self.write_control(packages.itervalues()) + self.write_makefile(makefile) + + def write_config(self): + f = file("debian/config.dump", 'w') + self.config.write(f) + f.close() def write_control(self, list): - self.write_rfc822(file("debian/control", 'w'), list) + self.write_rfc822(codecs.open("debian/control", 'w', 'utf-8'), list) def write_makefile(self, makefile): f = file("debian/rules.gen", 'w') @@ -304,7 +302,5 @@ class Gencontrol(object): def write_rfc822(self, f, list): for entry in list: for key, value in entry.iteritems(): - f.write("%s: %s\n" % (key, value)) - f.write('\n') - - + f.write(u"%s: %s\n" % (key, value)) + f.write(u'\n') diff --git a/debian/lib/python/debian_linux/utils.py b/debian/lib/python/debian_linux/utils.py index abe46b7b3d40..563104ed3dce 100644 --- a/debian/lib/python/debian_linux/utils.py +++ b/debian/lib/python/debian_linux/utils.py @@ -1,50 +1,20 @@ -import debian, re, os, textwrap +import codecs +import os +import re +import textwrap -class SortedDict(dict): - __slots__ = '_list', - def __init__(self, entries = None): - super(SortedDict, self).__init__() - self._list = [] - if entries is not None: - for key, value in entries: - self[key] = value - - def __delitem__(self, key): - super(SortedDict, self).__delitem__(key) - self._list.remove(key) - - def __setitem__(self, key, value): - super(SortedDict, self).__setitem__(key, value) - if key not in self._list: - self._list.append(key) - - def iterkeys(self): - for i in iter(self._list): - yield i - - def iteritems(self): - for i in iter(self._list): - yield (i, self[i]) - - def itervalues(self): - for i in iter(self._list): - yield self[i] - -class Templates(dict): - def __init__(self, dirs = ["debian/templates"]): +class Templates(object): + def __init__(self, dirs=["debian/templates"]): self.dirs = dirs - def __getitem__(self, key): - try: - return super(Templates, self).__getitem__(key) - except KeyError: pass - value = self._read(key) - super(Templates, self).__setitem__(key, value) - return value + self._cache = {} - def __setitem__(self, key, value): - raise NotImplemented() + def __getitem__(self, key): + ret = self.get(key) + if ret is not None: + return ret + raise KeyError(key) def _read(self, name): prefix, id = name.split('.', 1) @@ -52,49 +22,60 @@ class Templates(dict): for dir in self.dirs: filename = "%s/%s.in" % (dir, name) if os.path.exists(filename): - f = file(filename) + f = codecs.open(filename, 'r', 'utf-8') if prefix == 'control': - return self._read_control(f) + return read_control(f) return f.read() - raise KeyError(name) - def _read_control(self, f): - entries = [] + def get(self, key, default=None): + if key in self._cache: + return self._cache[key] + value = self._cache.setdefault(key, self._read(key)) + if value is None: + return default + return value + + +def read_control(f): + from .debian import Package + + entries = [] + eof = False + + while not eof: + e = Package() + last = None + lines = [] while True: - e = debian.Package() - last = None - lines = [] - while True: - line = f.readline() - if not line: - break - line = line.strip('\n') - if not line: - break - if line[0] in ' \t': - if not last: - raise ValueError('Continuation line seen before first header') - lines.append(line.lstrip()) - continue - if last: - e[last] = '\n'.join(lines) - i = line.find(':') - if i < 0: - raise ValueError("Not a header, not a continuation: ``%s''" % line) - last = line[:i] - lines = [line[i+1:].lstrip()] - if last: - e[last] = '\n'.join(lines) - if not e: + line = f.readline() + if not line: + eof = True break - + line = line.strip('\n') + if not line: + break + if line[0] in ' \t': + if not last: + raise ValueError(u'Continuation line seen before first header') + lines.append(line.lstrip()) + continue + if last: + e[last] = u'\n'.join(lines) + i = line.find(':') + if i < 0: + raise ValueError(u"Not a header, not a continuation: ``%s''" % line) + last = line[:i] + lines = [line[i + 1:].lstrip()] + if last: + e[last] = '\n'.join(lines) + if e: entries.append(e) - return entries + return entries + class TextWrapper(textwrap.TextWrapper): wordsep_re = re.compile( r'(\s+|' # any whitespace r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash -