| #!/usr/bin/python |
| # Copyright 2009-2010 Gentoo Foundation |
| # Distributed under the terms of the GNU General Public License v2 |
| |
| from __future__ import print_function |
| |
| import sys |
| # This block ensures that ^C interrupts are handled quietly. |
| try: |
| import signal |
| |
| def exithandler(signum,frame): |
| signal.signal(signal.SIGINT, signal.SIG_IGN) |
| signal.signal(signal.SIGTERM, signal.SIG_IGN) |
| sys.exit(1) |
| |
| signal.signal(signal.SIGINT, exithandler) |
| signal.signal(signal.SIGTERM, exithandler) |
| |
| except KeyboardInterrupt: |
| sys.exit(1) |
| |
| import codecs |
| import logging |
| import optparse |
| |
| try: |
| import portage |
| except ImportError: |
| from os import path as osp |
| sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) |
| import portage |
| |
| from portage import os, _encodings, _unicode_encode |
| from _emerge.MetadataRegen import MetadataRegen |
| from portage.cache.cache_errors import CacheError, StatCollision |
| from portage.util import cmp_sort_key, writemsg_level |
| from portage import cpv_getkey |
| from portage.dep import Atom, isjustname |
| from portage.versions import pkgcmp, pkgsplit |
| |
| try: |
| import xml.etree.ElementTree |
| except ImportError: |
| pass |
| else: |
| from repoman.utilities import parse_metadata_use |
| from xml.parsers.expat import ExpatError |
| |
| if sys.hexversion >= 0x3000000: |
| long = int |
| |
| def parse_args(args): |
| usage = "egencache [options] <action> ... [atom] ..." |
| parser = optparse.OptionParser(usage=usage) |
| |
| actions = optparse.OptionGroup(parser, 'Actions') |
| actions.add_option("--update", |
| action="store_true", |
| help="update metadata/cache/ (generate as necessary)") |
| actions.add_option("--update-use-local-desc", |
| action="store_true", |
| help="update the use.local.desc file from metadata.xml") |
| parser.add_option_group(actions) |
| |
| common = optparse.OptionGroup(parser, 'Common options') |
| common.add_option("--repo", |
| action="store", |
| help="name of repo to operate on (default repo is located at $PORTDIR)") |
| common.add_option("--config-root", |
| help="location of portage config files", |
| dest="portage_configroot") |
| common.add_option("--portdir", |
| help="override the portage tree location", |
| dest="portdir") |
| common.add_option("--tolerant", |
| action="store_true", |
| help="exit successfully if only minor errors occurred") |
| common.add_option("--ignore-default-opts", |
| action="store_true", |
| help="do not use the EGENCACHE_DEFAULT_OPTS environment variable") |
| parser.add_option_group(common) |
| |
| update = optparse.OptionGroup(parser, '--update options') |
| update.add_option("--cache-dir", |
| help="location of the metadata cache", |
| dest="cache_dir") |
| update.add_option("--jobs", |
| action="store", |
| help="max ebuild processes to spawn") |
| update.add_option("--load-average", |
| action="store", |
| help="max load allowed when spawning multiple jobs", |
| dest="load_average") |
| update.add_option("--rsync", |
| action="store_true", |
| help="enable rsync stat collision workaround " + \ |
| "for bug 139134 (use with --update)") |
| parser.add_option_group(update) |
| |
| uld = optparse.OptionGroup(parser, '--update-use-local-desc options') |
| uld.add_option("--preserve-comments", |
| action="store_true", |
| help="preserve the comments from the existing use.local.desc file") |
| uld.add_option("--use-local-desc-output", |
| help="output file for use.local.desc data (or '-' for stdout)", |
| dest="uld_output") |
| parser.add_option_group(uld) |
| |
| options, args = parser.parse_args(args) |
| |
| if options.jobs: |
| jobs = None |
| try: |
| jobs = int(options.jobs) |
| except ValueError: |
| jobs = -1 |
| |
| if jobs < 1: |
| parser.error("Invalid: --jobs='%s'" % \ |
| (options.jobs,)) |
| |
| options.jobs = jobs |
| |
| else: |
| options.jobs = None |
| |
| if options.load_average: |
| try: |
| load_average = float(options.load_average) |
| except ValueError: |
| load_average = 0.0 |
| |
| if load_average <= 0.0: |
| parser.error("Invalid: --load-average='%s'" % \ |
| (options.load_average,)) |
| |
| options.load_average = load_average |
| |
| else: |
| options.load_average = None |
| |
| options.config_root = options.portage_configroot |
| if options.config_root is not None and \ |
| not os.path.isdir(options.config_root): |
| parser.error("Not a directory: --config-root='%s'" % \ |
| (options.config_root,)) |
| |
| if options.cache_dir is not None and not os.path.isdir(options.cache_dir): |
| parser.error("Not a directory: --cache-dir='%s'" % \ |
| (options.cache_dir,)) |
| |
| for atom in args: |
| try: |
| atom = portage.dep.Atom(atom) |
| except portage.exception.InvalidAtom: |
| parser.error('Invalid atom: %s' % (atom,)) |
| |
| if not isjustname(atom): |
| parser.error('Atom is too specific: %s' % (atom,)) |
| |
| if options.update_use_local_desc: |
| try: |
| xml.etree.ElementTree |
| except NameError: |
| parser.error('--update-use-local-desc requires python with USE=xml!') |
| |
| if options.uld_output == '-' and options.preserve_comments: |
| parser.error('--preserve-comments can not be used when outputting to stdout') |
| |
| return parser, options, args |
| |
| class GenCache(object): |
| def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, |
| rsync=False): |
| self._portdb = portdb |
| # We can globally cleanse stale cache only if we |
| # iterate over every single cp. |
| self._global_cleanse = cp_iter is None |
| if cp_iter is not None: |
| self._cp_set = set(cp_iter) |
| cp_iter = iter(self._cp_set) |
| self._cp_missing = self._cp_set.copy() |
| else: |
| self._cp_set = None |
| self._cp_missing = set() |
| self._regen = MetadataRegen(portdb, cp_iter=cp_iter, |
| consumer=self._metadata_callback, |
| max_jobs=max_jobs, max_load=max_load) |
| self.returncode = os.EX_OK |
| metadbmodule = portdb.settings.load_best_module("portdbapi.metadbmodule") |
| self._trg_cache = metadbmodule(portdb.porttrees[0], |
| "metadata/cache", portage.auxdbkeys[:]) |
| if rsync: |
| self._trg_cache.raise_stat_collision = True |
| try: |
| self._trg_cache.ec = \ |
| portdb._repo_info[portdb.porttrees[0]].eclass_db |
| except AttributeError: |
| pass |
| self._existing_nodes = set() |
| |
| def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata): |
| self._existing_nodes.add(cpv) |
| self._cp_missing.discard(cpv_getkey(cpv)) |
| if metadata is not None: |
| if metadata.get('EAPI') == '0': |
| del metadata['EAPI'] |
| try: |
| try: |
| self._trg_cache[cpv] = metadata |
| except StatCollision as sc: |
| # If the content of a cache entry changes and neither the |
| # file mtime nor size changes, it will prevent rsync from |
| # detecting changes. Cache backends may raise this |
| # exception from _setitem() if they detect this type of stat |
| # collision. These exceptions are handled by bumping the |
| # mtime on the ebuild (and the corresponding cache entry). |
| # See bug #139134. |
| max_mtime = sc.mtime |
| for ec, (loc, ec_mtime) in metadata['_eclasses_'].items(): |
| if max_mtime < ec_mtime: |
| max_mtime = ec_mtime |
| if max_mtime == sc.mtime: |
| max_mtime += 1 |
| max_mtime = long(max_mtime) |
| try: |
| os.utime(ebuild_path, (max_mtime, max_mtime)) |
| except OSError as e: |
| self.returncode |= 1 |
| writemsg_level( |
| "%s writing target: %s\n" % (cpv, e), |
| level=logging.ERROR, noiselevel=-1) |
| else: |
| metadata['_mtime_'] = max_mtime |
| self._trg_cache[cpv] = metadata |
| self._portdb.auxdb[repo_path][cpv] = metadata |
| |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "%s writing target: %s\n" % (cpv, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| def run(self): |
| self._regen.run() |
| self.returncode |= self._regen.returncode |
| cp_missing = self._cp_missing |
| |
| trg_cache = self._trg_cache |
| dead_nodes = set() |
| if self._global_cleanse: |
| try: |
| for cpv in trg_cache: |
| cp = cpv_getkey(cpv) |
| if cp is None: |
| self.returncode |= 1 |
| writemsg_level( |
| "Unable to parse cp for '%s'\n" % (cpv,), |
| level=logging.ERROR, noiselevel=-1) |
| else: |
| dead_nodes.add(cpv) |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "Error listing cache entries for " + \ |
| "'%s/metadata/cache': %s, continuing...\n" % \ |
| (self._portdb.porttree_root, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| else: |
| cp_set = self._cp_set |
| try: |
| for cpv in trg_cache: |
| cp = cpv_getkey(cpv) |
| if cp is None: |
| self.returncode |= 1 |
| writemsg_level( |
| "Unable to parse cp for '%s'\n" % (cpv,), |
| level=logging.ERROR, noiselevel=-1) |
| else: |
| cp_missing.discard(cp) |
| if cp in cp_set: |
| dead_nodes.add(cpv) |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "Error listing cache entries for " + \ |
| "'%s/metadata/cache': %s, continuing...\n" % \ |
| (self._portdb.porttree_root, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| if cp_missing: |
| self.returncode |= 1 |
| for cp in sorted(cp_missing): |
| writemsg_level( |
| "No ebuilds or cache entries found for '%s'\n" % (cp,), |
| level=logging.ERROR, noiselevel=-1) |
| |
| if dead_nodes: |
| dead_nodes.difference_update(self._existing_nodes) |
| for k in dead_nodes: |
| try: |
| del trg_cache[k] |
| except KeyError: |
| pass |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "%s deleting stale cache: %s\n" % (k, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| if not trg_cache.autocommits: |
| try: |
| trg_cache.commit() |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "committing target: %s\n" % (ce,), |
| level=logging.ERROR, noiselevel=-1) |
| |
| class GenUseLocalDesc(object): |
| def __init__(self, portdb, output=None, |
| preserve_comments=False): |
| self.returncode = os.EX_OK |
| self._portdb = portdb |
| self._output = output |
| self._preserve_comments = preserve_comments |
| |
| def run(self): |
| repo_path = self._portdb.porttrees[0] |
| ops = {'<':0, '<=':1, '=':2, '>=':3, '>':4} |
| |
| if self._output is None or self._output != '-': |
| if self._output is None: |
| prof_path = os.path.join(repo_path, 'profiles') |
| desc_path = os.path.join(prof_path, 'use.local.desc') |
| try: |
| os.mkdir(prof_path) |
| except OSError: |
| pass |
| else: |
| desc_path = self._output |
| |
| try: |
| if self._preserve_comments: |
| # Probe in binary mode, in order to avoid |
| # potential character encoding issues. |
| output = open(_unicode_encode(desc_path, |
| encoding=_encodings['fs'], errors='strict'), 'r+b') |
| else: |
| output = codecs.open(_unicode_encode(desc_path, |
| encoding=_encodings['fs'], errors='strict'), |
| mode='w', encoding=_encodings['repo.content'], |
| errors='replace') |
| except IOError as e: |
| writemsg_level( |
| "ERROR: failed to open output file %s: %s\n" % (output_mode,e,), |
| level=logging.ERROR, noiselevel=-1) |
| self.returncode |= 2 |
| return |
| else: |
| output = sys.stdout |
| |
| if self._preserve_comments: |
| while True: |
| pos = output.tell() |
| if not output.readline().startswith(b'#'): |
| break |
| output.seek(pos) |
| output.truncate() |
| output.close() |
| |
| # Finished probing comments in binary mode, now append |
| # in text mode. |
| output = codecs.open(_unicode_encode(desc_path, |
| encoding=_encodings['fs'], errors='strict'), |
| mode='a', encoding=_encodings['repo.content'], |
| errors='replace') |
| output.write('\n') |
| else: |
| output.write(''' |
| # This file is deprecated as per GLEP 56 in favor of metadata.xml. Please add |
| # your descriptions to your package's metadata.xml ONLY. |
| # * generated automatically using egencache * |
| |
| '''.lstrip()) |
| |
| # The cmp function no longer exists in python3, so we'll |
| # implement our own here under a slightly different name |
| # since we don't want any confusion given that we never |
| # want to rely on the builtin cmp function. |
| def cmp_func(a, b): |
| return (a > b) - (a < b) |
| |
| for cp in self._portdb.cp_all(): |
| metadata_path = os.path.join(repo_path, cp, 'metadata.xml') |
| try: |
| metadata = xml.etree.ElementTree.parse(metadata_path) |
| except IOError: |
| pass |
| except (ExpatError, EnvironmentError) as e: |
| writemsg_level( |
| "ERROR: failed parsing %s/metadata.xml: %s\n" % (cp, e), |
| level=logging.ERROR, noiselevel=-1) |
| self.returncode |= 1 |
| else: |
| try: |
| usedict = parse_metadata_use(metadata) |
| except portage.exception.ParseError as e: |
| writemsg_level( |
| "ERROR: failed parsing %s/metadata.xml: %s\n" % (cp, e), |
| level=logging.ERROR, noiselevel=-1) |
| self.returncode |= 1 |
| else: |
| for flag in sorted(usedict): |
| def atomcmp(atoma, atomb): |
| # None is better than an atom, that's why we reverse the args |
| if atoma is None or atomb is None: |
| return cmp_func(atomb, atoma) |
| # Same for plain PNs (.operator is None then) |
| elif atoma.operator is None or atomb.operator is None: |
| return cmp_func(atomb.operator, atoma.operator) |
| # Version matching |
| elif atoma.cpv != atomb.cpv: |
| return pkgcmp(pkgsplit(atoma.cpv), pkgsplit(atomb.cpv)) |
| # Versions match, let's fallback to operator matching |
| else: |
| return cmp_func(ops.get(atoma.operator, -1), |
| ops.get(atomb.operator, -1)) |
| |
| def _Atom(key): |
| if key is not None: |
| return Atom(key) |
| return None |
| |
| resdict = usedict[flag] |
| if len(resdict) == 1: |
| resdesc = next(iter(resdict.items()))[1] |
| else: |
| try: |
| reskeys = dict((_Atom(k), k) for k in resdict) |
| except portage.exception.InvalidAtom as e: |
| writemsg_level( |
| "ERROR: failed parsing %s/metadata.xml: %s\n" % (cp, e), |
| level=logging.ERROR, noiselevel=-1) |
| self.returncode |= 1 |
| resdesc = next(iter(resdict.items()))[1] |
| else: |
| resatoms = sorted(reskeys, key=cmp_sort_key(atomcmp)) |
| resdesc = resdict[reskeys[resatoms[-1]]] |
| |
| output.write('%s:%s - %s\n' % (cp, flag, resdesc)) |
| |
| output.close() |
| |
| def egencache_main(args): |
| parser, options, atoms = parse_args(args) |
| |
| config_root = options.config_root |
| if config_root is None: |
| config_root = '/' |
| |
| # The calling environment is ignored, so the program is |
| # completely controlled by commandline arguments. |
| env = {} |
| |
| if options.repo is None: |
| env['PORTDIR_OVERLAY'] = '' |
| |
| if options.cache_dir is not None: |
| env['PORTAGE_DEPCACHEDIR'] = options.cache_dir |
| |
| if options.portdir is not None: |
| env['PORTDIR'] = options.portdir |
| |
| settings = portage.config(config_root=config_root, |
| target_root='/', local_config=False, env=env) |
| |
| default_opts = None |
| if not options.ignore_default_opts: |
| default_opts = settings.get('EGENCACHE_DEFAULT_OPTS', '').split() |
| |
| if default_opts: |
| parser, options, args = parse_args(default_opts + args) |
| |
| if options.config_root is not None: |
| config_root = options.config_root |
| |
| if options.cache_dir is not None: |
| env['PORTAGE_DEPCACHEDIR'] = options.cache_dir |
| |
| settings = portage.config(config_root=config_root, |
| target_root='/', local_config=False, env=env) |
| |
| if not options.update and not options.update_use_local_desc: |
| parser.error('No action specified (--update ' + \ |
| 'and/or --update-use-local-desc)') |
| return 1 |
| |
| if options.update and 'metadata-transfer' not in settings.features: |
| writemsg_level("ecachegen: warning: " + \ |
| "automatically enabling FEATURES=metadata-transfer\n", |
| level=logging.WARNING, noiselevel=-1) |
| settings.features.add('metadata-transfer') |
| |
| settings.lock() |
| |
| portdb = portage.portdbapi(mysettings=settings) |
| if options.repo is not None: |
| repo_path = portdb.getRepositoryPath(options.repo) |
| if repo_path is None: |
| parser.error("Unable to locate repository named '%s'" % \ |
| (options.repo,)) |
| return 1 |
| |
| # Limit ebuilds to the specified repo. |
| portdb.porttrees = [repo_path] |
| |
| ret = [os.EX_OK] |
| |
| if options.update: |
| cp_iter = None |
| if atoms: |
| cp_iter = iter(atoms) |
| |
| gen_cache = GenCache(portdb, cp_iter=cp_iter, |
| max_jobs=options.jobs, |
| max_load=options.load_average, |
| rsync=options.rsync) |
| gen_cache.run() |
| ret.append(gen_cache.returncode) |
| |
| if options.update_use_local_desc: |
| gen_desc = GenUseLocalDesc(portdb, |
| output=options.uld_output, |
| preserve_comments=options.preserve_comments) |
| gen_desc.run() |
| ret.append(gen_desc.returncode) |
| |
| if options.tolerant: |
| return ret[0] |
| return max(ret) |
| |
| if __name__ == "__main__": |
| portage._disable_legacy_globals() |
| portage.util.noiselimit = -1 |
| sys.exit(egencache_main(sys.argv[1:])) |