| #!/usr/bin/python |
| # Copyright 2009 Gentoo Foundation |
| # Distributed under the terms of the GNU General Public License v2 |
| # $Id$ |
| |
| from __future__ import print_function |
| |
| import sys |
| # This block ensures that ^C interrupts are handled quietly. |
| try: |
| import signal |
| |
| def exithandler(signum,frame): |
| signal.signal(signal.SIGINT, signal.SIG_IGN) |
| signal.signal(signal.SIGTERM, signal.SIG_IGN) |
| sys.exit(1) |
| |
| signal.signal(signal.SIGINT, exithandler) |
| signal.signal(signal.SIGTERM, exithandler) |
| |
| except KeyboardInterrupt: |
| sys.exit(1) |
| |
| import logging |
| import optparse |
| |
| try: |
| import portage |
| except ImportError: |
| from os import path as osp |
| sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) |
| import portage |
| |
| from portage import os |
| from _emerge.MetadataRegen import MetadataRegen |
| from portage.cache.cache_errors import CacheError, StatCollision |
| from portage.util import writemsg_level |
| from portage import cpv_getkey |
| from portage.dep import isjustname |
| |
| if sys.hexversion >= 0x3000000: |
| long = int |
| |
| def parse_args(args): |
| usage = "egencache [options] --update [atom] ..." |
| parser = optparse.OptionParser(usage=usage) |
| parser.add_option("--update", |
| action="store_true", |
| help="update metadata/cache/ (generate as necessary)") |
| parser.add_option("--repo", |
| action="store", |
| help="name of repo to operate on (default repo is located at $PORTDIR)") |
| parser.add_option("--cache-dir", |
| help="location of the metadata cache", |
| dest="cache_dir") |
| parser.add_option("--config-root", |
| help="location of portage config files", |
| dest="config_root") |
| parser.add_option("--jobs", |
| action="store", |
| help="max ebuild processes to spawn") |
| parser.add_option("--load-average", |
| action="store", |
| help="max load allowed when spawning multiple jobs", |
| dest="load_average") |
| parser.add_option("--rsync", |
| action="store_true", |
| help="enable rsync stat collision workaround " + \ |
| "for bug 139134 (use with --update)") |
| parser.add_option("--tolerant", |
| action="store_true", |
| help="exit successfully if only minor errors occurred") |
| parser.add_option("--ignore-default-opts", |
| action="store_true", |
| help="do not use the EGENCACHE_DEFAULT_OPTS environment variable") |
| options, args = parser.parse_args(args) |
| |
| if options.jobs: |
| jobs = None |
| try: |
| jobs = int(options.jobs) |
| except ValueError: |
| jobs = -1 |
| |
| if jobs < 1: |
| parser.error("Invalid: --jobs='%s'" % \ |
| (options.jobs,)) |
| |
| options.jobs = jobs |
| |
| else: |
| options.jobs = None |
| |
| if options.load_average: |
| try: |
| load_average = float(options.load_average) |
| except ValueError: |
| load_average = 0.0 |
| |
| if load_average <= 0.0: |
| parser.error("Invalid: --load-average='%s'" % \ |
| (options.load_average,)) |
| |
| options.load_average = load_average |
| |
| else: |
| options.load_average = None |
| |
| if options.config_root is not None and \ |
| not os.path.isdir(options.config_root): |
| parser.error("Not a directory: --config-root='%s'" % \ |
| (options.config_root,)) |
| |
| if options.cache_dir is not None and not os.path.isdir(options.cache_dir): |
| parser.error("Not a directory: --cache-dir='%s'" % \ |
| (options.cache_dir,)) |
| |
| for atom in args: |
| try: |
| atom = portage.dep.Atom(atom) |
| except portage.exception.InvalidAtom: |
| parser.error('Invalid atom: %s' % (atom,)) |
| |
| if not isjustname(atom): |
| parser.error('Atom is too specific: %s' % (atom,)) |
| |
| return parser, options, args |
| |
| class GenCache(object): |
| def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, |
| rsync=False): |
| self._portdb = portdb |
| # We can globally cleanse stale cache only if we |
| # iterate over every single cp. |
| self._global_cleanse = cp_iter is None |
| if cp_iter is not None: |
| self._cp_set = set(cp_iter) |
| cp_iter = iter(self._cp_set) |
| self._cp_missing = self._cp_set.copy() |
| else: |
| self._cp_set = None |
| self._cp_missing = set() |
| self._regen = MetadataRegen(portdb, cp_iter=cp_iter, |
| consumer=self._metadata_callback, |
| max_jobs=max_jobs, max_load=max_load) |
| self.returncode = os.EX_OK |
| metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule") |
| self._trg_cache = metadbmodule(portdb.porttrees[0], |
| "metadata/cache", portage.auxdbkeys[:]) |
| if rsync: |
| self._trg_cache.raise_stat_collision = True |
| try: |
| self._trg_cache.ec = \ |
| portdb._repo_info[portdb.porttrees[0]].eclass_db |
| except AttributeError: |
| pass |
| self._existing_nodes = set() |
| |
| def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata): |
| self._existing_nodes.add(cpv) |
| self._cp_missing.discard(cpv_getkey(cpv)) |
| if metadata is not None: |
| if metadata.get('EAPI') == '0': |
| del metadata['EAPI'] |
| try: |
| try: |
| self._trg_cache[cpv] = metadata |
| except StatCollision as sc: |
| # If the content of a cache entry changes and neither the |
| # file mtime nor size changes, it will prevent rsync from |
| # detecting changes. Cache backends may raise this |
| # exception from _setitem() if they detect this type of stat |
| # collision. These exceptions are handled by bumping the |
| # mtime on the ebuild (and the corresponding cache entry). |
| # See bug #139134. |
| max_mtime = sc.mtime |
| for ec, (loc, ec_mtime) in metadata['_eclasses_'].items(): |
| if max_mtime < ec_mtime: |
| max_mtime = ec_mtime |
| if max_mtime == sc.mtime: |
| max_mtime += 1 |
| max_mtime = long(max_mtime) |
| try: |
| os.utime(ebuild_path, (max_mtime, max_mtime)) |
| except OSError as e: |
| self.returncode |= 1 |
| writemsg_level( |
| "%s writing target: %s\n" % (cpv, e), |
| level=logging.ERROR, noiselevel=-1) |
| else: |
| metadata['_mtime_'] = max_mtime |
| self._trg_cache[cpv] = metadata |
| self._portdb.auxdb[repo_path][cpv] = metadata |
| |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "%s writing target: %s\n" % (cpv, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| def run(self): |
| self._regen.run() |
| self.returncode |= self._regen.returncode |
| cp_missing = self._cp_missing |
| |
| trg_cache = self._trg_cache |
| dead_nodes = set() |
| if self._global_cleanse: |
| try: |
| for cpv in trg_cache: |
| cp = cpv_getkey(cpv) |
| if cp is None: |
| self.returncode |= 1 |
| writemsg_level( |
| "Unable to parse cp for '%s'\n" % (cpv,), |
| level=logging.ERROR, noiselevel=-1) |
| else: |
| dead_nodes.add(cpv) |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "Error listing cache entries for " + \ |
| "'%s/metadata/cache': %s, continuing...\n" % \ |
| (self._portdb.porttree_root, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| else: |
| cp_set = self._cp_set |
| try: |
| for cpv in trg_cache: |
| cp = cpv_getkey(cpv) |
| if cp is None: |
| self.returncode |= 1 |
| writemsg_level( |
| "Unable to parse cp for '%s'\n" % (cpv,), |
| level=logging.ERROR, noiselevel=-1) |
| else: |
| cp_missing.discard(cp) |
| if cp in cp_set: |
| dead_nodes.add(cpv) |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "Error listing cache entries for " + \ |
| "'%s/metadata/cache': %s, continuing...\n" % \ |
| (self._portdb.porttree_root, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| if cp_missing: |
| self.returncode |= 1 |
| for cp in sorted(cp_missing): |
| writemsg_level( |
| "No ebuilds or cache entries found for '%s'\n" % (cp,), |
| level=logging.ERROR, noiselevel=-1) |
| |
| if dead_nodes: |
| dead_nodes.difference_update(self._existing_nodes) |
| for k in dead_nodes: |
| try: |
| del trg_cache[k] |
| except KeyError: |
| pass |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "%s deleting stale cache: %s\n" % (k, ce), |
| level=logging.ERROR, noiselevel=-1) |
| |
| if not trg_cache.autocommits: |
| try: |
| trg_cache.commit() |
| except CacheError as ce: |
| self.returncode |= 1 |
| writemsg_level( |
| "committing target: %s\n" % (ce,), |
| level=logging.ERROR, noiselevel=-1) |
| |
| def egencache_main(args): |
| parser, options, atoms = parse_args(args) |
| |
| config_root = options.config_root |
| if config_root is None: |
| config_root = '/' |
| |
| # The calling environment is ignored, so the program is |
| # completely controlled by commandline arguments. |
| env = {} |
| |
| if options.repo is None: |
| env['PORTDIR_OVERLAY'] = '' |
| |
| if options.cache_dir is not None: |
| env['PORTAGE_DEPCACHEDIR'] = options.cache_dir |
| |
| settings = portage.config(config_root=config_root, |
| target_root='/', local_config=False, env=env) |
| |
| default_opts = None |
| if not options.ignore_default_opts: |
| default_opts = settings.get('EGENCACHE_DEFAULT_OPTS', '').split() |
| |
| if default_opts: |
| parser, options, args = parse_args(default_opts + args) |
| |
| if options.config_root is not None: |
| config_root = options.config_root |
| |
| if options.cache_dir is not None: |
| env['PORTAGE_DEPCACHEDIR'] = options.cache_dir |
| |
| settings = portage.config(config_root=config_root, |
| target_root='/', local_config=False, env=env) |
| |
| if not options.update: |
| parser.error('No action specified (--update ' + \ |
| 'is the only available action)') |
| return 1 |
| |
| if 'metadata-transfer' not in settings.features: |
| writemsg_level("ecachegen: warning: " + \ |
| "automatically enabling FEATURES=metadata-transfer\n", |
| level=logging.WARNING, noiselevel=-1) |
| settings.features.add('metadata-transfer') |
| settings['FEATURES'] = ' '.join(sorted(settings.features)) |
| settings.backup_changes('FEATURES') |
| |
| settings.lock() |
| |
| portdb = portage.portdbapi(settings["PORTDIR"], mysettings=settings) |
| if options.repo is not None: |
| repo_path = portdb.getRepositoryPath(options.repo) |
| if repo_path is None: |
| parser.error("Unable to locate repository named '%s'" % \ |
| (options.repo,)) |
| return 1 |
| |
| # Limit ebuilds to the specified repo. |
| portdb.porttrees = [repo_path] |
| |
| cp_iter = None |
| if atoms: |
| cp_iter = iter(atoms) |
| |
| gen_cache = GenCache(portdb, cp_iter=cp_iter, |
| max_jobs=options.jobs, |
| max_load=options.load_average, |
| rsync=options.rsync) |
| gen_cache.run() |
| if options.tolerant: |
| return os.EX_OK |
| return gen_cache.returncode |
| |
| if __name__ == "__main__": |
| portage._disable_legacy_globals() |
| portage.util.noiselimit = -1 |
| sys.exit(egencache_main(sys.argv[1:])) |