| # Copyright: 2005 Gentoo Foundation |
| # Author(s): Brian Harring (ferringb@gentoo.org) |
| # License: GPL2 |
| |
| from __future__ import print_function |
| |
| __all__ = ["mirror_cache", "non_quiet_mirroring", "quiet_mirroring"] |
| |
| from itertools import chain |
| from portage.cache import cache_errors |
| from portage.localization import _ |
| |
| def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, verbose_instance=None): |
| |
| from portage import eapi_is_supported, \ |
| _validate_cache_for_unsupported_eapis |
| if not src_cache.complete_eclass_entries and not eclass_cache: |
| raise Exception("eclass_cache required for cache's of class %s!" % src_cache.__class__) |
| |
| if verbose_instance == None: |
| noise=quiet_mirroring() |
| else: |
| noise=verbose_instance |
| |
| dead_nodes = set(trg_cache) |
| count=0 |
| |
| if not trg_cache.autocommits: |
| trg_cache.sync(100) |
| |
| for x in valid_nodes_iterable: |
| # print "processing x=",x |
| count+=1 |
| dead_nodes.discard(x) |
| try: |
| entry = src_cache[x] |
| except KeyError as e: |
| noise.missing_entry(x) |
| del e |
| continue |
| except cache_errors.CacheError as ce: |
| noise.exception(x, ce) |
| del ce |
| continue |
| |
| eapi = entry.get('EAPI') |
| if not eapi: |
| eapi = '0' |
| eapi = eapi.lstrip('-') |
| eapi_supported = eapi_is_supported(eapi) |
| if not eapi_supported: |
| if not _validate_cache_for_unsupported_eapis: |
| noise.misc(x, _("unable to validate cache for EAPI='%s'") % eapi) |
| continue |
| |
| write_it = True |
| trg = None |
| try: |
| trg = trg_cache[x] |
| except (KeyError, cache_errors.CacheError): |
| pass |
| else: |
| if trg['_mtime_'] == entry['_mtime_'] and \ |
| eclass_cache.is_eclass_data_valid(trg['_eclasses_']) and \ |
| set(trg['_eclasses_']) == set(entry['_eclasses_']): |
| write_it = False |
| |
| for d in (entry, trg): |
| if d is not None and d.get('EAPI') in ('', '0'): |
| del d['EAPI'] |
| |
| if trg and not write_it: |
| """ We don't want to skip the write unless we're really sure that |
| the existing cache is identical, so don't trust _mtime_ and |
| _eclasses_ alone.""" |
| for k in set(chain(entry, trg)).difference( |
| ("_mtime_", "_eclasses_")): |
| if trg.get(k, "") != entry.get(k, ""): |
| write_it = True |
| break |
| |
| if write_it: |
| try: |
| inherited = entry.get("INHERITED", "") |
| eclasses = entry.get("_eclasses_") |
| except cache_errors.CacheError as ce: |
| noise.exception(x, ce) |
| del ce |
| continue |
| |
| if eclasses is not None: |
| if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]): |
| noise.eclass_stale(x) |
| continue |
| inherited = eclasses |
| else: |
| inherited = inherited.split() |
| |
| if inherited: |
| if src_cache.complete_eclass_entries and eclasses is None: |
| noise.corruption(x, "missing _eclasses_ field") |
| continue |
| |
| # Even if _eclasses_ already exists, replace it with data from |
| # eclass_cache, in order to insert local eclass paths. |
| try: |
| eclasses = eclass_cache.get_eclass_data(inherited) |
| except KeyError: |
| # INHERITED contains a non-existent eclass. |
| noise.eclass_stale(x) |
| continue |
| |
| if eclasses is None: |
| noise.eclass_stale(x) |
| continue |
| entry["_eclasses_"] = eclasses |
| |
| if not eapi_supported: |
| for k in set(entry).difference(("_mtime_", "_eclasses_")): |
| entry[k] = "" |
| entry["EAPI"] = "-" + eapi |
| |
| # by this time, if it reaches here, the eclass has been validated, and the entry has |
| # been updated/translated (if needs be, for metadata/cache mainly) |
| try: |
| trg_cache[x] = entry |
| except cache_errors.CacheError as ce: |
| noise.exception(x, ce) |
| del ce |
| continue |
| if count >= noise.call_update_min: |
| noise.update(x) |
| count = 0 |
| |
| if not trg_cache.autocommits: |
| trg_cache.commit() |
| |
| # ok. by this time, the trg_cache is up to date, and we have a dict |
| # with a crapload of cpv's. we now walk the target db, removing stuff if it's in the list. |
| for key in dead_nodes: |
| try: |
| del trg_cache[key] |
| except KeyError: |
| pass |
| except cache_errors.CacheError as ce: |
| noise.exception(ce) |
| del ce |
| noise.finish() |
| |
| |
| class quiet_mirroring(object): |
| # call_update_every is used by mirror_cache to determine how often to call in. |
| # quiet defaults to 2^24 -1. Don't call update, 'cept once every 16 million or so :) |
| call_update_min = 0xffffff |
| def update(self,key,*arg): pass |
| def exception(self,key,*arg): pass |
| def eclass_stale(self,*arg): pass |
| def missing_entry(self, key): pass |
| def misc(self,key,*arg): pass |
| def corruption(self, key, s): pass |
| def finish(self, *arg): pass |
| |
| class non_quiet_mirroring(quiet_mirroring): |
| call_update_min=1 |
| def update(self,key,*arg): print("processed",key) |
| def exception(self, key, *arg): print("exec",key,arg) |
| def missing(self,key): print("key %s is missing", key) |
| def corruption(self,key,*arg): print("corrupt %s:" % key,arg) |
| def eclass_stale(self,key,*arg):print("stale %s:"%key,arg) |
| |