| # -*- coding:utf-8 -*- |
| |
| import logging |
| import portage |
| |
| from itertools import chain |
| |
| from portage import normalize_path |
| from portage import os |
| from portage._sets.base import InternalPackageSet |
| from portage.output import green |
| from portage.util.futures.extendedfutures import ExtendedFuture |
| |
| from repoman.metadata import get_metadata_xsd |
| from repoman.modules.commit import repochecks |
| from repoman.modules.commit import manifest |
| from repoman.profile import check_profiles, dev_profile_keywords, setup_profile |
| from repoman.repos import repo_metadata |
| from repoman.modules.scan.module import ModuleConfig |
| from repoman.modules.scan.scan import scan |
| from repoman.modules.vcs.vcs import vcs_files_to_cps |
| |
| |
| DATA_TYPES = {"dict": dict, "Future": ExtendedFuture, "list": list, "set": set} |
| |
| |
| class Scanner: |
| """Primary scan class. Operates all the small Q/A tests and checks""" |
| |
| def __init__( |
| self, repo_settings, myreporoot, config_root, options, vcs_settings, mydir, env |
| ): |
| """Class __init__""" |
| self.repo_settings = repo_settings |
| self.config_root = config_root |
| self.options = options |
| self.vcs_settings = vcs_settings |
| self.env = env |
| |
| # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to |
| # behave incrementally. |
| self.repoman_incrementals = tuple( |
| x for x in portage.const.INCREMENTALS if x != "ACCEPT_KEYWORDS" |
| ) |
| |
| self.categories = [] |
| for path in self.repo_settings.repo_config.eclass_db.porttrees: |
| self.categories.extend( |
| portage.util.grabfile(os.path.join(path, "profiles", "categories")) |
| ) |
| self.repo_settings.repoman_settings.categories = frozenset( |
| portage.util.stack_lists([self.categories], incremental=1) |
| ) |
| self.categories = self.repo_settings.repoman_settings.categories |
| |
| self.portdb = repo_settings.portdb |
| self.portdb.settings = self.repo_settings.repoman_settings |
| |
| digest_only = ( |
| self.options.mode != "manifest-check" and self.options.digest == "y" |
| ) |
| self.generate_manifest = digest_only or self.options.mode in ( |
| "manifest", |
| "commit", |
| "fix", |
| ) |
| |
| # We really only need to cache the metadata that's necessary for visibility |
| # filtering. Anything else can be discarded to reduce memory consumption. |
| if not self.generate_manifest: |
| # Don't do this when generating manifests, since that uses |
| # additional keys if spawn_nofetch is called (RESTRICT and |
| # DEFINED_PHASES). |
| self.portdb._aux_cache_keys.clear() |
| self.portdb._aux_cache_keys.update( |
| ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"] |
| ) |
| |
| self.reposplit = myreporoot.split(os.path.sep) |
| self.repolevel = len(self.reposplit) |
| |
| if self.options.mode == "commit": |
| repochecks.commit_check(self.repolevel, self.reposplit) |
| repochecks.conflict_check(self.vcs_settings, self.options) |
| |
| # Make startdir relative to the canonical repodir, so that we can pass |
| # it to digestgen and it won't have to be canonicalized again. |
| if self.repolevel == 1: |
| startdir = self.repo_settings.repodir |
| else: |
| startdir = normalize_path(mydir) |
| startdir = os.path.join( |
| self.repo_settings.repodir, |
| *startdir.split(os.sep)[-2 - self.repolevel + 3 :] |
| ) |
| |
| # get lists of valid keywords, licenses, and use |
| new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings) |
| ( |
| kwlist, |
| liclist, |
| uselist, |
| profile_list, |
| global_pmaskdict, |
| liclist_deprecated, |
| ) = new_data |
| self.repo_metadata = { |
| "kwlist": kwlist, |
| "liclist": liclist, |
| "uselist": uselist, |
| "profile_list": profile_list, |
| "pmaskdict": global_pmaskdict, |
| "lic_deprecated": liclist_deprecated, |
| "package.deprecated": InternalPackageSet( |
| initial_atoms=portage.util.stack_lists( |
| [ |
| portage.util.grabfile_package( |
| os.path.join(path, "profiles", "package.deprecated"), |
| recursive=True, |
| ) |
| for path in self.portdb.porttrees |
| ], |
| incremental=True, |
| ) |
| ), |
| } |
| |
| self.repo_settings.repoman_settings["PORTAGE_ARCHLIST"] = " ".join( |
| sorted(kwlist) |
| ) |
| self.repo_settings.repoman_settings.backup_changes("PORTAGE_ARCHLIST") |
| |
| profiles = setup_profile(profile_list) |
| |
| check_profiles(profiles, self.repo_settings.repoman_settings.archlist()) |
| |
| scanlist = scan( |
| self.repolevel, |
| self.reposplit, |
| startdir, |
| self.categories, |
| self.repo_settings, |
| ) |
| |
| self.dev_keywords = dev_profile_keywords(profiles) |
| |
| self.qatracker = self.vcs_settings.qatracker |
| |
| if ( |
| self.options.echangelog is None |
| and self.repo_settings.repo_config.update_changelog |
| ): |
| self.options.echangelog = "y" |
| |
| if self.vcs_settings.vcs is None: |
| self.options.echangelog = "n" |
| |
| # Initialize the ModuleConfig class here |
| # TODO Add layout.conf masters repository.yml config to the list to load/stack |
| self.moduleconfig = ModuleConfig( |
| self.repo_settings.masters_list, |
| self.repo_settings.repoman_settings.valid_versions, |
| repository_modules=self.options.experimental_repository_modules == "y", |
| ) |
| |
| checks = {} |
| # The --echangelog option causes automatic ChangeLog generation, |
| # which invalidates changelog.ebuildadded and changelog.missing |
| # checks. |
| # Note: Some don't use ChangeLogs in distributed SCMs. |
| # It will be generated on server side from scm log, |
| # before package moves to the rsync server. |
| # This is needed because they try to avoid merge collisions. |
| # Gentoo's Council decided to always use the ChangeLog file. |
| # TODO: shouldn't this just be switched on the repo, iso the VCS? |
| is_echangelog_enabled = self.options.echangelog in ("y", "force") |
| self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ("cvs", "svn") |
| checks["changelog"] = ( |
| not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn |
| ) |
| |
| if self.options.mode == "manifest" or self.options.quiet: |
| pass |
| elif self.options.pretend: |
| print(green("\nRepoMan does a once-over of the neighborhood...")) |
| else: |
| print(green("\nRepoMan scours the neighborhood...")) |
| |
| self.changed = self.vcs_settings.changes |
| # bypass unneeded VCS operations if not needed |
| if self.options.if_modified == "y" or self.options.mode not in ( |
| "manifest", |
| "manifest-check", |
| ): |
| self.changed.scan() |
| |
| self.have = { |
| "pmasked": False, |
| "dev_keywords": False, |
| } |
| |
| # NOTE: match-all caches are not shared due to potential |
| # differences between profiles in _get_implicit_iuse. |
| self.caches = { |
| "arch": {}, |
| "arch_xmatch": {}, |
| "shared_xmatch": {"cp-list": {}}, |
| } |
| |
| self.include_arches = None |
| if self.options.include_arches: |
| self.include_arches = set() |
| self.include_arches.update( |
| *[x.split() for x in self.options.include_arches] |
| ) |
| self.include_profiles = None |
| if self.options.include_profiles: |
| self.include_profiles = set() |
| self.include_profiles.update( |
| *[x.split() for x in self.options.include_profiles] |
| ) |
| |
| # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and |
| # running `svn status` in every package dir will be too expensive. |
| checks["ebuild_notadded"] = not ( |
| self.vcs_settings.vcs == "svn" |
| and self.repolevel < 3 |
| and self.options.mode != "commit" |
| ) |
| |
| self.effective_scanlist = scanlist |
| if self.options.if_modified == "y": |
| self.effective_scanlist = sorted( |
| vcs_files_to_cps( |
| chain(self.changed.changed, self.changed.new, self.changed.removed), |
| self.repo_settings.repodir, |
| self.repolevel, |
| self.reposplit, |
| self.categories, |
| ) |
| ) |
| |
| # Create our kwargs dict here to initialize the plugins with |
| self.kwargs = { |
| "repo_settings": self.repo_settings, |
| "portdb": self.portdb, |
| "qatracker": self.qatracker, |
| "vcs_settings": self.vcs_settings, |
| "options": self.options, |
| "metadata_xsd": get_metadata_xsd(self.repo_settings), |
| "uselist": uselist, |
| "checks": checks, |
| "repo_metadata": self.repo_metadata, |
| "profiles": profiles, |
| "include_arches": self.include_arches, |
| "include_profiles": self.include_profiles, |
| "caches": self.caches, |
| "repoman_incrementals": self.repoman_incrementals, |
| "env": self.env, |
| "have": self.have, |
| "dev_keywords": self.dev_keywords, |
| "linechecks": self.moduleconfig.linechecks, |
| } |
| # initialize the plugin checks here |
| self.modules = {} |
| self._ext_futures = {} |
| self.pkg_level_futures = None |
| |
| def set_kwargs(self, mod): |
| """Creates a limited set of kwargs to pass to the module's __init__() |
| |
| @param mod: module name string |
| @returns: dictionary |
| """ |
| kwargs = {} |
| for key in self.moduleconfig.controller.modules[mod]["mod_kwargs"]: |
| kwargs[key] = self.kwargs[key] |
| return kwargs |
| |
| def set_func_kwargs(self, mod, dynamic_data=None): |
| """Updates the dynamic_data dictionary with any new key, value pairs. |
| Creates a limited set of kwargs to pass to the modulefunctions to run |
| |
| @param mod: module name string |
| @param dynamic_data: dictionary structure |
| @returns: dictionary |
| """ |
| func_kwargs = self.moduleconfig.controller.modules[mod]["func_kwargs"] |
| # determine new keys |
| required = set(list(func_kwargs)) |
| exist = set(list(dynamic_data)) |
| new = required.difference(exist) |
| # update dynamic_data with initialized entries |
| for key in new: |
| logging.debug("set_func_kwargs(); adding: %s, %s", key, func_kwargs[key]) |
| if func_kwargs[key][0] in ["Future", "ExtendedFuture"]: |
| if key not in self._ext_futures: |
| logging.debug( |
| "Adding a new key: %s to the ExtendedFuture dict", key |
| ) |
| self._ext_futures[key] = func_kwargs[key] |
| self._set_future(dynamic_data, key, func_kwargs[key]) |
| else: # builtin python data type |
| dynamic_data[key] = DATA_TYPES[func_kwargs[key][0]]() |
| kwargs = {} |
| for key in required: |
| kwargs[key] = dynamic_data[key] |
| return kwargs |
| |
| def reset_futures(self, dynamic_data): |
| """Reset any Future data types |
| |
| @param dynamic_data: dictionary |
| """ |
| for key in list(self._ext_futures): |
| if key not in self.pkg_level_futures: |
| self._set_future(dynamic_data, key, self._ext_futures[key]) |
| |
| @staticmethod |
| def _set_future(dynamic_data, key, data): |
| """Set a dynamic_data key to a new ExtendedFuture instance |
| |
| @param dynamic_data: dictionary |
| @param key: tuple of (dictionary-key, default-value) |
| """ |
| if data[0] in ["Future", "ExtendedFuture"]: |
| if data[1] in ["UNSET"]: |
| dynamic_data[key] = ExtendedFuture() |
| else: |
| if data[1] in DATA_TYPES: |
| default = DATA_TYPES[data[1]]() |
| else: |
| default = data[1] |
| dynamic_data[key] = ExtendedFuture(default) |
| |
| def scan_pkgs(self, can_force): |
| for xpkg in self.effective_scanlist: |
| xpkg_continue = False |
| # ebuilds and digests added to cvs respectively. |
| logging.info("checking package %s", xpkg) |
| # save memory by discarding xmatch caches from previous package(s) |
| self.caches["arch_xmatch"].clear() |
| catdir, pkgdir = xpkg.split("/") |
| checkdir = self.repo_settings.repodir + "/" + xpkg |
| checkdir_relative = "" |
| if self.repolevel < 3: |
| checkdir_relative = os.path.join(pkgdir, checkdir_relative) |
| if self.repolevel < 2: |
| checkdir_relative = os.path.join(catdir, checkdir_relative) |
| checkdir_relative = os.path.join(".", checkdir_relative) |
| |
| # Run the status check |
| if self.kwargs["checks"]["ebuild_notadded"]: |
| self.vcs_settings.status.check(checkdir, checkdir_relative, xpkg) |
| |
| if self.generate_manifest: |
| if not manifest.Manifest(**self.kwargs).update_manifest(checkdir): |
| self.qatracker.add_error( |
| "manifest.bad", os.path.join(xpkg, "Manifest") |
| ) |
| if self.options.mode == "manifest": |
| continue |
| checkdirlist = os.listdir(checkdir) |
| |
| dynamic_data = { |
| "changelog_modified": False, |
| "checkdirlist": ExtendedFuture(checkdirlist), |
| "checkdir": checkdir, |
| "xpkg": xpkg, |
| "changed": self.changed, |
| "checkdir_relative": checkdir_relative, |
| "can_force": can_force, |
| "repolevel": self.repolevel, |
| "catdir": catdir, |
| "pkgdir": pkgdir, |
| "validity_future": ExtendedFuture(True), |
| "y_ebuild": None, |
| # this needs to be reset at the pkg level only, |
| # easiest is to just initialize it here |
| "muselist": ExtendedFuture(set()), |
| "src_uri_error": ExtendedFuture(), |
| } |
| self.pkg_level_futures = [ |
| "checkdirlist", |
| "muselist", |
| "pkgs", |
| "src_uri_error", |
| "validity_future", |
| ] |
| # need to set it up for ==> self.modules or some other ordered list |
| logging.debug("***** starting pkgs_loop: %s", self.moduleconfig.pkgs_loop) |
| for mod in self.moduleconfig.pkgs_loop: |
| mod_class = self.moduleconfig.controller.get_class(mod) |
| logging.debug("Initializing class name: %s", mod_class.__name__) |
| self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) |
| logging.debug("scan_pkgs; module: %s", mod_class.__name__) |
| do_it, functions = self.modules[mod_class.__name__].runInPkgs |
| if do_it: |
| for func in functions: |
| _continue = func(**self.set_func_kwargs(mod, dynamic_data)) |
| if _continue: |
| # If we can't access all the metadata then it's totally unsafe to |
| # commit since there's no way to generate a correct Manifest. |
| # Do not try to do any more QA checks on this package since missing |
| # metadata leads to false positives for several checks, and false |
| # positives confuse users. |
| xpkg_continue = True |
| break |
| |
| if xpkg_continue: |
| continue |
| |
| # Sort ebuilds in ascending order for the KEYWORDS.dropped check. |
| pkgs = dynamic_data["pkgs"].get() |
| ebuildlist = sorted(pkgs.values()) |
| ebuildlist = [pkg.pf for pkg in ebuildlist] |
| |
| if self.kwargs["checks"]["changelog"] and "ChangeLog" not in checkdirlist: |
| self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") |
| |
| changelog_path = os.path.join(checkdir_relative, "ChangeLog") |
| dynamic_data["changelog_modified"] = ( |
| changelog_path in self.changed.changelogs |
| ) |
| |
| self._scan_ebuilds(ebuildlist, dynamic_data) |
| |
| def _scan_ebuilds(self, ebuildlist, dynamic_data): |
| |
| for y_ebuild in ebuildlist: |
| self.reset_futures(dynamic_data) |
| dynamic_data["y_ebuild"] = y_ebuild |
| |
| # initialize per ebuild plugin checks here |
| # need to set it up for ==> self.modules_list or some other ordered list |
| for mod in self.moduleconfig.ebuilds_loop: |
| if mod: |
| mod_class = self.moduleconfig.controller.get_class(mod) |
| if mod_class.__name__ not in self.modules: |
| logging.debug("Initializing class name: %s", mod_class.__name__) |
| self.modules[mod_class.__name__] = mod_class( |
| **self.set_kwargs(mod) |
| ) |
| logging.debug("scan_ebuilds: module: %s", mod_class.__name__) |
| do_it, functions = self.modules[mod_class.__name__].runInEbuilds |
| logging.debug( |
| "do_it: %s, functions: %s", do_it, [x.__name__ for x in functions] |
| ) |
| if do_it: |
| for func in functions: |
| logging.debug("\tRunning function: %s", func) |
| _continue = func(**self.set_func_kwargs(mod, dynamic_data)) |
| if _continue: |
| # If we can't access all the metadata then it's totally unsafe to |
| # commit since there's no way to generate a correct Manifest. |
| # Do not try to do any more QA checks on this package since missing |
| # metadata leads to false positives for several checks, and false |
| # positives confuse users. |
| # logging.debug("\t>>> Continuing") |
| break |
| |
| logging.debug("Finished ebuild plugin loop, continuing...") |
| |
| # Final checks |
| # initialize per pkg plugin final checks here |
| # need to set it up for ==> self.modules_list or some other ordered list |
| for mod in self.moduleconfig.final_loop: |
| if mod: |
| mod_class = self.moduleconfig.controller.get_class(mod) |
| if mod_class.__name__ not in self.modules: |
| logging.debug("Initializing class name: %s", mod_class.__name__) |
| self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) |
| logging.debug("scan_ebuilds final checks: module: %s", mod_class.__name__) |
| do_it, functions = self.modules[mod_class.__name__].runInFinal |
| logging.debug( |
| "do_it: %s, functions: %s", do_it, [x.__name__ for x in functions] |
| ) |
| if do_it: |
| for func in functions: |
| logging.debug("\tRunning function: %s", func) |
| _continue = func(**self.set_func_kwargs(mod, dynamic_data)) |
| if _continue: |
| # logging.debug("\t>>> Continuing") |
| break |
