| # Copyright 2012 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Library for validating ebuild license information, and generating credits. |
| |
| Documentation on this script is also available here: |
| https://dev.chromium.org/chromium-os/licensing |
| """ |
| |
| import codecs |
| import fnmatch |
| import html |
| import json |
| import logging |
| import os |
| from pathlib import Path |
| import re |
| from typing import List, Optional |
| |
| from chromite.lib import constants |
| from chromite.lib import cros_build_lib |
| from chromite.lib import osutils |
| from chromite.lib import portage_util |
| from chromite.lib import sysroot_lib |
| from chromite.lib.parser import package_info |
| from chromite.utils import gs_urls_util |
| from chromite.utils.parser import ebuild_license |
| |
| |
| # See https://crbug.com/207004 for discussion. |
| PER_PKG_LICENSE_DIR = portage_util.VDB_PATH |
| |
| STOCK_LICENSE_DIRS = [ |
| "src/third_party/portage-stable/licenses", |
| ] |
| |
| # The SDK does not have generated overlay info in it, so hardcode these |
| # fallbacks for them. |
| SDK_OVERLAY_DIRS = [ |
| "src/third_party/chromiumos-overlay", |
| "src/private-overlays/chromeos-overlay", |
| ] |
| |
| COPYRIGHT_ATTRIBUTION_DIR = os.path.join( |
| constants.SOURCE_ROOT, |
| "src/third_party/chromiumos-overlay/licenses/copyright-attribution", |
| ) |
| |
| # Virtual packages don't need to have a license and often don't, so we skip them |
| # chromeos-base contains google platform packages that are covered by the |
| # general license at top of tree, so we skip those too. |
| SKIPPED_CATEGORIES = [ |
| "virtual", |
| ] |
| |
| # If you have an early package for which license terms have yet to be decided, |
| # use this. It will cause licensing for the package to be mostly ignored. |
| # Tainted builds will fail signing with official keys. |
| TAINTED = "TAINTED" |
| |
| # HTML outputs will include this tag if tainted. |
| TAINTED_COMMENT_TAG = "<!-- tainted -->" |
| |
| SKIPPED_LICENSES = [ |
| # Some of our packages contain binary blobs for which we have special |
| # negotiated licenses, and no need to display anything publicly. Strongly |
| # consider using Google-TOS instead, if possible. |
| "Proprietary-Binary", |
| ] |
| |
| LICENSE_BASENAMES_REGEX = [ |
| r"^copyright$", |
| r"^copyright[.]txt$", |
| r"^copyright[.]regex$", # llvm |
| r"^copying.*$", |
| r"^licen[cs]e.*$", |
| r"^licensing.*$", # libatomic_ops |
| r"^ipa_font_license_agreement_v1[.]0[.]txt$", # ja-ipafonts |
| r"^MIT-LICENSE$", # rake |
| r"^PKG-INFO$", # copyright assignment for |
| # some python packages |
| # (netifaces, unittest2) |
| ] |
| |
| # Patterns to never classify as a license file. Unlike LICENSE_BASENAMES_REGEX, |
| # the patterns here are applied to the full path relative to the work dir root. |
| # Exclusions found this way are logged. We also skip `.git`, but don't log it. |
| LICENSE_PATHS_EXCLUDE_REGEX = ( |
| # We never want a GPL license when looking for copyright attribution, so we |
| # skip things like license.gpl. Only check the basename. |
| r"[^/]*GPL[^/]*$", |
| # Skip files that are likely source code or object files generated from |
| # source code. E.g., license.py, license.o, etc. |
| r"\.py$", |
| r"\.o$", |
| # Haskell source and object files. |
| r"\.hs$", |
| r"\.hi$", |
| r"\.dyn_hi$", |
| r"\.dyn_o$", |
| # This folder contains json files listing every license known to Cabal. |
| r"Cabal/license-list-data", |
| ) |
| |
| # Any license listed here found in the ebuild will make the code look for |
| # license files inside the package source code in order to get copyright |
| # attribution from them. |
| COPYRIGHT_ATTRIBUTION_LICENSES = { |
| "BSD", # requires distribution of copyright notice |
| "BSD-2", # so does BSD-2 https://opensource.org/licenses/BSD-2-Clause |
| "BSD-2-with-patent", |
| "BSD-3", # and BSD-3? https://opensource.org/licenses/BSD-3-Clause |
| "BSD-4", # and 4? |
| "BSD-with-attribution", |
| "BSD-with-disclosure", |
| "ISC", # so does ISC https://opensource.org/licenses/ISC |
| "MIT", |
| "MIT-with-advertising", |
| "Old-MIT", |
| } |
| |
| # The following licenses are not invalid or to show as a less helpful stock |
| # license, but it's better to look in the source code for a more specific |
| # license if there is one, but not an error if no better one is found. |
| # Note that you don't want to set just anything here since any license here |
| # will be included once in stock form and a second time in custom form if |
| # found (there is no good way to know that a license we found on disk is the |
| # better version of the stock version, so we show both). |
| LOOK_IN_SOURCE_LICENSES = [ |
| "PSF-2", # The custom license in python is more complete than the template. |
| # As far as I know, we have no requirement to do copyright attribution for |
| # these licenses, but the license included in the code has slightly better |
| # information than the stock Gentoo one (including copyright attribution). |
| "BZIP2", # Single use license, do copyright attribution. |
| "OFL", # Almost single use license, do copyright attribution. |
| "OFL-1.1", # Almost single use license, do copyright attribution. |
| "UoI-NCSA", # Only used by NSCA, might as well show their custom copyright. |
| ] |
| |
| # These are licenses we don't want to allow anyone to use. Globs allowed. |
| # |
| # Gentoo normally uses a naming convention of <name> or <name>-<ver>. We don't |
| # block AGPL* in the pathological case that an unrelated license comes up with |
| # the same first few letters. Not likely, but not impossible. But we do block |
| # AGPL-* as it's much more likely we'll get a newer version using the same base |
| # name which we want to be future proof against. |
| BAD_LICENSE_PATTERNS = { |
| # Google policy to avoid these licenses (and all versions of them). |
| # https://opensource.org/licenses/AGPL-3.0 |
| "AGPL", |
| "AGPL-*", |
| # Catch all versions & variants of CC-BY-NC, CC-BY-NC-ND, and CC-BY-NC-SA. |
| # https://creativecommons.org/licenses/by-nc/3.0/ |
| "CC-BY-NC*", |
| # https://opensource.org/licenses/CPAL-1.0 |
| "CPAL", |
| "CPAL-*", |
| # https://opensource.org/licenses/EUPL-1.2 |
| "EUPL", |
| "EUPL-*", |
| # https://en.wikipedia.org/wiki/WTFPL nocheck |
| "WTFPL*", # nocheck |
| # These aren't actual licenses :). |
| "as-is", |
| } |
| |
| # List of overlays that must use 'metapackage' license for virtual packages. |
| # Throw error for those, print a warning for others. |
| OVERLAYS_METAPACKAGES_MUST_USE_VIRTUAL = [ |
| "chromiumos-overlay", |
| "chromeos-overlay", |
| "chromeos-partner-overlay", |
| ] |
| |
| # The full names of packages which we want to generate license information for |
| # even though they have an empty installation size. |
| SIZE_EXEMPT_PACKAGES = [ |
| "net-print/fuji-xerox-printing-license", |
| "net-print/fujifilm-printing-license", |
| "net-print/konica-minolta-printing-license", |
| "net-print/nec-printing-license", |
| "net-print/xerox-printing-license", |
| ] |
| |
| # Find the directory of this script. |
| SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) |
| |
| # The template files we depend on for generating HTML. |
| TMPL = os.path.join(SCRIPT_DIR, "about_credits.tmpl") |
| ENTRY_TMPL = os.path.join(SCRIPT_DIR, "about_credits_entry.tmpl") |
| SHARED_LICENSE_TMPL = os.path.join( |
| SCRIPT_DIR, "about_credits_shared_license_entry.tmpl" |
| ) |
| |
| # dir_set constants for _GetLicenseDirectories. |
| _CUSTOM_DIRS = "custom" |
| _STOCK_DIRS = "stock" |
| _BOTH_DIRS = "both" |
| |
| # Banner shown when generating a placeholder credits. |
| _PLACEHOLDER_BANNER = """ |
| <div style="border: dashed; margin: 1em; padding: 1em;"> |
| <p style="font-weight: bold;">Placeholder Credits Page</p> |
| <p> |
| See <a href="https://dev.chromium.org/chromium-os/licensing/licensing-for-chromiumos-developers">Licensing for ChromiumOS Developers</a> and |
| <a href="https://dev.chromium.org/chromium-os/licensing/licensing-for-chromiumos-package-owners">Licensing for ChromiumOS Package Owners</a> |
| for more details on the credits system. |
| </p> |
| <p> |
| If you need to change styles, fonts, layout, etc of the |
| <a href="chrome://os-credits">chrome://os-credits</a> page, edit |
| <a href="https://chromium.googlesource.com/chromiumos/chromite/+/HEAD/licensing/about_credits.tmpl">chromite/licensing/about_credits.tmpl</a>. |
| The template is used to generate a device-dependent about_os_credits.html |
| when a CrOS image is built. |
| </p> |
| </div> |
| """ |
| |
| |
| def _convert_yaml_to_json(yaml_file: str, json_file: str) -> None: |
| """Migrate old YAML format to JSON. |
| |
| We have a lot of existing binpkgs that used yaml, so keep this for a while. |
| TODO(build): Make this fatal in Jan 2024. |
| |
| Args: |
| yaml_file: Path to the old existing yaml file. |
| json_file: Path to the json file to write. |
| """ |
| # The yaml files are legacy, so we don't normally load the module, and |
| # because it's not a common 3rd party install. |
| import yaml # pylint: disable=import-error |
| |
| class SaferLoader(yaml.SafeLoader): |
| """Augment the yaml.SafeLoader with unicode and tuple types.""" |
| |
| def construct_tuple(self, node): |
| return tuple(self.construct_sequence(node)) |
| |
| def construct_unicode(self, node): |
| return node.value |
| |
| SaferLoader.add_constructor( |
| "tag:yaml.org,2002:python/tuple", SaferLoader.construct_tuple |
| ) |
| SaferLoader.add_constructor( |
| "tag:yaml.org,2002:python/unicode", SaferLoader.construct_unicode |
| ) |
| |
| logging.debug("Migrating YAML (%s) to JSON (%s)", yaml_file, json_file) |
| |
| old_data = yaml.load(osutils.ReadFile(yaml_file), Loader=SaferLoader) |
| data = {} |
| for key, value in old_data: |
| if isinstance(value, set): |
| value = sorted(value) |
| data[key] = value |
| osutils.WriteFile(json_file, json.dumps(data), sudo=True) |
| |
| |
| # This is called directly by src/repohooks/pre-upload.py |
| def GetLicenseTypesFromEbuild( |
| ebuild_contents, overlay_path, buildroot=constants.SOURCE_ROOT |
| ): |
| """Returns a list of license types from an ebuild. |
| |
| This function does not always return the correct list, but it is |
| faster than using portageq for not having to access chroot. It is |
| intended to be used for tasks such as presubmission checks. |
| |
| Args: |
| ebuild_contents: contents of ebuild. |
| overlay_path: path of the overlay containing the ebuild |
| buildroot: base directory, usually constants.SOURCE_ROOT, useful for |
| tests |
| |
| Returns: |
| list of licenses read from ebuild. |
| |
| Raises: |
| ValueError: ebuild errors. |
| """ |
| ebuild_env_tmpl = """ |
| has() { [[ " ${*:2} " == *" $1 "* ]]; } |
| inherit() { |
| local overlay_list="%(overlay_list)s" |
| local eclass overlay f |
| for eclass; do |
| has ${eclass} ${_INHERITED_} && continue |
| _INHERITED_+=" ${eclass}" |
| for overlay in %(overlay_list)s; do |
| f="${overlay}/eclass/${eclass}.eclass" |
| if [[ -e ${f} ]]; then |
| source "${f}" |
| break |
| fi |
| done |
| done |
| } |
| %(ebuild)s""" |
| |
| repo_name = portage_util.GetOverlayName(overlay_path) |
| overlays = portage_util.FindOverlays( |
| constants.BOTH_OVERLAYS, repo_name, buildroot=buildroot |
| ) |
| tmpl_env = { |
| "ebuild": ebuild_contents, |
| "overlay_list": " ".join(overlays), |
| } |
| |
| with cros_build_lib.UnbufferedNamedTemporaryFile() as f: |
| osutils.WriteFile(f.name, ebuild_env_tmpl % tmpl_env) |
| env = osutils.SourceEnvironment( |
| f.name, allowlist=["LICENSE"], ifs=" ", multiline=True |
| ) |
| |
| if not env.get("LICENSE"): |
| raise ValueError("No LICENSE found in the ebuild.") |
| if re.search(r"[,;]", env["LICENSE"]): |
| raise ValueError( |
| "LICENSE field in the ebuild should be whitespace-limited." |
| ) |
| |
| return env["LICENSE"].split() |
| |
| |
| class PackageLicenseError(Exception): |
| """Thrown for failures while getting license information for a package. |
| |
| This will cause the processing to error in the end. |
| """ |
| |
| |
| class PackageCorrectnessError(Exception): |
| """Thrown if a package has build info incompatible with the license. |
| |
| For example, thrown when a package with LICENSE=metapackage installs files. |
| This will cause the processing to error in the end. |
| """ |
| |
| |
| class PackageInfo: |
| """Package specific information, mostly about licenses.""" |
| |
| def __init__(self, sysroot, fullnamerev) -> None: |
| """Package info initializer. |
| |
| Args: |
| sysroot: The board this package was built for. |
| fullnamerev: package name of the form 'x11-base/X.Org-1.9.3-r23' |
| |
| Raises: |
| ValueError if |fullnamerev| is not a valid package string. |
| """ |
| |
| self.sysroot = sysroot |
| |
| package = package_info.parse(fullnamerev) |
| self.category = package.category |
| self.name = package.package |
| self.version = package.version |
| self.revision = package.revision |
| self.fullname = package.cp |
| self.fullnamerev = package.cpvr |
| |
| if fullnamerev != self.fullnamerev: |
| # Parsed package doesn't match original, invalid package passed. |
| raise ValueError( |
| "portage couldn't find %s, missing version number?" |
| % fullnamerev |
| ) |
| |
| # |
| # These fields hold license information used to generate the credits |
| # page. |
| # |
| |
| # This contains licenses names for this package. |
| self.license_names = set() |
| |
| # Full Text of discovered license information. |
| self.license_text_scanned = [] |
| |
| self.homepages = [] |
| |
| # |
| # These fields show the results of processing. |
| # |
| |
| # After reading basic package information, we can mark the package as |
| # one to skip in licensing. |
| self.skip = False |
| |
| # Intelligently populate initial skip information. |
| self.LookForSkip() |
| |
| # Set to something by GetLicenses(). |
| self.tainted = None |
| |
| self.ebuild_path = None |
| |
| @property |
| def license_dump_path(self): |
| """e.g. /build/x86-alex/var/db/pkg/sys-apps/dtc-1.4.0/license.json. |
| |
| Only valid for packages that have already been emerged. |
| """ |
| return os.path.join( |
| self.sysroot, PER_PKG_LICENSE_DIR, self.fullnamerev, "license.json" |
| ) |
| |
| def _RunEbuildPhases(self, ebuild_path, phases): |
| """Run a list of ebuild phases on an ebuild. |
| |
| Args: |
| ebuild_path: exact path of the ebuild file. |
| phases: list of phases like ['clean', 'fetch'] or ['unpack']. |
| |
| Returns: |
| ebuild command output |
| """ |
| ebuild_cmd = cros_build_lib.GetSysrootToolPath(self.sysroot, "ebuild") |
| return cros_build_lib.run( |
| [ebuild_cmd, ebuild_path] + phases, stdout=True, encoding="utf-8" |
| ) |
| |
| def _GetOverrideLicense(self): |
| """Check COPYRIGHT_ATTRIBUTION_DIR for license w/ copyright attribution. |
| |
| For dev-util/bsdiff-4.3-r5, the code will look for |
| dev-util/bsdiff-4.3-r5 |
| dev-util/bsdiff-4.3 |
| dev-util/bsdiff |
| |
| It is ok to have more than one bsdiff license file, and an empty file |
| acts as a rubout (i.e. an empty dev-util/bsdiff-4.4 will shadow |
| dev-util/bsdiff and tell the licensing code to look in the package |
| source for a license instead of using dev-util/bsdiff as an override). |
| |
| Returns: |
| False (no license found) or a multiline license string. |
| """ |
| license_read = None |
| # dev-util/bsdiff-4.3-r5 -> bsdiff-4.3-r5 |
| filename = os.path.basename(self.fullnamerev) |
| license_path = os.path.join( |
| COPYRIGHT_ATTRIBUTION_DIR, os.path.dirname(self.fullnamerev) |
| ) |
| pv = package_info.parse(filename) |
| for filename in (pv.pvr, pv.pv, pv.package): |
| file_path = os.path.join(license_path, filename) |
| logging.debug( |
| "Looking for override copyright attribution license in %s", |
| file_path, |
| ) |
| if os.path.exists(file_path): |
| # Turn |
| # SRC_ROOT/CHROMIUMOS_OVERLAY_DIR/../dev-util/bsdiff |
| # into |
| # chromiumos-overlay/../dev-util/bsdiff |
| short_dir_path = os.path.join( |
| *file_path.rsplit(os.path.sep, 5)[1:] |
| ) |
| license_read = ( |
| "Copyright Attribution License %s:\n\n" % short_dir_path |
| ) |
| license_read += ReadUnknownEncodedFile( |
| file_path, "read copyright attribution license" |
| ) |
| break |
| |
| return license_read |
| |
| def _ExtractLicenses(self, src_dir, need_copyright_attribution) -> None: |
| """Scrounge for text licenses in the source of package we'll unpack. |
| |
| This is only called if we couldn't get usable licenses from the ebuild, |
| or one of them is BSD/MIT like which forces us to look for a file with |
| copyright attribution in the source code itself. |
| |
| First, we have a shortcut where we scan COPYRIGHT_ATTRIBUTION_DIR to see |
| if we find a license for this package. If so, we use that. |
| Typically, it'll be used if the unpacked source does not have the |
| license that we're required to display for copyright attribution (in |
| some cases it's plain absent, in other cases, it could be in a filename |
| we don't look for). |
| |
| Otherwise, we scan the unpacked source code for what looks like license |
| files as defined in LICENSE_BASENAMES_REGEX. |
| |
| Raises: |
| AssertionError: on runtime errors |
| PackageLicenseError: couldn't find copyright attribution file. |
| """ |
| license_override = self._GetOverrideLicense() |
| if license_override: |
| self.license_text_scanned = [license_override] |
| return |
| |
| if not src_dir: |
| self.ebuild_path = self._FindEbuildPath() |
| self._RunEbuildPhases(self.ebuild_path, ["clean", "fetch"]) |
| raw_output = self._RunEbuildPhases(self.ebuild_path, ["unpack"]) |
| output = raw_output.stdout.splitlines() |
| # pylint: disable=line-too-long |
| # |
| # Output is spammy, it looks like this: |
| # * gc-7.2d.tar.gz RMD160 SHA1 SHA256 size ;-) ... [ ok ] |
| # * checking gc-7.2d.tar.gz ;-) ... [ ok ] |
| # * Running stacked hooks for pre_pkg_setup |
| # * sysroot_build_bin_dir ... |
| # [ ok ] |
| # * Running stacked hooks for pre_src_unpack |
| # * python_multilib_setup ... |
| # [ ok ] |
| # >>> Unpacking source... |
| # >>> Unpacking gc-7.2d.tar.gz to /build/x86-alex/tmp/po/[...]ps-7.2d/work |
| # >>> Source unpacked in /build/x86-alex/tmp/portage/[...]ops-7.2d/work |
| # So we only keep the last 2 lines, the others we don't care about. |
| # |
| # pylint: enable=line-too-long |
| output = [ |
| line |
| for line in output |
| if line[0:3] == ">>>" and line != ">>> Unpacking source..." |
| ] |
| for line in output: |
| logging.info(line) |
| |
| tmpdir = portage_util.PortageqEnvvar( |
| "PORTAGE_TMPDIR", sysroot=self.sysroot |
| ) |
| # tmpdir gets something like /build/daisy/tmp/ |
| src_dir = os.path.join(tmpdir, "portage", self.fullnamerev, "work") |
| |
| if not os.path.exists(src_dir): |
| raise AssertionError( |
| "Unpack of %s didn't create %s. Version mismatch" |
| % (self.fullnamerev, src_dir) |
| ) |
| |
| # pylint: disable=line-too-long |
| # |
| # You may wonder how deep should we go? |
| # In case of packages with sub-packages, it could be deep. |
| # Let's just be safe and get everything we can find. |
| # In the case of libatomic_ops, it's actually required to look deep |
| # to find the MIT license: |
| # dev-libs/libatomic_ops-7.2d/work/gc-7.2/libatomic_ops/doc/LICENSING.txt |
| # |
| # pylint: enable=line-too-long |
| # Use "%P" to truncate results to look like this: swig-2.0.4/COPYRIGHT |
| args = ["find", src_dir, "-type", "f", "-printf", "%P\\n"] |
| result = cros_build_lib.run(args, stdout=True, encoding="utf-8") |
| license_files = FilterLicenseFileCandidates(result.stdout) |
| |
| if not license_files: |
| if need_copyright_attribution: |
| google_bsd_msg = "" |
| if need_copyright_attribution == {"BSD"}: |
| google_bsd_msg = """ |
| If this source code was entirely authored by Google employees, you can instead |
| just change the ebuild settings like so: |
| -LICENSE="BSD" |
| +LICENSE="BSD-Google"\ |
| """ |
| logging.error( |
| """\ |
| %s: unable to find usable license. |
| The ebuild says it uses at least %s which requires copyright attribution, |
| but there was no license file that this script could find in the package's |
| source distribution: |
| %s |
| |
| You will need to investigate that source directory to figure out which license |
| to assign. Once you've found it, copy the entire license file to: |
| %s |
| %s""", |
| self.fullnamerev, |
| need_copyright_attribution, |
| src_dir, |
| COPYRIGHT_ATTRIBUTION_DIR, |
| google_bsd_msg, |
| ) |
| raise PackageLicenseError( |
| "Missing copyright attribution for " |
| f"{need_copyright_attribution}" |
| ) |
| else: |
| # We can get called for a license like as-is where it's |
| # preferable to find a better one in the source, but not fatal |
| # if we didn't. |
| logging.info( |
| "Was not able to find a better license for %s " |
| "in %s to replace the more generic one from ebuild", |
| self.fullnamerev, |
| src_dir, |
| ) |
| |
| # Examples of multiple license matches: |
| # dev-lang/swig-2.0.4-r1: swig-2.0.4/COPYRIGHT swig-2.0.4/LICENSE |
| # dev-libs/glib-2.32.4-r1: glib-2.32.4/COPYING pkg-config-0.26/COPYING |
| # dev-libs/libnl-3.2.14: libnl-doc-3.2.14/COPYING libnl-3.2.14/COPYING |
| # dev-libs/libpcre-8.30-r2: pcre-8.30/LICENCE pcre-8.30/COPYING |
| # dev-libs/libusb-0.1.12-r6: libusb-0.1.12/COPYING libusb-0.1.12/LICENSE |
| # dev-libs/pyzy-0.1.0-r1: db/COPYING pyzy-0.1.0/COPYING |
| # net-misc/strongswan-5.0.2-r4: strongswan-5.0.2/COPYING |
| # strongswan-5.0.2/LICENSE |
| # sys-process/procps-3.2.8_p11: debian/copyright procps-3.2.8/COPYING |
| logging.info( |
| "License(s) for %s: %s", self.fullnamerev, " ".join(license_files) |
| ) |
| for license_file in sorted(license_files): |
| # Joy and pink ponies. Some license_files are encoded as latin1 |
| # while others are utf-8 and of course you can't know but only |
| # guess. |
| license_path = os.path.join(src_dir, license_file) |
| license_txt = ReadUnknownEncodedFile(license_path, "Adding License") |
| |
| self.license_text_scanned += [ |
| "Scanned Source License %s:\n\n%s" % (license_file, license_txt) |
| ] |
| |
| # We used to clean up here, but there have been many instances where |
| # looking at unpacked source to see where the licenses were, was useful |
| # so let's disable this for now |
| # self._RunEbuildPhases(['clean']) |
| |
| def LookForSkip(self): |
| """Look for a reason to skip over this package. |
| |
| Sets self.skip to True if a reason was found. |
| |
| Returns: |
| True if a reason was found. |
| """ |
| if self.category in SKIPPED_CATEGORIES: |
| logging.info( |
| "%s in SKIPPED_CATEGORIES, skip package", self.fullname |
| ) |
| self.skip = True |
| |
| # TODO(dgarrett): There are additional reasons that should be handled |
| # here. |
| |
| return self.skip |
| |
| def _FindEbuildPath(self): |
| """Discover the path to a package's associated ebuild. |
| |
| This method is not valid during the emerge hook process. |
| |
| Returns: |
| full path file name of the ebuild file for this package. |
| |
| Raises: |
| AssertionError if it can't be discovered for some reason. |
| """ |
| equery_cmd = cros_build_lib.GetSysrootToolPath(self.sysroot, "equery") |
| args = [equery_cmd, "-q", "-C", "which", self.fullnamerev] |
| try: |
| path = cros_build_lib.run( |
| args, print_cmd=True, encoding="utf-8", stdout=True |
| ).stdout.strip() |
| except cros_build_lib.RunCommandError: |
| path = None |
| |
| # Path can be false because of an exception, or a command result. |
| if not path: |
| raise AssertionError( |
| "_FindEbuildPath for %s failed.\n" |
| "Is your tree clean? Try a rebuild?" % self.fullnamerev |
| ) |
| |
| logging.debug("%s -> %s", " ".join(args), path) |
| |
| if not os.access(path, os.F_OK): |
| raise AssertionError("Can't access %s" % (path,)) |
| |
| return path |
| |
| def GetLicenses(self, build_info_dir, src_dir) -> None: |
| """Populate the license related fields. |
| |
| Fields populated: |
| license_names, license_text_scanned, homepages, skip, tainted |
| |
| Some packages have static license mappings applied to them that get |
| retrieved from the ebuild. |
| |
| For others, we figure out whether the package source should be scanned |
| to add licenses found there. |
| |
| Args: |
| build_info_dir: Path to the build_info for the ebuild. This can be |
| from the working directory during the emerge hook, or in the |
| portage pkg db. |
| src_dir: Directory to the expanded source code for this package. |
| If None, the source will be expanded, if needed (slow). |
| |
| Raises: |
| AssertionError: on runtime errors |
| PackageLicenseError: couldn't find license in ebuild and source. |
| """ |
| # If the total size installed is zero, we installed no content to |
| # license. |
| if _BuildInfo(build_info_dir, "SIZE").strip() == "0": |
| # Allow for license generation for the exempt empty packages. |
| if self.fullname not in SIZE_EXEMPT_PACKAGES: |
| logging.debug("Build directory is empty") |
| self.skip = True |
| return |
| |
| self.homepages = _BuildInfo(build_info_dir, "HOMEPAGE").split() |
| licenses = ebuild_license.parse(_BuildInfo(build_info_dir, "LICENSE")) |
| |
| # The ebuild license field can look like: |
| # LICENSE="GPL-3 LGPL-3 Apache-2.0" (this means AND, as in all 3) |
| # for third_party/portage-stable/app-admin/rsyslog/rsyslog-5.8.11.ebuild |
| # LICENSE="|| ( LGPL-2.1 MPL-1.1 )" |
| # for third_party/portage-stable/x11-libs/cairo/cairo-1.8.8.ebuild |
| # |
| # In order to save time needlessly unpacking packages and looking or a |
| # cleartext license (which is really a crapshoot), if we have a license |
| # like BSD that requires looking for copyright attribution, but we can |
| # chose another license like GPL, we do that. |
| def license_picker(choices): |
| for choice in choices: |
| if choice not in COPYRIGHT_ATTRIBUTION_LICENSES: |
| logging.info( |
| "Picking license '%s' from %s", choice, choices |
| ) |
| return choice |
| return choices[0] |
| |
| ebuild_license_names = licenses.reduce(anyof_reduce=license_picker) |
| |
| # Is this tainted? |
| self.tainted = TAINTED in ebuild_license_names |
| if self.tainted: |
| logging.warning("Package %s is tainted", self.fullnamerev) |
| |
| # If this ebuild only uses skipped licenses, skip it. |
| if ebuild_license_names and all( |
| l in SKIPPED_LICENSES for l in ebuild_license_names |
| ): |
| self.skip = True |
| |
| if self.skip: |
| return |
| |
| logging.info( |
| "Read licenses for %s: %s", |
| self.fullnamerev, |
| ",".join(ebuild_license_names), |
| ) |
| |
| if not self.skip and not ebuild_license_names: |
| logging.error( |
| "%s: no license found in ebuild. FIXME!", self.fullnamerev |
| ) |
| # In a bind, you could comment this out. I'm making the output fail |
| # to get your attention since this error really should be fixed, but |
| # if you comment out the next line, the script will try to find a |
| # license inside the source. |
| raise PackageLicenseError() |
| |
| need_copyright_attribution = set() |
| scan_source_for_licenses = False |
| |
| for license_name in ebuild_license_names: |
| # Licenses like BSD or MIT can't be used as is because they do not |
| # contain copyright self. They have to be replaced by copyright file |
| # given in the source code. |
| if license_name in COPYRIGHT_ATTRIBUTION_LICENSES: |
| logging.info( |
| "%s: can't use %s, will scan source code for copyright", |
| self.fullnamerev, |
| license_name, |
| ) |
| need_copyright_attribution.add(license_name) |
| scan_source_for_licenses = True |
| else: |
| self.license_names.add(license_name) |
| # We can't display just 2+ because it only contains text that |
| # says to read v2 or v3. |
| if license_name == "GPL-2+": |
| self.license_names.add("GPL-2") |
| if license_name == "LGPL-2+": |
| self.license_names.add("LGPL-2") |
| |
| if license_name in LOOK_IN_SOURCE_LICENSES: |
| logging.info( |
| "%s: Got %s, will try to find better license in source...", |
| self.fullnamerev, |
| license_name, |
| ) |
| scan_source_for_licenses = True |
| |
| if self.license_names: |
| logging.info( |
| "%s: using stock|cust license(s) %s", |
| self.fullnamerev, |
| ",".join(self.license_names), |
| ) |
| |
| # If the license(s) could not be found, or one requires copyright |
| # attribution, dig in the source code for license files: |
| # For instance: |
| # Read licenses from ebuild for net-dialup/ppp-2.4.5-r3: BSD,GPL-2 |
| # We need get the substitution file for BSD and add it to GPL. |
| if scan_source_for_licenses: |
| self._ExtractLicenses(src_dir, need_copyright_attribution) |
| |
| # This shouldn't run, but leaving as basic smoke check. |
| if not self.license_names and not self.license_text_scanned: |
| raise AssertionError( |
| "Didn't find usable licenses for %s" % self.fullnamerev |
| ) |
| |
| def SaveLicenseDump(self, save_file) -> None: |
| """Save PackageInfo contents for loading later. |
| |
| This is used to cache license results between the emerge hook phase and |
| credits page generation. |
| |
| Args: |
| save_file: File to save the state into. |
| """ |
| logging.debug("Saving license to %s", save_file) |
| dump = {} |
| for key, value in list(self.__dict__.items()): |
| if isinstance(value, set): |
| value = sorted(value) |
| dump[key] = value |
| osutils.WriteFile(save_file, json.dumps(dump), makedirs=True) |
| |
| def AssertCorrectness(self, build_info_dir, ebuild_path) -> None: |
| """AssertCorrectness runs various correctness checks on the package. |
| |
| Args: |
| build_info_dir: Path to the build_info for the ebuild. This can be |
| from the working directory during the emerge hook, or in the |
| portage pkg db. |
| ebuild_path: Path to the ebuild. Unknown and therefore None during |
| the emerge hook. |
| |
| Raises: |
| PackageCorrectnessError: if one of the checks fails. |
| """ |
| self._AssertMetapackageNoContent(build_info_dir) |
| if ebuild_path is not None: |
| self._AssertVirtualIsMetapackage(build_info_dir, ebuild_path) |
| |
| def _AssertMetapackageNoContent(self, build_info_dir) -> None: |
| """Ensures metapackages do not install files. |
| |
| Args: |
| build_info_dir: Path to the build_info for the ebuild. This can be |
| from the working directory during the emerge hook, or in the |
| portage pkg db. |
| |
| Raises: |
| PackageCorrectnessError: if metapackage installs files. |
| """ |
| if _BuildInfo(build_info_dir, "LICENSE") == "metapackage": |
| content = _BuildInfo(build_info_dir, "CONTENTS") |
| if content: |
| content_list = ", ".join( |
| x.split()[1] for x in content.splitlines() |
| ) |
| raise PackageCorrectnessError( |
| "Metapackage %s installs files: %s." |
| % (self.fullnamerev, content_list) |
| ) |
| |
| def _AssertVirtualIsMetapackage(self, build_info_dir, ebuild_path) -> None: |
| """Ensures that virtual pkgs are metapackages. |
| |
| Args: |
| ebuild_path: Path to the ebuild. |
| build_info_dir: Path to the build_info for the ebuild. This can be |
| from the working directory during the emerge hook, or in the |
| portage pkg db. |
| |
| Raises: |
| PackageCorrectnessError: if virtual pkg does not use metapackage |
| license. |
| """ |
| category = ebuild_path.split("/")[-3] |
| overlay = ebuild_path.split("/")[-4] |
| license_name = _BuildInfo(build_info_dir, "LICENSE") |
| if category == "virtual" and license_name != "metapackage": |
| err_msg = ( |
| f"Virtual package {ebuild_path} must use LICENSE=" |
| f'"metapackage". Got: {self.license_names}.' |
| ) |
| if overlay in OVERLAYS_METAPACKAGES_MUST_USE_VIRTUAL: |
| raise PackageCorrectnessError(err_msg) |
| else: |
| logging.warning(err_msg) |
| |
| |
| def _GetLicenseDirectories( |
| board: Optional[str] = None, |
| sysroot: Optional[str] = None, |
| dir_set: str = _BOTH_DIRS, |
| buildroot: Path = constants.SOURCE_ROOT, |
| ) -> List[str]: |
| """Get the "licenses" directories for all matching overlays. |
| |
| With a |board| argument, allows searching without a compiled sysroot. |
| |
| Args: |
| board: Which board to use to search a hierarchy. Does not require the |
| board be setup or compiled yet. |
| sysroot: A setup board sysroot to query. |
| dir_set: Whether to fetch stock, custom, or both sets of directories. |
| See the _(STOCK|CUSTOM|BOTH)_DIRS constants. |
| buildroot: The root chromiumos path. |
| |
| Returns: |
| list - all matching "licenses" directories |
| """ |
| stock = [os.path.join(buildroot, d) for d in STOCK_LICENSE_DIRS] |
| if board is not None: |
| custom_paths = portage_util.FindOverlays( |
| constants.BOTH_OVERLAYS, board, buildroot |
| ) |
| elif sysroot is not None and sysroot != "/": |
| portdir_overlay = ( |
| sysroot_lib.Sysroot(sysroot).GetStandardField( |
| sysroot_lib.STANDARD_FIELD_PORTDIR_OVERLAY |
| ) |
| or "" |
| ) |
| custom_paths = portdir_overlay.split() if portdir_overlay else [] |
| else: |
| custom_paths = [ |
| os.path.join(constants.SOURCE_ROOT, x) for x in SDK_OVERLAY_DIRS |
| ] |
| custom = [os.path.join(d, "licenses") for d in custom_paths] |
| |
| if dir_set == _STOCK_DIRS: |
| return stock |
| elif dir_set == _CUSTOM_DIRS: |
| return custom |
| else: |
| return stock + custom |
| |
| |
| def _CheckForKnownBadLicenses(cpf, licenses) -> None: |
| """Make sure all the |licenses| are ones we allow. |
| |
| We have a bunch of licenses we don't want people to use, but some packages |
| were using them by the time we noticed. Still allow those existing ones, |
| but prevent new users from showing up. |
| """ |
| bad_licenses = set() |
| for pattern in BAD_LICENSE_PATTERNS: |
| bad_licenses |= {x for x in licenses if fnmatch.fnmatch(x, pattern)} |
| |
| # We allow ghostscript for public builds to use AGPL but nothing else. |
| if cpf.startswith("app-text/ghostscript-gpl"): |
| bad_licenses -= {"AGPL-3"} |
| |
| if bad_licenses: |
| raise PackageLicenseError(f"Licenses {bad_licenses} are not allowed") |
| |
| # TODO(crbug.com/401332): Remove this entirely. |
| # We allow a few packages for now so new packages will stop showing up. |
| if "Proprietary-Binary" in licenses: |
| # Note: DO NOT ADD ANY MORE PACKAGES HERE. |
| LEGACY_PKGS = ( |
| "chromeos-base/infineon-firmware", |
| "sys-boot/coreboot-private-files-", |
| "sys-boot/nhlt-blobs", |
| "sys-boot/mma-blobs", |
| ) |
| if not any(cpf.startswith(x) for x in LEGACY_PKGS): |
| raise PackageLicenseError( |
| f"{cpf}: Proprietary-Binary is not a valid license." |
| ) |
| |
| # TODO(b/187789754): Remove this entirely. |
| # We allow a few packages for now so new packages will stop showing up. |
| if "Google-TOS" in licenses: |
| # Note: DO NOT ADD ANY MORE PACKAGES HERE. |
| LEGACY_PKGS = { |
| "chromeos-base/chromeos-board-default-apps-atlas", |
| "chromeos-base/chromeos-board-default-apps-cave", |
| "chromeos-base/chromeos-board-default-apps-chell", |
| "chromeos-base/chromeos-board-default-apps-nocturne", |
| "chromeos-base/chromeos-board-default-apps-setzer", |
| "chromeos-base/chromeos-board-default-apps-snappy", |
| "chromeos-base/chromeos-board-default-apps-soraka", |
| "chromeos-base/chromeos-board-default-apps-terra", |
| "chromeos-base/chromeos-bsp-trogdor-private", |
| "chromeos-base/chromeos-chrome", |
| "chromeos-base/chromeos-default-apps", |
| "chromeos-base/chromeos-firmware-anx7688", |
| "chromeos-base/fibocom-firmware", |
| "chromeos-base/google-sans-fonts", |
| "chromeos-base/houdini", |
| "chromeos-base/houdini-pi", |
| "chromeos-base/houdini-qt", |
| "chromeos-base/intel-hdcp", |
| "chromeos-base/monotype-fonts", |
| "chromeos-base/qc7180-modem-firmware", |
| "chromeos-base/sc7280-modem-firmware", |
| "chromeos-base/modem-logger-sc", |
| "chromeos-base/modem-fw-dlc", |
| "dev-embedded/meta-embedded-toolkit", |
| "media-libs/a630-fw", |
| "media-libs/apl-hotword-support", |
| "media-libs/arc-img-ddk", |
| "media-libs/arc-mali-drivers", |
| "media-libs/arc-mali-drivers-bifrost", |
| "media-libs/cros-camera-hal-qti", |
| "media-libs/cros-camera-hal-qti-bin", |
| "media-libs/glk-hotword-support", |
| "media-libs/go2001-fw", |
| "media-libs/img-ddk", |
| "media-libs/img-ddk-bin", |
| "media-libs/kbl-hotword-support", |
| "media-libs/kbl-rt5514-hotword-support", |
| "media-libs/mali-drivers", |
| "media-libs/mali-drivers-bifrost", |
| "media-libs/mali-drivers-bifrost-bin", |
| "media-libs/mali-drivers-bin", |
| "media-libs/qti-7c-camera-bins", |
| "media-libs/rk3399-hotword-support", |
| "media-libs/skl-hotword-support", |
| "net-print/brother_mlaser", |
| "sys-apps/loonix-hydrogen", |
| "sys-boot/chromeos-firmware-ps8751", |
| "sys-boot/chromeos-vendor-strings-wilco", |
| "sys-boot/coreboot-private-files-chipset-picasso", |
| "sys-boot/coreboot-private-files-chipset-stnyridge", |
| "sys-boot/coreboot-private-files-drallion", |
| "sys-boot/coreboot-private-files-grunt", |
| "sys-boot/coreboot-private-files-hatch", |
| "sys-boot/amd-picasso-fsp", |
| "sys-boot/intel-cflfsp", |
| "sys-boot/intel-glkfsp", |
| "sys-boot/intel-iclfsp", |
| "sys-boot/mma-blobs", |
| "sys-boot/nhlt-blobs", |
| "sys-boot/rk3399-hdcp-fw", |
| "sys-boot/qclib", |
| "sys-boot/qtiseclib", |
| "sys-firmware/analogix-anx3429-firmware", |
| "sys-firmware/displaylink-firmware", |
| "sys-firmware/falcon-firmware", |
| "sys-firmware/huddly-firmware", |
| "sys-firmware/iq-firmware", |
| "sys-firmware/sis-firmware", |
| } |
| if not any(cpf.startswith(x) for x in LEGACY_PKGS): |
| raise PackageLicenseError( |
| f"{cpf}: Google-TOS is not a valid license." |
| ) |
| |
| |
| class Licensing: |
| """Do the actual work of extracting licensing info and outputting html.""" |
| |
| def __init__( |
| self, |
| sysroot, |
| package_fullnames, |
| gen_licenses, |
| placeholder: bool = False, |
| ) -> None: |
| self.sysroot = sysroot |
| # List of stock and custom licenses referenced in ebuilds. Used to |
| # print a report. Dict value says which packages use that license. |
| self.licenses = {} |
| |
| # Licenses are supposed to be generated at package build time and be |
| # ready for us, but in case they're not, they can be generated. |
| self.gen_licenses = gen_licenses |
| |
| # Generate placeholder values instead of inspecting real data. |
| self.placeholder = placeholder |
| |
| self.entry_template = None |
| |
| # We need to have a dict for the list of packages objects, index by |
| # package fullnamerev, so that when we scan our licenses at the end, and |
| # find out some shared licenses are only used by one package, we can |
| # access that package object by name, and add the license directly in |
| # that object. |
| self.packages = {} |
| self._package_fullnames = package_fullnames |
| |
| # Set by ProcessPackageLicenses(). |
| self.tainted_pkgs = [] |
| |
| @property |
| def sorted_licenses(self): |
| return sorted(self.licenses, key=lambda x: x.lower()) |
| |
| def _LoadLicenseDump(self, pkg) -> None: |
| save_file = pkg.license_dump_path |
| logging.debug("Getting license from %s for %s", save_file, pkg.name) |
| with open(save_file, "rb") as fp: |
| dump = json.load(fp) |
| for key, value in dump.items(): |
| if isinstance(pkg.__dict__[key], set): |
| value = set(value) |
| pkg.__dict__[key] = value |
| |
| def LicensedPackages(self, license_name): |
| """Return list of packages using a given license.""" |
| return self.licenses[license_name] |
| |
| def LoadPackageInfo(self) -> None: |
| """Populate basic package info for all packages from their ebuild.""" |
| for package_name in self._package_fullnames: |
| pkg = PackageInfo(self.sysroot, package_name) |
| self.packages[package_name] = pkg |
| |
| def ProcessPackageLicenses(self) -> None: |
| """Iterate through all packages provided and gather their licenses. |
| |
| GetLicenses will scrape licenses from the code and/or gather stock |
| license names. We gather the list of stock and custom ones for later |
| processing. |
| |
| Do not call this after adding virtual packages with AddExtraPkg. |
| """ |
| # TODO(b/236161656): Fix. |
| # pylint: disable-next=consider-using-dict-items |
| for package_name in self.packages: |
| pkg = self.packages[package_name] |
| |
| if pkg.skip: |
| logging.debug("Package %s is in skip list", package_name) |
| continue |
| |
| if self.placeholder: |
| pkg.license_names = ["Placeholder-License"] |
| pkg.license_text_scanned = ["Custom placeholder license text"] |
| pkg.homepages = ["https://dev.chromium.org/"] |
| continue |
| |
| # Do inplace/ondemand migration. |
| if not os.path.exists(pkg.license_dump_path): |
| yaml_file = pkg.license_dump_path[0:-4] + "yaml" |
| if os.path.exists(yaml_file): |
| _convert_yaml_to_json(yaml_file, pkg.license_dump_path) |
| |
| # Other skipped packages get dumped with incomplete info and the |
| # skip flag |
| if not os.path.exists(pkg.license_dump_path): |
| if not self.gen_licenses: |
| logging.error( |
| "Run licenses.py with --generate-licenses to try to " |
| "generate the missing licenses." |
| ) |
| raise PackageLicenseError( |
| f"{pkg.fullnamerev}: license is missing" |
| ) |
| |
| logging.warning( |
| ">>> License for %s is missing, creating now <<<", |
| package_name, |
| ) |
| build_info_path = os.path.join( |
| self.sysroot, PER_PKG_LICENSE_DIR, pkg.fullnamerev |
| ) |
| pkg.GetLicenses(build_info_path, None) |
| pkg.AssertCorrectness(build_info_path, self.ebuild_path) |
| |
| # We dump packages where licensing failed too. |
| pkg.SaveLicenseDump(pkg.license_dump_path) |
| |
| # Load the pre-cached version, if the in-memory version is |
| # incomplete. |
| if not pkg.license_names: |
| logging.debug("loading dump for %s", pkg.fullnamerev) |
| self._LoadLicenseDump(pkg) |
| |
| # Store all tainted packages to print at the top of generated |
| # license. If any package is tainted, the whole thing is tainted. |
| if pkg.tainted: |
| self.tainted_pkgs.append(package_name) |
| |
| _CheckForKnownBadLicenses(pkg.fullnamerev, pkg.license_names) |
| |
| def AddExtraPkg( |
| self, fullnamerev, homepages, license_names, license_texts |
| ) -> None: |
| """Allow adding pre-created virtual packages. |
| |
| GetLicenses will not work on them, so add them after having run |
| ProcessPackages. |
| |
| Args: |
| fullnamerev: package name of the form x11-base/X.Org-1.9.3-r23 |
| homepages: list of url strings. |
| license_names: list of license name strings. |
| license_texts: custom license text to use, mostly for attribution. |
| """ |
| pkg = PackageInfo(self.sysroot, fullnamerev) |
| pkg.homepages = homepages |
| pkg.license_names = license_names |
| pkg.license_text_scanned = license_texts |
| self.packages[fullnamerev] = pkg |
| |
| _CheckForKnownBadLicenses(fullnamerev, pkg.license_names) |
| |
| # Called directly by src/repohooks/pre-upload.py |
| @staticmethod |
| def FindLicenseType( |
| license_name, |
| board=None, |
| sysroot=None, |
| overlay_path=None, |
| buildroot=constants.SOURCE_ROOT, |
| placeholder: bool = False, |
| ): |
| """Says if a license is stock Gentoo, custom, tainted, or doesn't exist. |
| |
| Will check the old, static locations by default, but supplying either |
| the overlay directory or sysroot allows searching in the overlay |
| hierarchy. Ignores the overlay_path if sysroot is provided. |
| |
| Args: |
| license_name: The license name |
| board: Which board to use to search a hierarchy. Does not require |
| the board be setup or compiled yet. |
| sysroot: A setup board sysroot to query. |
| overlay_path: Which overlay directory to use as the search base |
| buildroot: source root |
| placeholder: Whether to generate a stub placeholder page. |
| |
| Returns: |
| str - license type |
| |
| Raises: |
| AssertionError when the license couldn't be found |
| """ |
| if license_name == TAINTED: |
| return TAINTED |
| |
| if placeholder: |
| return "Placeholder Stock" |
| |
| # Check the stock licenses first since those may appear in the generated |
| # list of overlay directories for a board |
| stock = _GetLicenseDirectories(dir_set=_STOCK_DIRS, buildroot=buildroot) |
| for directory in stock: |
| path = os.path.join(directory, license_name) |
| if os.path.exists(path): |
| return "Gentoo Package Stock" |
| |
| # Not stock, find and check relevant custom directories |
| if board is None and overlay_path is not None: |
| board = portage_util.GetOverlayName(overlay_path) |
| |
| # Check the custom licenses |
| custom = _GetLicenseDirectories( |
| board=board, |
| sysroot=sysroot, |
| dir_set=_CUSTOM_DIRS, |
| buildroot=buildroot, |
| ) |
| for directory in custom: |
| path = os.path.join(directory, license_name) |
| if os.path.exists(path): |
| return "Custom" |
| |
| if license_name in SKIPPED_LICENSES: |
| return "Custom" |
| |
| raise AssertionError( |
| """ |
| license %s could not be found in %s |
| If the license in the ebuild is correct, |
| a) a stock license should be added to portage-stable/licenses : |
| running `cros_portage_upgrade` inside of the chroot should clone this repo |
| to /tmp/portage/: |
| https://chromium.googlesource.com/chromiumos/overlays/portage/+/gentoo |
| find the new licenses under licenses, and add them to portage-stable/licenses |
| |
| b) if it's a non gentoo package with a custom license, you can copy that license |
| to third_party/chromiumos-overlay/licenses/ |
| |
| Try re-running the script with -p cat/package-ver --generate |
| after fixing the license.""" |
| % (license_name, "\n".join(set(stock + custom))) |
| ) |
| |
| @staticmethod |
| def ReadSharedLicense( |
| license_name, |
| board=None, |
| sysroot=None, |
| buildroot=constants.SOURCE_ROOT, |
| placeholder: bool = False, |
| ): |
| """Read and return stock or cust license file specified in an ebuild.""" |
| if placeholder: |
| return "Placeholder license text" |
| |
| directories = _GetLicenseDirectories( |
| board=board, |
| sysroot=sysroot, |
| dir_set=_BOTH_DIRS, |
| buildroot=buildroot, |
| ) |
| license_path = None |
| for directory in directories: |
| path = os.path.join(directory, license_name) |
| if os.path.exists(path): |
| license_path = path |
| break |
| |
| if license_path: |
| return ReadUnknownEncodedFile(license_path, "read license") |
| else: |
| raise AssertionError( |
| "license %s could not be found in %s" |
| % (license_name, "\n".join(directories)) |
| ) |
| |
| @staticmethod |
| def EvaluateTemplate(template, env): |
| """Expand |template| with vars like {{foo}} using |env| expansions.""" |
| # TODO switch to stock python templates. |
| for key, val in env.items(): |
| template = template.replace("{{%s}}" % key, val) |
| return template |
| |
| def _GeneratePackageLicenseHTML(self, pkg, license_text): |
| """Concatenate all licenses related to a pkg in HTML format. |
| |
| This means a combination of ebuild shared licenses and licenses read |
| from the pkg source tree, if any. |
| |
| Args: |
| pkg: PackageInfo object |
| license_text: the license in plain text. |
| |
| Returns: |
| The license for a file->package in HTML format. |
| |
| Raises: |
| AssertionError: on runtime errors |
| """ |
| license_pointers = [] |
| # sln: shared license name. |
| for sln in pkg.license_names: |
| # Says whether it's a stock gentoo or custom license. |
| try: |
| license_type = self.FindLicenseType( |
| sln, sysroot=self.sysroot, placeholder=self.placeholder |
| ) |
| except Exception as e: |
| logging.error( |
| "Failed to find the type of %s license, used by %s " |
| "package. If this license is not used anymore, it may be " |
| "still cached as a binpkg—run emerge on the package to " |
| "rebuild. See more info at " |
| "https://dev.chromium.org/chromium-os/licensing", |
| sln, |
| pkg.fullnamerev, |
| ) |
| cros_build_lib.Die(e) |
| license_pointers.append( |
| "<li><a href='#%s'>%s License %s</a></li>" |
| % (sln, license_type, sln) |
| ) |
| |
| # This should get caught earlier, but one extra check. |
| if not license_text + license_pointers: |
| raise AssertionError( |
| "Ended up with no license_text for %s" % pkg.fullnamerev |
| ) |
| |
| env = { |
| "comments": TAINTED_COMMENT_TAG if pkg.tainted else "", |
| "name": pkg.name, |
| "namerev": "%s-%s" % (pkg.name, pkg.version), |
| "url": html.escape(pkg.homepages[0]) if pkg.homepages else "", |
| "licenses_txt": html.escape("\n".join(license_text)) or "", |
| "licenses_ptr": "\n".join(license_pointers) or "", |
| } |
| return self.EvaluateTemplate(self.entry_template, env) |
| |
| def _GeneratePackageLicenseText(self, pkg): |
| """Concatenate all licenses related to a pkg. |
| |
| This means a combination of ebuild shared licenses and licenses read |
| from the pkg source tree, if any. |
| |
| Args: |
| pkg: PackageInfo object |
| """ |
| license_text = [] |
| for license_text_scanned in pkg.license_text_scanned: |
| license_text.append(license_text_scanned) |
| license_text.append("%s\n" % ("-=" * 40)) |
| |
| return license_text |
| |
| def GenerateLicenseText(self): |
| """Generate the license text for all packages.""" |
| license_txts = {} |
| # Keep track of which licenses are used by which packages. |
| for pkg in self.packages.values(): |
| if pkg.tainted: |
| license_txts[pkg] = TAINTED |
| continue |
| if pkg.skip: |
| continue |
| for sln in pkg.license_names: |
| self.licenses.setdefault(sln, []).append(pkg.fullnamerev) |
| |
| # Find licenses only used once, and roll them in the package that uses |
| # them. We use list() because licenses is modified in the loop, so we |
| # can't use an iterator. |
| for sln in list(self.licenses): |
| if len(self.licenses[sln]) == 1: |
| pkg_fullnamerev = self.licenses[sln][0] |
| logging.info( |
| "Collapsing shared license %s into single use license " |
| "(only used by %s)", |
| sln, |
| pkg_fullnamerev, |
| ) |
| license_type = self.FindLicenseType( |
| sln, sysroot=self.sysroot, placeholder=self.placeholder |
| ) |
| license_txt = self.ReadSharedLicense( |
| sln, sysroot=self.sysroot, placeholder=self.placeholder |
| ) |
| single_license = "%s License %s:\n\n%s" % ( |
| license_type, |
| sln, |
| license_txt, |
| ) |
| pkg = self.packages[pkg_fullnamerev] |
| pkg.license_text_scanned.append(single_license) |
| pkg.license_names.remove(sln) |
| del self.licenses[sln] |
| |
| for pkg in self.packages.values(): |
| if pkg.skip: |
| logging.debug("Skipping empty package %s", pkg.fullnamerev) |
| continue |
| license_txts[pkg] = self._GeneratePackageLicenseText(pkg) |
| |
| return license_txts |
| |
| def GenerateHTMLLicenseOutput( |
| self, |
| output_file, |
| output_template=TMPL, |
| entry_template=ENTRY_TMPL, |
| license_template=SHARED_LICENSE_TMPL, |
| os_version: Optional[str] = None, |
| milestone_version: Optional[str] = None, |
| compress_output=False, |
| ) -> None: |
| """Generate the combined html license file. |
| |
| Args: |
| output_file: resulting HTML license output. |
| output_template: template for the entire HTML file. |
| entry_template: template for per package entries. |
| license_template: template for shared license entries. |
| os_version: OS version. |
| milestone_version: Milestone version. |
| compress_output: whether to compress based on suffix of output_file. |
| """ |
| self.entry_template = ReadUnknownEncodedFile(entry_template) |
| license_txts = self.GenerateLicenseText() |
| sorted_license_txt = [] |
| for pkg in sorted( |
| license_txts.keys(), |
| key=lambda x: (x.name.lower(), x.version, x.revision), |
| ): |
| sorted_license_txt += [ |
| self._GeneratePackageLicenseHTML(pkg, license_txts[pkg]) |
| ] |
| |
| # Now generate the bottom of the page that will contain all the shared |
| # licenses and a list of who is pointing to them. |
| license_template = ReadUnknownEncodedFile(license_template) |
| |
| licenses_txt = [] |
| for license_name in self.sorted_licenses: |
| env = { |
| "license_name": license_name, |
| "license": html.escape( |
| self.ReadSharedLicense( |
| license_name, |
| sysroot=self.sysroot, |
| placeholder=self.placeholder, |
| ) |
| ), |
| "license_type": self.FindLicenseType( |
| license_name, |
| sysroot=self.sysroot, |
| placeholder=self.placeholder, |
| ), |
| "license_packages": " ".join( |
| self.LicensedPackages(license_name) |
| ), |
| } |
| licenses_txt += [self.EvaluateTemplate(license_template, env)] |
| |
| if self.placeholder: |
| if not os_version: |
| os_version = "1000.10.0" |
| if not milestone_version: |
| milestone_version = "100" |
| reciprocal_txt = "" |
| if os_version and milestone_version: |
| env = { |
| "chromeos-manifest-link": gs_urls_util.GsUrlToHttp( |
| "gs://chromeos-manifest-versions/buildspecs/" |
| f"{milestone_version}/{os_version}.xml" |
| ), |
| "chromiumos-manifest-link": gs_urls_util.GsUrlToHttp( |
| "gs://chromiumos-manifest-versions/buildspecs/" |
| f"{milestone_version}/{os_version}.xml" |
| ), |
| "os-version": os_version, |
| "milestone-version": milestone_version, |
| } |
| reciprocal_txt = self.EvaluateTemplate( |
| osutils.ReadFile( |
| os.path.join(SCRIPT_DIR, "about_credits_reciprocal.tmpl") |
| ), |
| env, |
| ) |
| |
| file_template = ReadUnknownEncodedFile(output_template) |
| tainted_warning = "" |
| if self.tainted_pkgs: |
| tained_pkg_lis = "\n".join( |
| f" <li>{x}</li>" for x in self.tainted_pkgs |
| ) |
| tainted_warning = ( |
| TAINTED_COMMENT_TAG |
| + "\n" |
| + "<h1>Image is TAINTED due to the following " |
| + 'packages:</h1>\n<ul style="font-size:large">\n' |
| + tained_pkg_lis |
| + "\n</ul>\n" |
| ) |
| for tainted_pkg in self.tainted_pkgs: |
| logging.warning("Package %s is tainted", tainted_pkg) |
| logging.warning( |
| "Image is tainted. See licensing docs to fix this: " |
| "https://dev.chromium.org/chromium-os/licensing" |
| ) |
| env = { |
| "tainted_warning_if_any": tainted_warning, |
| "entries": "\n".join(sorted_license_txt), |
| "licenses": "\n".join(licenses_txt), |
| "placeholder": _PLACEHOLDER_BANNER if self.placeholder else "", |
| "reciprocal-license-statement": reciprocal_txt, |
| } |
| contents = self.EvaluateTemplate(file_template, env).encode("utf-8") |
| if not compress_output: |
| # Just write it. |
| osutils.WriteFile(output_file, contents, mode="wb") |
| else: |
| # Write to a temp file, then compress it to the final destination, |
| # using the file extension specified to determine compression type. |
| with cros_build_lib.UnbufferedNamedTemporaryFile() as f: |
| osutils.WriteFile(f.name, contents, mode="wb") |
| cros_build_lib.CompressFile(f.name, output_file) |
| |
| |
| def ListInstalledPackages(sysroot, all_packages=False): |
| """Return a list of all packages installed for a particular board.""" |
| |
| # If all_packages is set to True, all packages visible in the build |
| # chroot are used to generate the licensing file. This is not what you want |
| # for a release license file, but it's a way to run licensing checks against |
| # all packages. |
| # If it's set to False, it will only generate a licensing file that contains |
| # packages used for a release build (as determined by the dependencies for |
| # virtual/target-os). |
| |
| if all_packages: |
| # The following returns all packages that were part of the build tree |
| # (many get built or used during the build, but do not get shipped). |
| # Note that it also contains packages that are in the build as |
| # defined by cros build-packages but not part of the image we ship. |
| equery_cmd = cros_build_lib.GetSysrootToolPath(sysroot, "equery") |
| args = [equery_cmd, "list", "*"] |
| packages = cros_build_lib.run( |
| args, encoding="utf-8", stdout=True |
| ).stdout.splitlines() |
| else: |
| # The following returns all packages that were part of the build tree |
| # (many get built or used during the build, but do not get shipped). |
| # Note that it also contains packages that are in the build as |
| # defined by cros build-packages but not part of the image we ship. |
| emerge_cmd = cros_build_lib.GetSysrootToolPath(sysroot, "emerge") |
| args = [ |
| emerge_cmd, |
| "--with-bdeps=y", |
| "--usepkgonly", |
| "--emptytree", |
| "--pretend", |
| "--color=n", |
| "virtual/target-os", |
| ] |
| emerge = cros_build_lib.run( |
| args, encoding="utf-8", stdout=True |
| ).stdout.splitlines() |
| # Another option which we've decided not to use, is bdeps=n. This |
| # outputs just the packages we ship, but not packages that were used to |
| # build them, including a package like flex which generates a .a that is |
| # included and shipped in ChromeOS. |
| # We've decided to credit build packages, even if we're not legally |
| # required to (it's always nice to do), and that way we get corner case |
| # packages like flex. This is why we use bdep=y and not bdep=n. |
| |
| packages = [] |
| bad_packages = [] |
| # [binary R ] x11-libs/libva-1.1.1 to /build/x86-alex/ |
| pkg_rgx = re.compile(r"\[[^]]+R[^]]+\] (.+) to /build/.*") |
| # pylint: disable=line-too-long |
| # If we match something else without the 'R' like |
| # [binary U ] chromeos-base/some-package-13.0.0.133-r1 [12.0.0.77-r1] |
| # this is bad, and we should die on this. |
| # pylint: enable=line-too-long |
| pkg_rgx2 = re.compile(r"(\[[^]]+\] .+) to /build/.*") |
| for line in emerge: |
| match = pkg_rgx.search(line) |
| match2 = pkg_rgx2.search(line) |
| if match: |
| packages.append(match.group(1)) |
| elif match2: |
| bad_packages.append(match2.group(1)) |
| |
| if bad_packages: |
| raise AssertionError( |
| "Package incorrectly installed, try reinstalling", |
| "\n%s" % "\n".join(bad_packages), |
| ) |
| |
| return packages |
| |
| |
| def FilterLicenseFileCandidates(stdout: str) -> List[str]: |
| """Returns the subset of files in `stdout` that look like licenses.""" |
| license_files: List[str] = [] |
| basename_include_pattern = re.compile( |
| "|".join(f"({x})" for x in LICENSE_BASENAMES_REGEX), re.IGNORECASE |
| ) |
| path_exclude_pattern = re.compile( |
| "|".join(f"({x})" for x in LICENSE_PATHS_EXCLUDE_REGEX), re.IGNORECASE |
| ) |
| for path in stdout.splitlines(): |
| # When we scan a source tree managed by git, this can contain license |
| # files that are not part of the source. Exclude those. |
| # (e.g. .git/refs/heads/licensing) |
| if ".git/" in path: |
| continue |
| basename = os.path.basename(path) |
| if basename_include_pattern.search(basename): |
| if path_exclude_pattern.search(path): |
| logging.info("Ignoring %s (matches exclude regex).", path) |
| else: |
| license_files.append(path) |
| return license_files |
| |
| |
| def ReadUnknownEncodedFile(file_path, logging_text=None): |
| """Read a file of unknown encoding (UTF-8 or latin) by trying in sequence. |
| |
| Args: |
| file_path: what to read. |
| logging_text: what to display for logging depending on file read. |
| |
| Returns: |
| File content, possibly converted from latin1 to UTF-8. |
| """ |
| try: |
| with codecs.open(file_path, encoding="utf-8") as c: |
| file_txt = c.read() |
| if logging_text: |
| logging.info("%s %s (UTF-8)", logging_text, file_path) |
| except UnicodeDecodeError: |
| with codecs.open(file_path, encoding="latin1") as c: |
| file_txt = c.read() |
| if logging_text: |
| logging.info("%s %s (latin1)", logging_text, file_path) |
| |
| # Remove characters that are not XML 1.0 legal. |
| # XML 1.0 acceptable character range: |
| # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | \ |
| # [#x10000-#x10FFFF] |
| |
| # Strip out common/OK values silently. |
| silent_chars_re = re.compile("[\x0c]") |
| file_txt = silent_chars_re.sub("", file_txt) |
| |
| illegal_chars_re = re.compile( |
| "[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]" |
| ) |
| |
| if illegal_chars_re.findall(file_txt): |
| logging.warning("Found illegal XML characters, stripping them out.") |
| file_txt = illegal_chars_re.sub("", file_txt) |
| |
| return file_txt |
| |
| |
| def _BuildInfo(build_info_path, filename): |
| """Fetch contents of a file from portage build_info directory. |
| |
| Portage maintains a build_info directory that exists both during the process |
| of emerging an ebuild, and (in a different location) after the ebuild has |
| been emerged. |
| |
| Various useful data files exist there like: |
| 'CATEGORY', 'PF', 'SIZE', 'HOMEPAGE', 'LICENSE' |
| |
| Args: |
| build_info_path: Path to the build_info directory to read from. |
| filename: Name of the file to read. |
| |
| Returns: |
| Contents of the file as a string, or "". |
| """ |
| filename = os.path.join(build_info_path, filename) |
| |
| # Buildinfo properties we read are in US-ASCII, not Unicode. |
| try: |
| bi = osutils.ReadFile(filename).rstrip() |
| # Some properties like HOMEPAGE may be absent. |
| except IOError: |
| bi = "" |
| return bi |
| |
| |
| def HookPackageProcess( |
| pkg_build_path: str, sysroot: Optional[str] = "/" |
| ) -> None: |
| """Different entry point to populate a packageinfo. |
| |
| This is called instead of LoadPackageInfo when called by a package build. |
| |
| Args: |
| pkg_build_path: unpacked being built by emerge. |
| sysroot: The sysroot we're building for. |
| """ |
| build_info_dir = os.path.join(pkg_build_path, "build-info") |
| if not os.path.isdir(build_info_dir): |
| raise ValueError( |
| '%s is not a valid build path (missing "build-info/")' |
| % (pkg_build_path,) |
| ) |
| |
| fullnamerev = "%s/%s" % ( |
| _BuildInfo(build_info_dir, "CATEGORY"), |
| _BuildInfo(build_info_dir, "PF"), |
| ) |
| logging.debug( |
| "Computed package name %s from %s", fullnamerev, pkg_build_path |
| ) |
| |
| pkg = PackageInfo(None, fullnamerev) |
| |
| src_dir = os.path.join(pkg_build_path, "work") |
| pkg.GetLicenses(build_info_dir, src_dir) |
| |
| pkg.AssertCorrectness(build_info_dir, None) |
| # Make sure the licenses are valid at build time even if we don't load them. |
| _CheckForKnownBadLicenses(fullnamerev, pkg.license_names) |
| for license_name in pkg.license_names: |
| Licensing.FindLicenseType(license_name, sysroot=sysroot) |
| |
| pkg.SaveLicenseDump(os.path.join(build_info_dir, "license.json")) |
