| #!/usr/bin/python |
| # |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| |
| """Script that attempts to generate an HTML file containing license |
| information and homepage links for all installed packages. |
| |
| WARNING: this script in its current form is not finished or considered |
| production quality/code style compliant. This is an intermediate checkin |
| to allow for incremental cleanups and improvements that will make it |
| production quality. |
| |
| Usage: |
| For this script to work, you must have built the architecture |
| this is being run against, _after_ you've last run repo sync. |
| Otherwise, it will query newer source code and then fail to work on packages |
| that are out of date in your build. |
| |
| Recommended build: |
| cros_sdk |
| export board=x86-alex |
| sudo rm -rf /build/$board |
| cd ~/trunk/src/scripts |
| # TODO: setup_board should generate static list of packages that can be used |
| # by this script. |
| ./setup_board --board=$board |
| # If you wonder why we need to build |
| # chromeos just to run emerge -p -v chromeos-base/chromeos on it, we don't. |
| # However, later we run ebuild unpack, and this will apply patches and run |
| # configure. Configure will fail due to aclocal macros missing in |
| # /build/x86-alex/usr/share/aclocal (those are generated during build). |
| # This will take about 10mn on a Z620. |
| ./build_packages --board=$board --nowithautotest --nowithtest --nowithdev \ |
| --nowithfactory |
| cd ~/trunk/chromite/license-generation |
| %(prog)s [--debug] $board out.html 2>&1 | tee output.sav |
| |
| For debugging during development, you can get a faster run of just one package |
| with: |
| %(prog)s --testpkg "dev-libs/libatomic_ops-7.2d" $board out.html |
| |
| The output file is meant to update |
| http://src.chromium.org/viewvc/chrome/trunk/src/chrome/browser/resources/ + |
| chromeos/about_os_credits.html?view=log |
| (gclient config svn://svn.chromium.org/chrome/trunk/src) |
| For an example CL, see https://codereview.chromium.org/13496002/ |
| |
| It is recommended that you use a fancy differ like 'meld' to review license |
| diffs. GNU diff will show too much irrelevant noise and not resync properly. |
| |
| UPDATE: gcl will probably fail now, because the file is too big. Before it |
| gets moved somewhere else, you should just use svn diff and svn commit. |
| |
| Recommended way to diff the html, go outside of the cros_sdk chroot: |
| grep -E -A5 '(class="title|class="homepage|Provided Stock|Source license)' \ |
| out.html > /tmp/new |
| grep -E -A5 '(class="title|class="homepage|Provided Stock|Source license)' \ |
| out-sav-new3.html > /tmp/old |
| meld /tmp/old /tmp/new (or your favourite fancy diff program) |
| |
| If you don't get this in before the freeze window, it'll need to be merged into |
| the branch being released, which is done by adding a Merge-Requested label to |
| Iteration-xx in the tracking bug. |
| Once it's been updated to "Merge-Approved" by a TPM, please merge into the |
| required release branch. You can ask karen@ for merge approve help. |
| Example: http://crbug.com/221281 |
| |
| Usage: %(prog)s [opts] <board> <output> |
| |
| """ |
| |
| import cgi |
| import codecs |
| import logging |
| import os |
| import re |
| import sys |
| |
| from chromite.lib import commandline |
| from chromite.lib import cros_build_lib |
| import portage |
| |
| |
| debug = False |
| |
| STOCK_LICENSE_DIRS = [ |
| os.path.expanduser('~/trunk/src/third_party/portage/licenses'), |
| os.path.expanduser('~/trunk/src/third_party/portage-stable/licenses'), |
| ] |
| |
| # There are licenses for custom software we got and isn't part of |
| # upstream gentoo. |
| CUSTOM_LICENSE_DIRS = [ |
| os.path.expanduser('~/trunk/src/third_party/chromiumos-overlay/licenses'), |
| # All licenses there should go away, they're a stopgap. |
| os.path.abspath(os.path.join(os.path.dirname(__file__), 'licenses')), |
| ] |
| |
| # Virtual packages don't need to have a license and often don't, so we skip them |
| # chromeos-base contains google platform packages that are covered by the |
| # general license at top of tree, so we skip those too. |
| SKIPPED_CATEGORIES = [ |
| 'chromeos-base', # TODO: this shouldn't be excluded ? |
| 'virtual', |
| ] |
| |
| SKIPPED_PACKAGES = [ |
| # Fix these packages by adding a real license in the code. |
| # You should not skip packages just because the license scraping doesn't |
| # work. Stick those special cases into PACKAGE_LICENSES. |
| # Packages should only be here because they are sub/split packages already |
| # covered by the license of the main package. |
| |
| # These are Chrome-OS-specific packages, copyright BSD-Google |
| 'sys-kernel/chromeos-kernel', # already manually credit Linux |
| |
| # These have been split across several packages, so we skip listing the |
| # individual components (and just list the main package instead). |
| 'app-editors/vim-core', |
| 'x11-apps/mesa-progs', |
| |
| # Portage metapackage. |
| 'x11-base/xorg-drivers', |
| |
| # These are covered by app-i18n/ibus-mozc (BSD, copyright Google). |
| 'app-i18n/ibus-mozc-chewing', |
| 'app-i18n/ibus-mozc-hangul', |
| 'app-i18n/ibus-mozc-pinyin', |
| |
| # Those have License: Proprietary in the ebuild. |
| 'app-i18n/GoogleChineseInput-cangjie', |
| 'app-i18n/GoogleChineseInput-pinyin', |
| 'app-i18n/GoogleChineseInput-wubi', |
| 'app-i18n/GoogleChineseInput-zhuyin', |
| 'app-i18n/GoogleKoreanInput', |
| |
| # These are all X.org sub-packages; shouldn't be any need to list them |
| # individually. |
| 'media-fonts/encodings', |
| 'x11-apps/iceauth', |
| 'x11-apps/intel-gpu-tools', |
| 'x11-apps/mkfontdir', |
| 'x11-apps/rgb', |
| 'x11-apps/setxkbmap', |
| 'x11-apps/xauth', |
| 'x11-apps/xcursorgen', |
| 'x11-apps/xdpyinfo', |
| 'x11-apps/xdriinfo', |
| 'x11-apps/xev', |
| 'x11-apps/xgamma', |
| 'x11-apps/xhost', |
| 'x11-apps/xinit', |
| 'x11-apps/xinput', |
| 'x11-apps/xkbcomp', |
| 'x11-apps/xlsatoms', |
| 'x11-apps/xlsclients', |
| 'x11-apps/xmodmap', |
| 'x11-apps/xprop', |
| 'x11-apps/xrandr', |
| 'x11-apps/xrdb', |
| 'x11-apps/xset', |
| 'x11-apps/xset-mini', |
| 'x11-apps/xwininfo', |
| 'x11-base/xorg-server', |
| 'x11-drivers/xf86-input-evdev', |
| 'x11-drivers/xf86-input-keyboard', |
| 'x11-drivers/xf86-input-mouse', |
| 'x11-drivers/xf86-input-synaptics', |
| 'x11-drivers/xf86-video-intel', |
| 'x11-drivers/xf86-video-vesa', |
| 'x11-drivers/xf86-video-vmware', |
| 'x11-libs/libICE', |
| 'x11-libs/libSM', |
| 'x11-libs/libX11', |
| 'x11-libs/libXScrnSaver', |
| 'x11-libs/libXau', |
| 'x11-libs/libXcomposite', |
| 'x11-libs/libXcursor', |
| 'x11-libs/libXdamage', |
| 'x11-libs/libXdmcp', |
| 'x11-libs/libXext', |
| 'x11-libs/libXfixes', |
| 'x11-libs/libXfont', |
| 'x11-libs/libXfontcache', |
| 'x11-libs/libXft', |
| 'x11-libs/libXi', |
| 'x11-libs/libXinerama', |
| 'x11-libs/libXmu', |
| 'x11-libs/libXp', |
| 'x11-libs/libXrandr', |
| 'x11-libs/libXrender', |
| 'x11-libs/libXres', |
| 'x11-libs/libXt', |
| 'x11-libs/libXtst', |
| 'x11-libs/libXv', |
| 'x11-libs/libXvMC', |
| 'x11-libs/libXxf86vm', |
| 'x11-libs/libdrm', |
| 'x11-libs/libfontenc', |
| 'x11-libs/libpciaccess', |
| 'x11-libs/libxkbfile', |
| 'x11-libs/libxkbui', |
| 'x11-libs/pixman', |
| 'x11-libs/xtrans', |
| 'x11-misc/util-macros', |
| 'x11-misc/xbitmaps', |
| 'x11-proto/bigreqsproto', |
| 'x11-proto/compositeproto', |
| 'x11-proto/damageproto', |
| 'x11-proto/dri2proto', |
| 'x11-proto/fixesproto', |
| 'x11-proto/fontcacheproto', |
| 'x11-proto/fontsproto', |
| 'x11-proto/inputproto', |
| 'x11-proto/kbproto', |
| 'x11-proto/printproto', |
| 'x11-proto/randrproto', |
| 'x11-proto/recordproto', |
| 'x11-proto/renderproto', |
| 'x11-proto/resourceproto', |
| 'x11-proto/scrnsaverproto', |
| 'x11-proto/trapproto', |
| 'x11-proto/videoproto', |
| 'x11-proto/xcmiscproto', |
| 'x11-proto/xextproto', |
| 'x11-proto/xf86bigfontproto', |
| 'x11-proto/xf86dgaproto', |
| 'x11-proto/xf86driproto', |
| 'x11-proto/xf86rushproto', |
| 'x11-proto/xf86vidmodeproto', |
| 'x11-proto/xineramaproto', |
| 'x11-proto/xproto', |
| ] |
| |
| # TODO(merlin): replace matching with regex matching to simplify this |
| # Matching is done in lowercase, you MUST give lowercase names. |
| LICENSE_FILENAMES = [ |
| 'copying', |
| 'copyright', |
| 'ipa_font_license_agreement_v1.0.txt', # used by ja-ipafonts |
| 'licence', # used by openssh |
| 'license', |
| 'license.txt', # used by hdparm, NumPy, glew |
| 'licensing.txt', # used by libatomic_ops |
| ] |
| |
| # These are _temporary_ license mappings for packages that do not have a valid |
| # shared/custom license, or LICENSE file we can use. |
| # Once this script runs earlier (during the package build process), it will |
| # block new source without a LICENSE file if the ebuild contains a license |
| # that requires copyright assignment (BSD and friends). |
| # At that point, new packages will get fixed to include LICENSE instead of |
| # adding workaround mappings like those below. |
| # We should also fix the packages listed below so that the hardcoded |
| # mappings can be obsoleted (i.e. FIXME for this entire list). |
| PACKAGE_LICENSES = { |
| # One off licenses. Should we check in a custom LICENSE file in upstream? |
| 'sys-libs/ncurses': ['ncurses'], |
| |
| # BSD and MIT license authorship mapping. |
| # Ideally we should have a custom LICENSE file in the upstream source. |
| # TODO: BSD-2: bsdiff is missing a license file, add one upstream. |
| 'dev-util/bsdiff': ['BSD-bsdiff'], |
| # TODO: libevent is missing a license file, add one upstream. |
| 'dev-libs/libevent': ['BSD-libevent'], |
| # TODO: dhcpcd is missing a license file, (c) in README. Add one upstream. |
| 'net-misc/dhcpcd': ['BSD-dhcpcd'], |
| # TODO: iputils is missing a license file, add one upstream. |
| 'net-misc/iputils': ['BSD-iputils'], |
| # TODO: c-ares is missing a license file, add one upstream. |
| 'net-dns/c-ares': ['MIT-MIT'], |
| |
| # TODO: We should just check in a LICENSE file in all of these: |
| 'app-i18n/input-tools': ['BSD-Google'], |
| 'app-i18n/nacl-mozc': ['BSD-Google'], |
| 'app-i18n/ibus-mozc': ['BSD-Google'], |
| 'media-plugins/o3d': ['BSD-Google'], |
| 'dev-python/unittest2': ['BSD-Google'], |
| |
| # Fix ebuild multi license definitions when they define licenses that do |
| # not apply to us because we don't use the resulting binaries. |
| |
| # Mesa ebuild says MIT and omits LGPL-3 and SGI-B-2.0 mentioned in the |
| # docs directory? Either way, I had to create a text license file like so: |
| # mesa-9.1-r9/work/Mesa-9.1/docs$ lynx --dump license.html -nolist > license |
| 'media-libs/mesa': ['MIT-Mesa', 'LGPL-3', 'SGI-B-2.0'], |
| |
| # TODO: Ebuild seems to wrongfully say BSD + public-domain. |
| # I scanned the unpacked source with licensecheck and didn't find any BSD. |
| # FIXME: Do a second review and fix upstream gentoo package |
| 'sys-libs/timezone-data': ['public-domain'], |
| |
| # Ebuild only says 'LGPL-2.1', but source disagrees. I'll include 'as-is' |
| # to force reading files from the source (which states some parts are as-is) |
| # FIXME? Update ebuild license to match xz-4.999.9beta/COPYING? |
| 'app-arch/xz-utils': ['public-domain', 'as-is', 'LGPL-2.1', 'GPL-2'], |
| |
| # These packages are not in Alex, check and remove later (might be used in |
| # other platforms). |
| # 'media-libs/freeimage': ['GPL-2'], |
| # 'sys-libs/talloc': ['LGPL-3'], # ebuild incorrectly says GPL-3 |
| # 'media-libs/jpeg': ['jpeg'], |
| # 'app-editors/gentoo-editor': ['MIT-gentoo-editor'], |
| # |
| # 'media-fonts/font-util': ['font-util'], # COPYING file from git repo |
| # 'net-wireless/iwl1000-ucode': ['Intel-iwl1000'], |
| # 'sys-process/vixie-cron': ['vixie-cron'], |
| } |
| |
| # Any license listed list here found in the ebuild will make the code look for |
| # license files inside the package source code in order to get copyright |
| # attribution from them. |
| COPYRIGHT_ATTRIBUTION_LICENSES = [ |
| 'BSD', # requires distribution of copyright notice |
| 'BSD-2', # so does BSD-2 http://opensource.org/licenses/BSD-2-Clause |
| 'BSD-3', # and BSD-3? http://opensource.org/licenses/BSD-3-Clause |
| 'BSD-4', # and 4? |
| 'BSD-with-attribution', |
| 'MIT', |
| 'MIT-with-advertising', |
| 'Old-MIT', |
| ] |
| |
| # The following licenses are not invalid or to show as a less helpful stock |
| # license, but it's better to look in the source code for a more specific |
| # license if there is one, but not an error if no better one is found. |
| # Note that you don't want to set just anything here since any license here |
| # will be included once in stock form and a second time in custom form if |
| # found (there is no good way to know that a license we found on disk is the |
| # better version of the stock version, so we show both). |
| LOOK_IN_SOURCE_LICENSES = [ |
| 'as-is', # The stock license is very vague, source always has more details. |
| 'PSF-2', # The custom license in python is more complete than the template. |
| |
| # As far as I know, we have no requirement to do copyright attribution for |
| # these licenses, but the license included in the code has slightly better |
| # information than the stock Gentoo one (including copyright attribution). |
| 'BZIP2', # Single use license, do copyright attribution. |
| 'OFL', # Almost single use license, do copyright attribution. |
| 'OFL-1.1', # Almost single use license, do copyright attribution. |
| 'UoI-NCSA', # Only used by NSCA, might as well show their custom copyright. |
| ] |
| |
| PACKAGE_HOMEPAGES = { |
| # Example: |
| # 'x11-proto/glproto': ['http://www.x.org/'], |
| } |
| |
| # These are tokens found in LICENSE= in an ebuild that aren't licenses we |
| # can actually read from disk. |
| # You should not use this to blacklist real licenses. |
| LICENCES_IGNORE = [ |
| ')', # Ignore OR tokens from LICENSE="|| ( LGPL-2.1 MPL-1.1 )" |
| '(', |
| '||', |
| ] |
| |
| TMPL = 'about_credits.tmpl' |
| ENTRY_TMPL = 'about_credits_entry.tmpl' |
| SHARED_LICENSE_TMPL = 'about_credits_shared_license_entry.tmpl' |
| |
| |
| class PackageLicenseError(Exception): |
| """Thrown if something fails while getting license information for a package. |
| This will cause the processing to error in the end. |
| """ |
| |
| |
| class PackageSkipped(Exception): |
| """Non error to exclude packages from license processing.""" |
| |
| |
| class PackageInfo(object): |
| """Package info containers, mostly for storing licenses.""" |
| |
| def __init__(self, board): |
| |
| self.board = board |
| self.revision = None |
| |
| # Array of scanned license texts. |
| self.license_text_scanned = [] |
| |
| self.category = None |
| self.name = None |
| self.version = None |
| |
| # Looks something like this |
| # /mnt/host/source/src/ |
| # third_party/portage-stable/net-misc/rsync/rsync-3.0.8.ebuild |
| self.ebuild_path = None |
| # dirname of ebuild_path. |
| self.ebuild_dir = None |
| |
| # Array of license names retrieved from ebuild or override in this code. |
| self.ebuild_license_names = [] |
| self.description = None |
| self.homepages = [] |
| # This contains licenses names we can read from Gentoo or custom licenses. |
| # These are supposed to be shared licenses (i.e. licenses referenced by |
| # more then one package), but after all processing, we may find out that |
| # some are only used once and they get taken out of the shared pool and |
| # pasted directly in the sole package that was using them (see |
| # GenerateHTMLLicenseOutput). |
| self.license_names = set() |
| |
| # We set this if the ebuild has a BSD/MIT like license that requires |
| # scanning for a LICENSE file in the source code, or a static mapping |
| # in PACKAGE_LICENSES. Not finding one once this is set, is fatal. |
| self.need_copyright_attribution = False |
| # This flag just says we'd like to include licenses from the source, but |
| # not finding any is not fatal. |
| self.scan_source_for_licenses = False |
| |
| @property |
| def fullnamerev(self): |
| s = '%s-%s' % (self.fullname, self.version) |
| if self.revision: |
| s += '-r%s' % self.revision |
| return s |
| |
| @property |
| def fullname(self): |
| return '%s/%s' % (self.category, self.name) |
| |
| def _RunEbuildPhases(self, phases): |
| """Run a list of ebuild phases on an ebuild. |
| |
| Args: |
| phases: list of phases like ['clean', 'fetch'] or ['unpack']. |
| |
| Returns: |
| ebuild command output |
| """ |
| |
| return cros_build_lib.RunCommand( |
| ['ebuild-%s' % self.board, self.ebuild_path] + phases, print_cmd=debug, |
| redirect_stdout=True) |
| |
| def _ExtractLicenses(self): |
| """Scrounge for text licenses in the source of package we'll unpack. |
| |
| This is only called if we couldn't get usable licenses from the ebuild, |
| or one of them is BSD/MIT like which forces us to look for a file with |
| copyright attribution in the source code itself. |
| It'll scan the unpacked source code for what looks like license files |
| as defined in LICENSE_FILENAMES. |
| |
| Raises: |
| PackageLicenseError: couldn't find copyright attribution file. |
| """ |
| |
| self._RunEbuildPhases(['clean', 'fetch']) |
| output = self._RunEbuildPhases(['unpack']).output.splitlines() |
| # Output is spammy, it looks like this: |
| # * gc-7.2d.tar.gz RMD160 SHA1 SHA256 size ;-) ... [ ok ] |
| # * checking gc-7.2d.tar.gz ;-) ... [ ok ] |
| # * Running stacked hooks for pre_pkg_setup |
| # * sysroot_build_bin_dir ... |
| # [ ok ] |
| # * Running stacked hooks for pre_src_unpack |
| # * python_multilib_setup ... |
| # [ ok ] |
| # >>> Unpacking source... |
| # >>> Unpacking gc-7.2d.tar.gz to /build/x86-alex/tmp/po/[...]tops-7.2d/work |
| # >>> Source unpacked in /build/x86-alex/tmp/portage/[...]ops-7.2d/work |
| # So we only keep the last 2 lines, the others we don't care about. |
| output = [line for line in output if line[0:3] == ">>>" and |
| line != ">>> Unpacking source..."] |
| for line in output: |
| logging.info(line) |
| |
| args = ['portageq-%s' % self.board, 'envvar', 'PORTAGE_TMPDIR'] |
| result = cros_build_lib.RunCommand(args, print_cmd=debug, |
| redirect_stdout=True) |
| tmpdir = result.output.splitlines()[0] |
| # tmpdir gets something like /build/daisy/tmp/ |
| workdir = os.path.join(tmpdir, 'portage', self.fullnamerev, 'work') |
| |
| # You may wonder how deep should we go? |
| # In case of packages with sub-packages, it could be deep. |
| # Let's just be safe and get everything we can find. |
| # In the case of libatomic_ops, it's actually required to look deep |
| # to find the MIT license: |
| # dev-libs/libatomic_ops-7.2d/work/gc-7.2/libatomic_ops/doc/LICENSING.txt |
| args = ['find', workdir, '-type', 'f'] |
| result = cros_build_lib.RunCommand(args, print_cmd=debug, |
| redirect_stdout=True).output.splitlines() |
| # Truncate results to look like this: swig-2.0.4/COPYRIGHT |
| files = [x[len(workdir):].lstrip('/') for x in result] |
| license_files = [] |
| for name in files: |
| if os.path.basename(name).lower() in LICENSE_FILENAMES: |
| license_files.append(name) |
| |
| if not license_files: |
| if self.need_copyright_attribution: |
| logging.error("%s used license with copyright attribution, but " |
| "couldn't find license file in %s", |
| self.fullnamerev, workdir) |
| raise PackageLicenseError() |
| else: |
| # We can get called for a license like as-is where it's preferable |
| # to find a better one in the source, but not fatal if we didn't. |
| logging.info("Was not able to find a better license for %s " |
| "in %s to replace the more generic one from ebuild", |
| self.fullnamerev, workdir) |
| |
| # Examples of multiple license matches: |
| # dev-lang/swig-2.0.4-r1: swig-2.0.4/COPYRIGHT swig-2.0.4/LICENSE |
| # dev-libs/glib-2.32.4-r1: glib-2.32.4/COPYING pkg-config-0.26/COPYING |
| # dev-libs/libnl-3.2.14: libnl-doc-3.2.14/COPYING libnl-3.2.14/COPYING |
| # dev-libs/libpcre-8.30-r2: pcre-8.30/LICENCE pcre-8.30/COPYING |
| # dev-libs/libusb-0.1.12-r6: libusb-0.1.12/COPYING libusb-0.1.12/LICENSE |
| # dev-libs/pyzy-0.1.0-r1: db/COPYING pyzy-0.1.0/COPYING |
| # net-misc/strongswan-5.0.2-r4: strongswan-5.0.2/COPYING |
| # strongswan-5.0.2/LICENSE |
| # sys-process/procps-3.2.8_p11: debian/copyright procps-3.2.8/COPYING |
| logging.info('License(s) for %s: %s', self.fullnamerev, |
| ' '.join(license_files)) |
| for license_file in sorted(license_files): |
| # Joy and pink ponies. Some license_files are encoded as latin1 while |
| # others are utf-8 and of course you can't know but only guess. |
| license_path = os.path.join(workdir, license_file) |
| license_txt = ReadUnknownEncodedFile(license_path, "Adding License") |
| |
| self.license_text_scanned += [ |
| "Scanned Source License %s:\n\n%s" % (license_file, license_txt)] |
| |
| # We used to clean up here, but there have been many instances where |
| # looking at unpacked source to see where the licenses were, was useful |
| # so let's disable this for now |
| # self._RunEbuildPhases(['clean']) |
| |
| def _SetPackageInfo(self, fullnamewithrev): |
| """Populate PackageInfo with package license, homepage and description. |
| |
| Some packages have static license mappings applied to them. |
| |
| self.ebuild_license_names will not be filled if the package is skipped |
| or if there was an issue getting data from the ebuild. |
| self.license_names will only get the licenses that we can paste |
| as shared licenses. |
| scan_source_for_licenses will be set if we should unpack the source to look |
| for licenses |
| if need_copyright_attribution is also set, not finding a license in the |
| source is fatal (PackageLicenseError will get raised). |
| |
| Args: |
| fullnamewithrev: e.g. dev-libs/libatomic_ops-7.2d |
| |
| Raises: |
| AssertionError: on runtime errors |
| PackageSkipped: if in skip list |
| """ |
| |
| try: |
| self.category, self.name, self.version, self.revision = \ |
| portage.versions.catpkgsplit(fullnamewithrev) |
| except TypeError: |
| raise AssertionError("portage couldn't find %s, missing version number?" % |
| fullnamewithrev) |
| |
| if self.revision is not None: |
| self.revision = str(self.revision).lstrip('r') |
| if self.revision == '0': |
| self.revision = None |
| |
| if self.category in SKIPPED_CATEGORIES: |
| raise PackageSkipped("%s in SKIPPED_CATEGORIES, skip package" % |
| self.fullname) |
| |
| if self.fullname in SKIPPED_PACKAGES: |
| raise PackageSkipped("%s in SKIPPED_PACKAGES, skip package" % |
| self.fullname) |
| |
| args = ['equery-%s' % self.board, 'which', self.fullname] |
| path = cros_build_lib.RunCommand(args, print_cmd=debug, |
| redirect_stdout=True).output.strip() |
| logging.debug("%s -> %s", " ".join(args), path) |
| if not path: |
| raise AssertionError('GetEbuildPath for %s failed.\n' |
| 'Is your tree clean? Delete /build/%s and rebuild' % |
| (self.name, self.board)) |
| |
| if not os.access(path, os.F_OK): |
| raise AssertionError("Can't access %s", path) |
| |
| self.ebuild_dir = os.path.dirname(path) |
| self.ebuild_path = path |
| |
| args = ['portageq-%s' % self.board, 'metadata', '/build/%s' % self.board, |
| 'ebuild', self.fullnamerev, 'HOMEPAGE', 'LICENSE', 'DESCRIPTION'] |
| lines = cros_build_lib.RunCommand(args, print_cmd=debug, |
| redirect_stdout=True).output.splitlines() |
| # Runs: |
| # portageq metadata /build/x86-alex ebuild net-misc/wget-1.12-r2 \ |
| # HOMEPAGE LICENSE DESCRIPTION |
| # Returns: |
| # http://www.gnu.org/software/wget/ |
| # GPL-3 |
| # Network utility to retrieve files from the WWW |
| |
| (self.homepages, self.ebuild_license_names, self.description) = ( |
| lines[0].split(), lines[1].split(), lines[2:]) |
| |
| if self.fullname in PACKAGE_HOMEPAGES: |
| self.homepages = PACKAGE_HOMEPAGES[self.fullname] |
| |
| # Packages with missing licenses or licenses that need mapping (like |
| # BSD/MIT) are hardcoded here: |
| if self.fullname in PACKAGE_LICENSES: |
| self.ebuild_license_names = PACKAGE_LICENSES[self.fullname] |
| logging.info("Static license mapping for %s: %s", self.fullnamerev, |
| ",".join(self.ebuild_license_names)) |
| else: |
| logging.info("Read licenses from ebuild for %s: %s", self.fullnamerev, |
| ",".join(self.ebuild_license_names)) |
| |
| def GetLicenses(self, fullnamewithrev): |
| """Get licenses from the ebuild field and the unpacked source code. |
| |
| After populating the package info and licenses, this figures |
| out whether the package source should be scanned to add licenses found |
| there. |
| |
| Args: |
| fullnamewithrev: e.g. dev-libs/libatomic_ops-7.2d |
| |
| Raises: |
| AssertionError: on runtime errors |
| PackageLicenseError: couldn't find license in ebuild. |
| """ |
| |
| # First populate the package basic information |
| self._SetPackageInfo(fullnamewithrev) |
| |
| # The ebuild license field can look like: |
| # LICENSE="GPL-3 LGPL-3 Apache-2.0" (this means AND, as in all 3) |
| # for third_party/portage-stable/app-admin/rsyslog/rsyslog-5.8.11.ebuild |
| # LICENSE="|| ( LGPL-2.1 MPL-1.1 )" |
| # for third_party/portage-stable/x11-libs/cairo/cairo-1.8.8.ebuild |
| |
| # The parser isn't very smart and only has basic support for the |
| # || ( X Y ) OR logic to do the following: |
| # In order to save time needlessly unpacking packages and looking or a |
| # cleartext license (which is really a crapshoot), if we have a license |
| # like BSD that requires looking for copyright attribution, but we can |
| # chose another license like GPL, we do that. |
| |
| if not self.ebuild_license_names: |
| logging.error("%s: no license found in ebuild. FIXME!", self.fullnamerev) |
| # In a bind, you could comment this out. I'm making the output fail to |
| # get your attention since this error really should be fixed, but if you |
| # comment out the next line, the script will try to find a license inside |
| # the source. |
| raise PackageLicenseError() |
| |
| # This is not invalid, but the parser can't deal with it, so if it ever |
| # happens, error out to tell the programmer to do something. |
| if "||" in self.ebuild_license_names[1:]: |
| raise AssertionError("%s: Can't parse || in the middle of a license: %s" |
| % (self.fullnamerev, |
| ' '.join(self.ebuild_license_names))) |
| |
| or_licenses_and_one_is_no_attribution = False |
| # We do a quick early pass first so that the longer pass below can |
| # run accordingly. |
| for license_name in [x for x in self.ebuild_license_names |
| if x not in LICENCES_IGNORE]: |
| # Here we have an OR case, and one license that we can use stock, so |
| # we remember that in order to be able to skip license attributions if |
| # any were in the OR. |
| if (self.ebuild_license_names[0] == "||" and |
| license_name not in COPYRIGHT_ATTRIBUTION_LICENSES): |
| or_licenses_and_one_is_no_attribution = True |
| |
| for license_name in [x for x in self.ebuild_license_names |
| if x not in LICENCES_IGNORE]: |
| # Licenses like BSD or MIT can't be used as is because they do not contain |
| # copyright self. They have to be replaced by copyright file given in the |
| # source code, or manually mapped by us in PACKAGE_LICENSES |
| if license_name in COPYRIGHT_ATTRIBUTION_LICENSES: |
| # To limit needless efforts, if a package is BSD or GPL, we ignore BSD |
| # and use GPL to avoid scanning the package, but we can only do this if |
| # or_licenses_and_one_is_no_attribution has been set above. |
| # This ensures that if we have License: || (BSD3 BSD4), we will |
| # look in the source. |
| if or_licenses_and_one_is_no_attribution: |
| logging.info("%s: ignore license %s because ebuild LICENSES had %s", |
| self.fullnamerev, license_name, |
| ' '.join(self.ebuild_license_names)) |
| else: |
| logging.info("%s: can't use %s, will scan source code for copyright", |
| self.fullnamerev, license_name) |
| self.need_copyright_attribution = True |
| self.scan_source_for_licenses = True |
| else: |
| self.license_names.add(license_name) |
| # We can't display just 2+ because it only contains text that says to |
| # read v2 or v3. |
| if license_name == 'GPL-2+': |
| self.license_names.add('GPL-2') |
| if license_name == 'LGPL-2+': |
| self.license_names.add('LGPL-2') |
| |
| if license_name in LOOK_IN_SOURCE_LICENSES: |
| logging.info("%s: Got %s, will try to find better license in source...", |
| self.fullnamerev, license_name) |
| self.scan_source_for_licenses = True |
| |
| if self.license_names: |
| logging.info('%s: using stock|cust license(s) %s', |
| self.fullnamerev, ','.join(self.license_names)) |
| |
| # If the license(s) could not be found, or one requires copyright |
| # attribution, dig in the source code for license files: |
| # For instance: |
| # Read licenses from ebuild for net-dialup/ppp-2.4.5-r3: BSD,GPL-2 |
| # We need get the substitution file for BSD and add it to GPL. |
| if self.scan_source_for_licenses: |
| self._ExtractLicenses() |
| |
| if not self.license_names and not self.license_text_scanned: |
| logging.error(""" |
| %s: unable to find usable license. |
| Typically this will happen because the ebuild says it's MIT or BSD, but there |
| was no license file that this script could find to include along with a |
| copyright attribution (required for BSD/MIT). |
| Go investigate the unpacked source in /tmp/boardname/tmp/portage/..., and |
| find which license to assign. Once you found it, add a static mapping to the |
| PACKAGE_LICENSES dict if that license is not in a file, or teach this script |
| to find the license file.""", |
| self.fullname) |
| raise PackageLicenseError() |
| |
| |
| class Licensing(object): |
| """Do the actual work of extracting licensing info and outputting html.""" |
| |
| def __init__(self, board, package_fullnames, |
| entry_template_file=ENTRY_TMPL): |
| |
| # eg x86-alex |
| self.board = board |
| # List of stock and custom licenses referenced in ebuilds. Used to |
| # print a report. Dict value says which packages use that license. |
| self.licenses = {} |
| |
| # This keeps track of whether we have an incomplete license file due to |
| # package errors during parsing. |
| # Any non empty list at the end shows the list of packages that caused |
| # errors. |
| self.incomplete_packages = [] |
| |
| self.package_text = {} |
| with codecs.open(entry_template_file, mode='rb', encoding="utf-8") as c: |
| self.entry_template = c.read() |
| |
| # We need to have a dict for the list of packages objects, index by package |
| # fullnamerev, so that when we scan our licenses at the end, and find out |
| # some shared licenses are only used by one package, we can access that |
| # package object by name, and add the license directly in that object. |
| self.packages = {} |
| self._package_fullnames = package_fullnames |
| |
| @property |
| def sorted_licenses(self): |
| return sorted(self.licenses.keys(), key=str.lower) |
| |
| def LicensedPackages(self, license_name): |
| """Return list of packages using a given license.""" |
| return self.licenses[license_name] |
| |
| def ProcessPackages(self): |
| """Iterate through all packages provided and gather their licenses. |
| |
| GetLicenses will scrape licenses from the code and/or gather stock license |
| names. We gather the list of stock and custom ones for later processing. |
| |
| Do not call this after adding virtual packages with AddExtraPkg. |
| """ |
| for package_name in self._package_fullnames: |
| pkg = PackageInfo(self.board) |
| try: |
| pkg.GetLicenses(package_name) |
| self.packages[package_name] = pkg |
| except PackageSkipped as e: |
| logging.info(e) |
| except PackageLicenseError: |
| self.incomplete_packages += [pkg.fullnamerev] |
| |
| def AddExtraPkg(self, pkg_data): |
| """Allow adding pre-created virtual packages. |
| |
| GetLicenses will not work on them, so add them after having run |
| ProcessPackages. |
| |
| Args: |
| pkg_data: array of package data as defined below |
| """ |
| pkg = PackageInfo(self.board) |
| pkg.category = pkg_data[0] |
| pkg.name = pkg_data[1] |
| pkg.version = pkg_data[2] |
| pkg.homepages = pkg_data[3] # this is a list |
| pkg.license_names = pkg_data[4] # this is also a list |
| pkg.ebuild_license_names = pkg_data[4] |
| self.packages[pkg.fullnamerev] = pkg |
| |
| @staticmethod |
| def _FindLicenseType(license_name): |
| """Says if a license is stock Gentoo, custom, or doesn't exist.""" |
| |
| for directory in STOCK_LICENSE_DIRS: |
| path = '%s/%s' % (directory, license_name) |
| if os.path.exists(path): |
| return "Gentoo Package Stock" |
| |
| for directory in CUSTOM_LICENSE_DIRS: |
| path = '%s/%s' % (directory, license_name) |
| if os.path.exists(path): |
| return "Custom" |
| |
| raise AssertionError("license %s could not be found in %s" |
| % (license_name, |
| '\n'.join(STOCK_LICENSE_DIRS + CUSTOM_LICENSE_DIRS)) |
| ) |
| |
| @staticmethod |
| def _ReadSharedLicense(license_name): |
| """Read and return stock or cust license file specified in an ebuild.""" |
| |
| license_path = None |
| for directory in STOCK_LICENSE_DIRS + CUSTOM_LICENSE_DIRS: |
| path = '%s/%s' % (directory, license_name) |
| if os.access(path, os.F_OK): |
| license_path = path |
| break |
| |
| if license_path: |
| return ReadUnknownEncodedFile(license_path, "read license") |
| else: |
| raise AssertionError("license %s could not be found in %s" |
| % (license_name, |
| '\n'.join(STOCK_LICENSE_DIRS + |
| CUSTOM_LICENSE_DIRS))) |
| |
| @staticmethod |
| def EvaluateTemplate(template, env): |
| """Expand a template with vars like {{foo}} using a dict of expansions.""" |
| # TODO switch to stock python templates. |
| for key, val in env.iteritems(): |
| template = template.replace('{{%s}}' % key, val) |
| return template |
| |
| def _GeneratePackageLicenseText(self, package): |
| """Concatenate all licenses related to a package. |
| |
| This means a combination of ebuild shared licenses and licenses read from |
| the package source tree, if any. |
| |
| Args: |
| package: PackageInfo object |
| |
| Raises: |
| AssertionError: on runtime errors |
| """ |
| |
| license_text = [] |
| for license_text_scanned in package.license_text_scanned: |
| license_text.append(license_text_scanned) |
| license_text.append('%s\n' % ('-=' * 40)) |
| |
| license_pointers = [] |
| # sln: shared license name. |
| for sln in package.license_names: |
| # Says whether it's a stock gentoo or custom license. |
| license_type = self._FindLicenseType(sln) |
| license_pointers.append( |
| "<li><a href='#%s'>%s License %s</a></li>" % ( |
| sln, license_type, sln)) |
| |
| # This should get caught earlier, but one extra check. |
| if not (license_text + license_pointers): |
| raise AssertionError('Ended up with no license_text') |
| |
| env = { |
| 'name': "%s-%s" % (package.name, package.version), |
| 'url': package.homepages[0] if package.homepages else '', |
| 'licenses_txt': cgi.escape('\n'.join(license_text)) or '', |
| 'licenses_ptr': '\n'.join(license_pointers) or '', |
| } |
| self.package_text[package] = self.EvaluateTemplate(self.entry_template, env) |
| |
| def GenerateHTMLLicenseOutput(self, output_file, |
| output_template=TMPL, |
| license_template=SHARED_LICENSE_TMPL): |
| """Generate the combined html license file used in ChromeOS. |
| |
| Args: |
| output_file: resulting HTML license output. |
| output_template: template for the entire HTML file. |
| license_template: template for shared license entries. |
| """ |
| sorted_license_txt = [] |
| |
| # Keep track of which licenses are used by which packages. |
| for pkg in self.packages.values(): |
| for sln in pkg.license_names: |
| self.licenses.setdefault(sln, []).append(pkg.fullnamerev) |
| |
| # Find licenses only used once, and roll them in the package that uses them. |
| # We use keys() because licenses is modified in the loop, so we can't use |
| # an iterator. |
| for sln in self.licenses.keys(): |
| if len(self.licenses[sln]) == 1: |
| pkg_fullnamerev = self.licenses[sln][0] |
| logging.info("Collapsing shared license %s into single use license " |
| "(only used by %s)", sln, pkg_fullnamerev) |
| license_type = self._FindLicenseType(sln) |
| license_txt = self._ReadSharedLicense(sln) |
| single_license = "%s License %s:\n\n%s" % (license_type, sln, |
| license_txt) |
| pkg = self.packages[pkg_fullnamerev] |
| pkg.license_text_scanned.append(single_license) |
| pkg.license_names.remove(sln) |
| del self.licenses[sln] |
| |
| for pkg in sorted(self.packages.values(), |
| key=lambda x: (x.name.lower(), x.version, x.revision)): |
| self._GeneratePackageLicenseText(pkg) |
| sorted_license_txt += [self.package_text[pkg]] |
| |
| # Now generate the bottom of the page that will contain all the shared |
| # licenses and a list of who is pointing to them. |
| with codecs.open(license_template, mode='rb', encoding="utf-8") as c: |
| license_template = c.read() |
| licenses_txt = [] |
| for license_name in self.sorted_licenses: |
| env = { |
| 'license_name': license_name, |
| 'license': self._ReadSharedLicense(license_name), |
| 'license_type': self._FindLicenseType(license_name), |
| 'license_packages': ' '.join(self.LicensedPackages(license_name)), |
| } |
| licenses_txt += [self.EvaluateTemplate(license_template, env)] |
| |
| with codecs.open(output_template, mode='rb', encoding="utf-8") as c: |
| file_template = c.read() |
| env = { |
| 'entries': '\n'.join(sorted_license_txt), |
| 'licenses': '\n'.join(licenses_txt), |
| } |
| with codecs.open(output_file, mode='w', encoding="utf-8") as c: |
| c.write(self.EvaluateTemplate(file_template, env)) |
| |
| |
| def ListInstalledPackages(board): |
| """Return a list of all packages installed for a particular board.""" |
| |
| # Please leave the following as documentation of what options are available |
| # and why we chose the current one. |
| |
| # The following returns all packages that were part of the build tree |
| # (many get built or used during the build, but do not get shipped). |
| # Note that it also contains packages that are in the build as |
| # defined by build_packages but not part of the image we ship. |
| # args = "/usr/local/bin/equery-%s list '*'" % board |
| |
| # This is better because it only lists packages that are in the |
| # chromeos-base/chromeos dependency chain and ignores packages that |
| # just happen to be available in your build tree. |
| # args = "emerge-%s --with-bdeps=y --usepkgonly --emptytree \ |
| # --pretend chromeos-base/chromeos | \ |
| # sed -n -E -e 's/^\[[^]]*\] ([^ ]*) .*$/\\1/p'" % board |
| |
| # Another option which we've decided not to use, is bdeps=n. This |
| # outputs just the packages we ship, but does not list packages |
| # that were used to build them, including a package like flex which |
| # generates a .a that is included and shipped in ChromeOS. |
| # We've decided to credit build packages, even if we're not legally |
| # required to (it's always nice to do), and that way we get corner |
| # case packages like flex. |
| # |
| # return cros_build_lib.RunCommand(args, print_cmd=debug, shell=True, |
| # redirect_stdout=True).output.splitlines() |
| |
| args = ["emerge-%s" % board, "--with-bdeps=y", "--usepkgonly", "--emptytree", |
| "--pretend", "chromeos-base/chromeos"] |
| emerge = cros_build_lib.RunCommand(args, print_cmd=debug, |
| redirect_stdout=True).output.splitlines() |
| |
| packages = [] |
| # [binary R ] x11-libs/libva-1.1.1 to /build/x86-alex/ |
| pkg_rgx = re.compile(r'\[[^]]+\] (.+) to /build/.*') |
| for line in emerge: |
| match = pkg_rgx.search(line) |
| if match: |
| packages.append(match.group(1)) |
| |
| return packages |
| |
| |
| def ReadUnknownEncodedFile(file_path, logging_text): |
| """Read a file of unknown encoding (UTF-8 or latin) by trying in sequence. |
| |
| Args: |
| file_path: what to read. |
| logging_text: what to display for logging depending on file read. |
| |
| Returns: |
| file content, possibly converted from latin1 to UTF-8. |
| """ |
| |
| try: |
| with codecs.open(file_path, encoding="utf-8") as c: |
| file_txt = c.read() |
| logging.info("%s %s (UTF-8)", logging_text, file_path) |
| except UnicodeDecodeError: |
| with codecs.open(file_path, encoding="latin1") as c: |
| file_txt = c.read() |
| logging.info("%s %s (latin1)", logging_text, file_path) |
| return file_txt |
| |
| |
| def main(args): |
| # pylint: disable=W0603 |
| global SKIPPED_PACKAGES |
| global debug |
| # pylint: enable=W0603 |
| |
| parser = commandline.ArgumentParser(usage=__doc__) |
| parser.add_argument("-t", "--testpkg", |
| help="force a single package for debugging, like" |
| " dev-libs/libatomic_ops-7.2d") |
| parser.add_argument("board", |
| help="which board to run for, like x86-alex") |
| parser.add_argument("output_file", type="path", |
| help="which html file to create with output") |
| opts = parser.parse_args(args) |
| debug = opts.debug |
| |
| board, output_file, testpkg = opts.board, opts.output_file, opts.testpkg |
| |
| # We have a hardcoded list of skipped packages for various reasons, but we |
| # also exclude any google platform package from needing a license since they |
| # are covered by the top license in the tree. |
| cmd = "cros_workon info --all --host | grep src/platform/ | awk '{print $1}'" |
| packages = cros_build_lib.RunCommand(cmd, shell=True, print_cmd=debug, |
| redirect_stdout=True).output.splitlines() |
| SKIPPED_PACKAGES += packages |
| |
| # For temporary single package debugging (make sure to include trailing -ver): |
| if testpkg: |
| logging.info("Will only generate license for %s", testpkg) |
| packages = [testpkg] |
| else: |
| packages = ListInstalledPackages(board) |
| # If the caller forgets to set $board, it'll default to beaglebone, and return |
| # no packages. Catch this and give a hint that the wrong board was given. |
| if not packages: |
| raise AssertionError('FATAL: Could not get any packages for board %s' % |
| board) |
| logging.debug("Package list to work through:") |
| logging.debug('\n'.join(packages)) |
| logging.debug("Will skip these packages:") |
| logging.debug('\n'.join(SKIPPED_PACKAGES)) |
| licensing = Licensing(board, packages) |
| licensing.ProcessPackages() |
| if not testpkg: |
| # We add 2 virtual packages as well as 2 boot packages that are included |
| # with some hardware, but not in the image or package list. |
| for extra_pkg in [ |
| ['x11-base', 'X.Org', '1.9.3', ['http://www.x.org/'], ['X']], |
| ['sys-kernel', 'Linux', '2.6', ['http://www.kernel.org/'], ['GPL-2']], |
| ['sys-boot', 'u-boot', '2013.06', ['http://www.denx.de/wiki/U-Boot'], |
| ['GPL-2+']], |
| ['sys-boot', 'coreboot', '2013.04', ['http://www.coreboot.org/'], |
| ['GPL-2']], |
| ]: |
| licensing.AddExtraPkg(extra_pkg) |
| licensing.GenerateHTMLLicenseOutput(output_file) |
| if licensing.incomplete_packages: |
| raise AssertionError(""" |
| DO NOT USE OUTPUT!!! |
| Some packages are missing due to errors, please look at errors generated during |
| this run. |
| List of packages with errors: |
| %s |
| """ % '\n'.join(licensing.incomplete_packages)) |
| |
| if __name__ == '__main__': |
| main(sys.argv[1:]) |
