| #!/usr/bin/python |
| # |
| # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| # TODO?: recursively look in packages to see if they have license files not |
| # at their top level. |
| # |
| # FIXME(merlin): remove this after fixing the current code. |
| # pylint: disable-msg=W0621 |
| |
| """Script that attempts to generate an HTML file containing license |
| information and homepage links for all installed packages. |
| |
| WARNING: this script in its current form is not finished or considered |
| production quality/code style compliant. This is an intermediate checkin |
| to allow for incremental cleanups and improvements that will make it |
| production quality. |
| |
| Usage: |
| For this script to work, you must have built the architecture |
| this is being run against, _after_ you've last run repo sync. |
| Otherwise, it will query newer source code and then fail to work on packages |
| that are out of date in your build. |
| |
| Recommended build: |
| cros_sdk |
| export board=x86-alex |
| sudo rm -rf /build/$board |
| cd ~/trunk/src/scripts |
| ./setup_board --board=$board |
| ./build_packages --board=$board --nowithautotest --nowithtest --nowithdev |
| cd ~/trunk/chromite/scripts/license-generation |
| ./licenses.py --debug $board out.html | tee output.sav |
| |
| The script is still experimental. Review at least ERROR output from it. |
| |
| The output file is meant to update |
| http://src.chromium.org/viewvc/chrome/trunk/src/chrome/browser/resources/ + |
| chromeos/about_os_credits.html?view=log |
| (gclient config svn://svn.chromium.org/chrome/trunk/src) |
| For an example CL, see https://codereview.chromium.org/13496002/ |
| |
| It is recommended that you use a fancy differ like 'meld' to review license |
| diffs. GNU diff will show too much irrelevant noise and not resync properly. |
| |
| UPDATE: gcl will probably fail now, because the file is too big. Before it |
| gets moved somewhere else, you should just use svn diff and svn commit. |
| |
| If you don't get this in before the freeze window, it'll need to be merged into |
| the branch being released, which is done by adding a Merge-Requested label to |
| Iteration-xx in the tracking bug. |
| Once it's been updated to "Merge-Approved" by a TPM, please merge into the |
| required release branch. You can ask karen@ for merge approve help. |
| Example: http://crbug.com/221281 |
| """ |
| |
| import cgi |
| import logging |
| import os |
| import portage |
| import subprocess |
| import sys |
| |
| # This keeps track of whether we have an incomplete license file due to package |
| # errors during parsing. |
| # Any non empty list at the end shows the list of packages that caused errors. |
| OUTPUT_INCOMPLETE = [] |
| |
| EQUERY_BASE = '/usr/local/bin/equery-%s' |
| |
| STOCK_LICENSE_DIRS = [ |
| os.path.expanduser('~/trunk/src/third_party/portage/licenses'), |
| os.path.expanduser('~/trunk/src/third_party/portage-stable/licenses'), |
| os.path.abspath(os.path.join(os.path.dirname(__file__), 'licenses')), |
| ] |
| |
| # Virtual packages don't need to have a license and often don't, so we skip them |
| # chromeos-base contains google platform packages that are covered by the |
| # general license at top of tree, so we skip those too. |
| SKIPPED_CATEGORIES = [ |
| 'chromeos-base', # TODO: this shouldn't be excluded. |
| 'virtual', |
| ] |
| |
| SKIPPED_PACKAGES = [ |
| # Fix these packages by adding a real license in the code. |
| # You should not skip packages just because the license scraping doesn't |
| # work. Stick those special cases into PACKAGE_LICENSES. |
| # Packages should only be here because they are sub/split packages already |
| # covered by the license of the main package. |
| |
| # These are Chrome-OS-specific packages, copyright BSD-Google |
| 'sys-kernel/chromeos-kernel', # already manually credit Linux |
| |
| # These have been split across several packages, so we skip listing the |
| # individual components (and just list the main package instead). |
| 'app-editors/vim-core', |
| 'x11-apps/mesa-progs', |
| |
| # Portage metapackage. |
| 'x11-base/xorg-drivers', |
| |
| # These are covered by app-i18n/ibus-mozc (BSD, copyright Google). |
| 'app-i18n/ibus-mozc-chewing', |
| 'app-i18n/ibus-mozc-hangul', |
| 'app-i18n/ibus-mozc-pinyin', |
| |
| # These are all X.org sub-packages; shouldn't be any need to list them |
| # individually. |
| 'media-fonts/encodings', |
| 'x11-apps/iceauth', |
| 'x11-apps/intel-gpu-tools', |
| 'x11-apps/mkfontdir', |
| 'x11-apps/rgb', |
| 'x11-apps/setxkbmap', |
| 'x11-apps/xauth', |
| 'x11-apps/xcursorgen', |
| 'x11-apps/xdpyinfo', |
| 'x11-apps/xdriinfo', |
| 'x11-apps/xev', |
| 'x11-apps/xgamma', |
| 'x11-apps/xhost', |
| 'x11-apps/xinit', |
| 'x11-apps/xinput', |
| 'x11-apps/xkbcomp', |
| 'x11-apps/xlsatoms', |
| 'x11-apps/xlsclients', |
| 'x11-apps/xmodmap', |
| 'x11-apps/xprop', |
| 'x11-apps/xrandr', |
| 'x11-apps/xrdb', |
| 'x11-apps/xset', |
| 'x11-apps/xset-mini', |
| 'x11-apps/xwininfo', |
| 'x11-base/xorg-server', |
| 'x11-drivers/xf86-input-evdev', |
| 'x11-drivers/xf86-input-keyboard', |
| 'x11-drivers/xf86-input-mouse', |
| 'x11-drivers/xf86-input-synaptics', |
| 'x11-drivers/xf86-video-intel', |
| 'x11-drivers/xf86-video-vesa', |
| 'x11-drivers/xf86-video-vmware', |
| 'x11-libs/libICE', |
| 'x11-libs/libSM', |
| 'x11-libs/libX11', |
| 'x11-libs/libXScrnSaver', |
| 'x11-libs/libXau', |
| 'x11-libs/libXcomposite', |
| 'x11-libs/libXcursor', |
| 'x11-libs/libXdamage', |
| 'x11-libs/libXdmcp', |
| 'x11-libs/libXext', |
| 'x11-libs/libXfixes', |
| 'x11-libs/libXfont', |
| 'x11-libs/libXfontcache', |
| 'x11-libs/libXft', |
| 'x11-libs/libXi', |
| 'x11-libs/libXinerama', |
| 'x11-libs/libXmu', |
| 'x11-libs/libXp', |
| 'x11-libs/libXrandr', |
| 'x11-libs/libXrender', |
| 'x11-libs/libXres', |
| 'x11-libs/libXt', |
| 'x11-libs/libXtst', |
| 'x11-libs/libXv', |
| 'x11-libs/libXvMC', |
| 'x11-libs/libXxf86vm', |
| 'x11-libs/libdrm', |
| 'x11-libs/libfontenc', |
| 'x11-libs/libpciaccess', |
| 'x11-libs/libxkbfile', |
| 'x11-libs/libxkbui', |
| 'x11-libs/pixman', |
| 'x11-libs/xtrans', |
| 'x11-misc/util-macros', |
| 'x11-misc/xbitmaps', |
| 'x11-proto/bigreqsproto', |
| 'x11-proto/compositeproto', |
| 'x11-proto/damageproto', |
| 'x11-proto/dri2proto', |
| 'x11-proto/fixesproto', |
| 'x11-proto/fontcacheproto', |
| 'x11-proto/fontsproto', |
| 'x11-proto/inputproto', |
| 'x11-proto/kbproto', |
| 'x11-proto/printproto', |
| 'x11-proto/randrproto', |
| 'x11-proto/recordproto', |
| 'x11-proto/renderproto', |
| 'x11-proto/resourceproto', |
| 'x11-proto/scrnsaverproto', |
| 'x11-proto/trapproto', |
| 'x11-proto/videoproto', |
| 'x11-proto/xcmiscproto', |
| 'x11-proto/xextproto', |
| 'x11-proto/xf86bigfontproto', |
| 'x11-proto/xf86dgaproto', |
| 'x11-proto/xf86driproto', |
| 'x11-proto/xf86rushproto', |
| 'x11-proto/xf86vidmodeproto', |
| 'x11-proto/xineramaproto', |
| 'x11-proto/xproto', |
| ] |
| |
| ARCHIVE_SUFFIXES = [ '.tar.gz', '.tgz', '.tar.bz2', '.tbz2' ] |
| |
| # TODO(merlin): Normalize case in comparison check without mangling the case |
| # Matching is done in lowercase, you MUST give lowercase names. |
| LICENSE_FILENAMES = [ |
| 'copying', |
| 'copyright', |
| 'licence', # used by openssh |
| 'license', |
| 'license.txt', # used by hdparm, used by NumPy, glew |
| 'copyright', |
| 'ipa_font_license_agreement_v1.0.txt', # used by ja-ipafonts |
| ] |
| |
| SKIPPED_LICENSE_FILENAME_COMPONENTS = [ |
| # FIXME: check whether this should be excluded. |
| 'third_party' |
| ] |
| |
| # These are _temporary_ license mappings for packages that do not have a valid |
| # stock license, or LICENSE file we can use. |
| # Once this script runs earlier (during the package build process), it will |
| # block new source without a LICENSE file. |
| # At that point, new packages will get fixed to include LICENSE instead of |
| # adding workaround mappings like those below. |
| # We should also fix the packages listed below so that the hardcoded |
| # mappings can be obsoleted. |
| PACKAGE_LICENSES = { |
| # One off licenses. Should we check in a custom LICENSE file in upstream? |
| 'dev-python/netifaces': ['netiface'], |
| 'net-dialup/ppp': ['ppp-2.4.4'], |
| 'sys-libs/ncurses': ['ncurses'], |
| |
| # BSD and MIT license authorship mapping. |
| # Ideally we should have a custom LICENSE file in the upstream source. |
| # TODO: BSD-2: bsdiff is missing a license file, add one upstream. |
| 'dev-util/bsdiff': ['BSD-bsdiff'], |
| # TODO: libevent is missing a license file, add one upstream. |
| 'dev-libs/libevent': ['BSD-libevent'], |
| # TODO: dhcpcd is missing a license file, (c) in README. Add one upstream. |
| 'net-misc/dhcpcd': ['BSD-dhcpcd'], |
| # TODO: iputils is missing a license file, add one upstream. |
| 'net-misc/iputils': ['BSD-iputils'], |
| # TODO: c-ares is missing a license file, add one upstream. |
| 'net-dns/c-ares': ['MIT-MIT'], |
| |
| # TODO: We should just check in a LICENSE file in all of these: |
| 'app-i18n/input-tools': ['BSD-Google'], |
| 'app-i18n/nacl-mozc': ['BSD-Google'], |
| 'app-i18n/ibus-mozc': ['BSD-Google'], |
| 'media-plugins/o3d': ['BSD-Google'], |
| 'dev-python/unittest2': ['BSD-Google'], |
| |
| # These packages are not in Alex, check and remove later (might be used in |
| # other platforms). |
| #'media-libs/freeimage': ['GPL-2'], |
| #'sys-libs/talloc': ['LGPL-3'], # ebuild incorrectly says GPL-3 |
| #'app-crypt/nss': ['MPL-1.1'], |
| #'media-libs/jpeg': ['jpeg'], |
| #'app-editors/gentoo-editor': ['MIT-gentoo-editor'], |
| # |
| # 'media-fonts/font-util': ['font-util'], # COPYING file from git repo |
| # 'net-wireless/iwl1000-ucode': ['Intel-iwl1000'], |
| # 'sys-process/vixie-cron': ['vixie-cron'], |
| } |
| |
| INVALID_STOCK_LICENSES = [ |
| 'BSD', # requires distribution of copyright notice |
| 'BSD-2',# so does BSD-2 http://opensource.org/licenses/BSD-2-Clause |
| 'BSD-3',# and BSD-3 http://opensource.org/licenses/BSD-3-Clause ? |
| 'BSD-4', |
| 'BSD-with-attribution', |
| 'MIT', # requires distribution of copyright notice |
| 'MIT-with-advertising', # requires distribution of copyright notice |
| ] |
| |
| PACKAGE_HOMEPAGES = { |
| 'app-editors/vim': ['http://www.vim.org/'], |
| 'x11-proto/glproto': ['http://www.x.org/'], |
| } |
| |
| TEMPLATE_FILE = 'about_credits.tmpl' |
| ENTRY_TEMPLATE_FILE = 'about_credits_entry.tmpl' |
| |
| class PackageInfo: |
| def __init__(self, category=None, name=None, version=None, revision=None): |
| self.category = category |
| self.name = name |
| self.version = version |
| if revision is not None: |
| revision = str(revision).lstrip('r') |
| if revision == '0': |
| revision = None |
| self.revision = revision |
| self.description = None |
| self.homepages = [] |
| self.license_names = [] |
| self.license_text = None |
| |
| @property |
| def fullnamerev(self): |
| s = '%s-%s' % (self.fullname, self.version) |
| if self.revision: |
| s += '-r%s' % self.revision |
| return s |
| |
| @property |
| def fullname(self): |
| return '%s/%s' % (self.category, self.name) |
| |
| def _RunEbuildPhases(self, path, *phases, **kwargs): |
| """Receives something like: |
| path = /mnt/host/source/src/ |
| third_party/portage-stable/net-misc/rsync/rsync-3.0.8.ebuild |
| phases = ['clean', 'fetch'] or ['unpack'].""" |
| |
| #logging.debug('ebuild-%s | %s | %s', board, path, str(list(phases))) |
| subprocess.check_call( |
| ['ebuild-%s' % board, path] + list(phases), **kwargs) |
| |
| def ExtractLicense(self): |
| """Try to get a license from the package by unpacking it with ebuild |
| and looking for license files in the unpacked tree. |
| """ |
| |
| # Some packages have hardcoded licenses and are generated in |
| # GetPackageInfo, so we skip the license extraction and exit early. |
| # FIXME, this is wrong: a hardcoded license now should mean that we're |
| # adding a license to whatever can also be found in the source. |
| # Here's an example: |
| # Read licenses from ebuild for net-dialup/ppp-2.4.5-r3: BSD,GPL-2 |
| # We need to both get the substitution file for BSD, and display the GPL |
| # license. |
| # The new code will display both the licenses mapped manually, and the ones |
| # found in the source code. |
| if self.fullname in PACKAGE_LICENSES: |
| return False |
| |
| path = GetEbuildPath(board, self.fullnamerev) |
| self._RunEbuildPhases( |
| path, 'clean', 'fetch', |
| stdout=open('/dev/null', 'wb'), |
| stderr=subprocess.STDOUT) |
| self._RunEbuildPhases(path, 'unpack') |
| try: |
| return self._ExtractLicense() |
| finally: |
| if not debug: |
| # In debug mode, leave unpacked trees so that we can look for files |
| # inside them. |
| self._RunEbuildPhases(path, 'clean') |
| |
| def _ExtractLicense(self): |
| """Scan the unpacked source code for what looks like license files |
| as defined in LICENSE_FILENAMES. |
| """ |
| p = subprocess.Popen(['portageq-%s' % board, 'envvar', |
| 'PORTAGE_TMPDIR'], stdout=subprocess.PIPE) |
| tmpdir = p.communicate()[0].strip() |
| ret = p.wait() |
| if ret != 0: |
| raise AssertionError('exit code was not 0: got %s' % ret) |
| |
| # tmpdir gets something like /build/daisy/tmp/ |
| workdir = os.path.join(tmpdir, 'portage', self.fullnamerev, 'work') |
| |
| args = ['find', workdir + '/', '-maxdepth', '3', |
| '-mindepth', '1', '-type', 'f'] |
| p = subprocess.Popen(args, stdout=subprocess.PIPE) |
| files = p.communicate()[0].splitlines() |
| ret = p.wait() |
| if ret != 0: |
| raise AssertionError('exit code was not 0: got %s' % ret) |
| |
| files = [x[len(workdir):].lstrip('/') for x in files] |
| licenses = [] |
| for name in files: |
| if os.path.basename(name).lower() in LICENSE_FILENAMES and \ |
| (name.count('/') == 1 or (name.count('/') == 2 and |
| name.split('/')[1] == 'doc')): |
| has_skipped_component = False |
| # FIXME: Should we really exclude third_party? |
| for comp in SKIPPED_LICENSE_FILENAME_COMPONENTS: |
| if comp in name: |
| has_skipped_component = True |
| break |
| if not has_skipped_component: |
| licenses.append(name) |
| |
| if not licenses: |
| logging.warn("%s: couldn't find license file in %s", |
| self.fullnamerev, workdir) |
| return False |
| |
| # Examples of multiple license matches: |
| # dev-lang/swig-2.0.4-r1: swig-2.0.4/COPYRIGHT swig-2.0.4/LICENSE |
| # dev-libs/glib-2.32.4-r1: glib-2.32.4/COPYING pkg-config-0.26/COPYING |
| # dev-libs/libnl-3.2.14: libnl-doc-3.2.14/COPYING libnl-3.2.14/COPYING |
| # dev-libs/libpcre-8.30-r2: pcre-8.30/LICENCE pcre-8.30/COPYING |
| # dev-libs/libusb-0.1.12-r6: libusb-0.1.12/COPYING libusb-0.1.12/LICENSE |
| # dev-libs/pyzy-0.1.0-r1: db/COPYING pyzy-0.1.0/COPYING |
| # net-misc/strongswan-5.0.2-r4: strongswan-5.0.2/COPYING |
| # strongswan-5.0.2/LICENSE |
| # sys-process/procps-3.2.8_p11: debian/copyright procps-3.2.8/COPYING |
| logging.info('License(s) for %s: %s', self.fullnamerev, ' '.join(licenses)) |
| self.license_text = "" |
| for license_file in licenses: |
| logging.debug("Adding license %s:", os.path.join(workdir, license_file)) |
| self.license_text += "Source license %s:\n\n" % license_file |
| self.license_text += open(os.path.join(workdir, license_file)).read() |
| self.license_text += "\n\n" |
| return True |
| |
| # See if the ebuild file itself contains a license, in case there is no text |
| # license in the source code. |
| def GetStockLicense(self): |
| if not self.license_names: |
| logging.warning('%s: no stock licenses from ebuild', self.fullnamerev) |
| return False |
| |
| logging.info('%s: using stock license(s) %s', |
| self.fullnamerev, ','.join(self.license_names)) |
| |
| license_texts = [] |
| for license_name in self.license_names: |
| logging.debug("looking for license %s for %s", license_name, |
| self.fullnamerev) |
| license_path = None |
| for directory in STOCK_LICENSE_DIRS: |
| path = '%s/%s' % (directory, license_name) |
| if os.access(path, os.F_OK): |
| license_path = path |
| break |
| if license_path: |
| logging.info('%s: reading license %s', self.fullnamerev, license_path) |
| license_texts.append("Gentoo Package Provided Stock License %s:" % |
| license_name) |
| license_texts.append(open(license_path).read()) |
| license_texts.append("\n\n") |
| else: |
| # If a package with multiple stock licenses has one that we don't have, |
| # we report this, but it's ok to continue since we only have to honor/ |
| # repeat one of the licenses. Still, worth looking into just in case. |
| # sys-apps/hwids currently has a LICENSE field that triggers this: |
| # LICENSE="|| ( GPL-2 BSD )" |
| logging.error('%s: stock license %s could not be found in %s', |
| self.fullnamerev, license_name, '\n'.join(STOCK_LICENSE_DIRS)) |
| |
| if not license_texts: |
| logging.error('%s: couldn\'t find any stock licenses', self.fullnamerev) |
| return False |
| |
| self.license_text = '\n'.join(license_texts) |
| return True |
| |
| |
| def ListInstalledPackages(board): |
| """Return a list of all packages installed for a particular board.""" |
| # FIXME(merlin): davidjames pointed out that this is |
| # not the right way to get the package list as it does not apply |
| # filters. This should change to ~/trunk/src/scripts/get_package_list |
| args = [EQUERY_BASE % board, 'list', '*'] |
| p = subprocess.Popen(args, stdout=subprocess.PIPE) |
| return [s.strip() for s in p.stdout.readlines()] |
| |
| |
| def BuildMetaPackages(): |
| pkgs = [] |
| |
| pkg = PackageInfo('x11-base', 'X.Org', '1.9.3') |
| pkg.homepages = [ 'http://www.x.org/' ] |
| pkg.license_names = [ 'X' ] |
| pkgs.append(pkg) |
| |
| pkg = PackageInfo('sys-kernel', 'Linux', '2.6') |
| pkg.homepages = [ 'http://www.kernel.org/' ] |
| pkg.license_names = [ 'GPL-2' ] |
| pkgs.append(pkg) |
| |
| for pkg in pkgs: |
| pkg.GetStockLicense() |
| return pkgs |
| |
| |
| def GetEbuildPath(board, name): |
| """Turns (x86-alex, net-misc/wget-1.12) into |
| /mnt/host/source/src/third_party/portage-stable/net-misc/wget/wget-1.12.ebuild |
| """ |
| p = subprocess.Popen( |
| ['equery-%s' % board, 'which', name], stdout=subprocess.PIPE) |
| stdout = p.communicate()[0] |
| p.wait() |
| path = stdout.strip() |
| logging.debug("equery-%s which %s", board, name) |
| logging.debug(" -> %s", path) |
| if not path: |
| raise AssertionError('GetEbuildPath for %s failed.\n' |
| 'Is your tree clean? Delete /build/%s and rebuild' % |
| (name, board)) |
| return path |
| |
| |
| def GetPackageInfo(fullnamewithrev): |
| """Create a PackageInfo object and populate its license, homepage and |
| description if they are valid. |
| Returns a package info object if the package isn't in a skip list. |
| A package without license is returned with incomplete data, a return of |
| None actually means we don't want to keep track of this package.""" |
| |
| # (category, name, version, revision) |
| info = PackageInfo(*portage.versions.catpkgsplit(fullnamewithrev)) |
| # The above will error if portage returns Null because you fed a bad package |
| # name, or forgot to append the version number: |
| # TypeError: PackageInfo constructor argument after * must be a sequence, |
| # not NoneType |
| |
| |
| if info.category in SKIPPED_CATEGORIES: |
| logging.info("%s in SKIPPED_CATEGORIES, skip info object creation", |
| info.fullname) |
| return None |
| |
| if info.fullname in SKIPPED_PACKAGES: |
| logging.info("%s in SKIPPED_PACKAGES, skip info object creation", |
| info.fullname) |
| return None |
| |
| ebuild = GetEbuildPath(board, info.fullnamerev) |
| |
| # FIXME(merlin): Is it ok to just return an unprocessed object if the |
| # ebuild can't be found? I think not. Consider dying here. |
| if not os.access(ebuild, os.F_OK): |
| logging.error("Can't access %s", ebuild) |
| return info |
| |
| cmd = [ |
| 'portageq', |
| 'metadata', |
| '/build/%s' % board, |
| 'ebuild', |
| info.fullnamerev, |
| 'HOMEPAGE', 'LICENSE', 'DESCRIPTION', |
| ] |
| p = subprocess.Popen(cmd, stdout=subprocess.PIPE) |
| lines = [s.strip() for s in p.stdout.readlines()] |
| p.wait() |
| |
| if p.returncode != 0: |
| raise AssertionError("%s failed" % cmd) |
| |
| # Runs: |
| # portageq metadata /build/x86-alex ebuild net-misc/wget-1.12-r2 \ |
| # HOMEPAGE LICENSE DESCRIPTION |
| # Returns: |
| # http://www.gnu.org/software/wget/ |
| # GPL-3 |
| # Network utility to retrieve files from the WWW |
| |
| (info.homepages, licenses, info.description) = ( |
| lines[0].split(), lines[1].split(), lines[2:]) |
| |
| # NOTE: the ebuild license field can look like: |
| # LICENSE="GPL-3 LGPL-3 Apache-2.0" (this means AND, as in all 3) |
| # for third_party/portage-stable/app-admin/rsyslog/rsyslog-5.8.11.ebuild |
| # LICENSE="|| ( LGPL-2.1 MPL-1.1 )" |
| # for third_party/portage-stable/x11-libs/cairo/cairo-1.8.8.ebuild |
| # LICENSE="Marvell International Ltd." |
| # for net-wireless/marvell_sd8787/marvell_sd8787-14.64.2.47-r16.ebuild |
| |
| # The parser does not know the || ( X Y ) OR logic |
| # It partially ignores the AND logic by skipping BSD licenses |
| # And it craps out on the Marvel license trying to look for |
| # 'Marvel' (found), 'International' (not found), 'Ltd' (not found). |
| # This blows... |
| |
| # Solution: show all licenses listed, ignore AND/OR and ignore errors |
| # like not finding an 'Ltd' license (show a warning, but do not die). |
| # This might skip a copyright attribution for a BSD license that does not |
| # include its own license file. We can fix on case by case basis with override |
| # mappings or putting a LICENSE file in the upstream code. |
| # TODO(merlin): have a list of licenses tokens to ignore to cut down on |
| # warnings '||' '(' ')' 'International' 'Ltd.' |
| |
| if info.fullname in PACKAGE_HOMEPAGES: |
| info.homepages = PACKAGE_HOMEPAGES[info.fullname] |
| |
| # Packages with missing licenses or licenses that need mapping (like BSD/MIT) |
| # are hardcoded here: |
| if info.fullname in PACKAGE_LICENSES: |
| licenses = PACKAGE_LICENSES[info.fullname] |
| logging.debug("Static license mapping for %s: %s", info.fullnamerev, |
| ",".join(licenses)) |
| else: |
| logging.debug("Read licenses from ebuild for %s: %s", info.fullnamerev, |
| ",".join(licenses)) |
| info.license_names = [] |
| for license_name in licenses: |
| # Licenses like BSD or MIT can't be used as it because they do not contain |
| # copyright info. They have to be replaced by a custom file generated by us. |
| if license_name in INVALID_STOCK_LICENSES: |
| logging.warning('%s: cannot use stock license %s, skipping...', |
| info.fullnamerev, license_name) |
| else: |
| info.license_names.append(license_name) |
| |
| return info |
| |
| |
| def EvaluateTemplate(template, env, escape=True): |
| """Expand a template with variables like {{foo}} using a |
| dictionary of expansions.""" |
| for key, val in env.iteritems(): |
| if escape: |
| val = cgi.escape(val) |
| template = template.replace('{{%s}}' % key, val) |
| return template |
| |
| def ProcessPkg(package): |
| # First, we try to retrieve package and license data from the ebuild, and |
| # return this in a created info object. |
| # This will also set static license mappings stored in this script. |
| info = GetPackageInfo(package) |
| # None is returned if the package is in a skip list, not if the license is |
| # invalid or missing. |
| if not info: |
| return None |
| |
| # From that info object, either get a mapped license file as per the |
| # PACKAGE_LICENSES dict, or retrieve/unpack the source to look for license |
| # files (scanning recursively by file name). |
| # Note that finding a license file OVERRIDES any license specified in |
| # the ebuild. |
| if (not info.ExtractLicense() and |
| not info.GetStockLicense()): |
| # ^^^^^^^^^^^^^^^^^^^^^^ |
| # If no license looking file is found in source archive, use our own copy |
| # of a stock license file matching what's defined in the ebuild (if any): |
| logging.error(""" |
| %s: unable to find usable license. |
| Typically this will happen because the ebuild says it's MIT or BSD, but there |
| was no license file that this script could find to include along with a |
| copyright attribution (required for BSD/MIT). |
| Go investigate the unpacked source in /tmp/boardname/tmp/portage/..., and |
| find which license to assign. Once you found it, add a static mapping to the |
| PACKAGE_LICENSES dict. |
| If there was a usable license file, you may also want to teach this script to |
| find it if you have time.""", |
| info.fullname) |
| OUTPUT_INCOMPLETE.append(info.fullname) |
| |
| return info |
| |
| |
| if __name__ == '__main__': |
| debug = False |
| if len(sys.argv) > 1 and sys.argv[1] == "--debug": |
| debug = True |
| sys.argv.pop(0) |
| logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG) |
| else: |
| logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) |
| |
| |
| if len(sys.argv) != 3: |
| print >> sys.stderr, (__doc__) |
| sys.exit(1) |
| |
| entry_template = open(ENTRY_TEMPLATE_FILE, 'rb').read() |
| |
| # We have a hardcoded list of skipped packages for various reasons, but we |
| # also exclude any google platform package from needing a license since they |
| # are covered by the top license in the tree. |
| cmd = "cros_workon info --all --host | grep src/platform/ |"\ |
| "awk '{print $1}'" |
| p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) |
| packages = p.communicate()[0].splitlines() |
| ret = p.wait() |
| if ret != 0: |
| raise AssertionError('%s exit code was not 0: got %s' % (cmd, ret)) |
| SKIPPED_PACKAGES += packages |
| |
| # FIXME(merlin): we should have proper command line parsing and allow passing |
| # a single package to generate a license for, as an argument. |
| board = sys.argv[1] |
| packages = ListInstalledPackages(board) |
| # If the caller forgets to set $board, it'll default to beaglebone, and return |
| # no packages. Catch this and give a hint that the wrong board was given. |
| if not packages: |
| raise AssertionError('FATAL: Could not get any packages for board %s' % |
| board) |
| # For temporary single package debugging (make sure to include trailing -ver): |
| #packages = [ "dev-python/unittest2-0.5.1" ] |
| logging.debug("Package list to work through:") |
| logging.debug('\n'.join(packages)) |
| logging.debug("Will skip these packages:") |
| logging.debug('\n'.join(SKIPPED_PACKAGES)) |
| |
| infos = filter(None, (ProcessPkg(x) for x in packages)) |
| infos += BuildMetaPackages() |
| infos.sort(key=lambda x: (x.name, x.version, x.revision)) |
| |
| entries = [] |
| seen_package_names = set() |
| for info in infos: |
| if info.name in seen_package_names: |
| continue |
| seen_package_names.add(info.name) |
| env = { |
| 'name': info.name, |
| 'url': info.homepages[0] if info.homepages else '', |
| 'license': info.license_text or '', |
| } |
| entries.append(EvaluateTemplate(entry_template, env)) |
| |
| file_template = open(TEMPLATE_FILE, 'rb').read() |
| out_file = open(sys.argv[2], "w") |
| out_file.write(EvaluateTemplate(file_template, |
| { 'entries': '\n'.join(entries) }, |
| escape=False)) |
| |
| if OUTPUT_INCOMPLETE: |
| raise AssertionError(""" |
| DO NOT USE OUTPUT!!! |
| Some packages are missing due to errors, please look at errors generated during |
| this run. |
| List of packages with errors: |
| %s |
| """ % '\n'.join(OUTPUT_INCOMPLETE)) |