scripts/license-generation/licenses.py

#!/usr/bin/python
#
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# TODO?: recursively look in packages to see if they have license files not
# at their top level.
#
# FIXME(merlin): remove this after fixing the current code.
# pylint: disable-msg=W0621

"""Script that attempts to generate an HTML file containing license
information and homepage links for all installed packages.

WARNING: this script in its current form is not finished or considered
production quality/code style compliant. This is an intermediate checkin
to allow for incremental cleanups and improvements that will make it
production quality.

Usage:
For this script to work, you must have built the architecture
this is being run against, _after_ you've last run repo sync.
Otherwise, it will query newer source code and then fail to work on packages
that are out of date in your build.

Recommended build:
  cros_sdk
  sudo rm -rf /build/$board
  setup_board --board=$board
  build_packages --board=$board --nowithautotest --nowithtest --nowithdev
  cd ~/trunk/chromite/scripts/license-generation
  ./license.py $board out.html

The output file is meant to update
http://src.chromium.org/viewvc/chrome/trunk/src/chrome/browser/resources/ +
  chromeos/about_os_credits.html?view=log
(gclient config svn://svn.chromium.org/chrome/trunk/src)
For an example CL, see https://codereview.chromium.org/13496002/

After that, it needs to be merged into the branch being released, which is
apparently done by adding a Merge-Requested label to Iteration-xx in the
tracking bug, and having karen@ merge it for you:
http://crbug.com/221281
"""

import cgi
import logging
import multiprocessing
import os
import portage
import subprocess
import sys

EQUERY_BASE = '/usr/local/bin/equery-%s'

PROCESS_EBUILD_SCRIPT = './process_ebuild.sh'

DISTFILES_DIR = '/var/lib/portage/distfiles'

STOCK_LICENSE_DIRS = [
  os.path.expanduser('~/trunk/src/third_party/portage/licenses'),
  os.path.expanduser('~/trunk/src/third_party/portage-stable/licenses'),
  os.path.abspath(os.path.join(os.path.dirname(__file__), 'licenses')),
]

# Virtual packages don't need to have a license and often don't, so we skip them
# chromeos-base contains google platforms packages that are covered by the
# general license at top of tree, so we skip those too.
SKIPPED_CATEGORIES = [
  'chromeos-base', # TODO: this shouldn't be excluded.
  'virtual',
]

SKIPPED_PACKAGES = [
  # Fix these packages by adding a real license in the code.
  'chromeos-base/madison-cromo-plugin-0.1-r40',
  'dev-python/unittest2',

  # These are Chrome-OS-specific packages.
  'dev-util/hdctools',
  'media-libs/libresample',
  'sys-apps/rootdev',
  'sys-kernel/chromeos-kernel',  # already manually credit Linux
  'sys-apps/flashmap',

  'www-plugins/adobe-flash',

  # These have been split across several packages, so we skip listing the
  # individual components (and just list the main package instead).
  'app-editors/vim-core',
  'x11-apps/mesa-progs',

  # Portage metapackage.
  'x11-base/xorg-drivers',

  # Written by google;
  'dev-db/leveldb',

  # These are covered by app-i18n/ibus-mozc (BSD, copyright Google).
  'app-i18n/ibus-mozc',
  'app-i18n/ibus-mozc-chewing',
  'app-i18n/ibus-mozc-hangul',
  'app-i18n/ibus-mozc-pinyin',

  # These are all X.org sub-packages; shouldn't be any need to list them
  # individually.
  'media-fonts/encodings',
  'x11-apps/iceauth',
  'x11-apps/intel-gpu-tools',
  'x11-apps/mkfontdir',
  'x11-apps/rgb',
  'x11-apps/setxkbmap',
  'x11-apps/xauth',
  'x11-apps/xcursorgen',
  'x11-apps/xdpyinfo',
  'x11-apps/xdriinfo',
  'x11-apps/xev',
  'x11-apps/xgamma',
  'x11-apps/xhost',
  'x11-apps/xinit',
  'x11-apps/xinput',
  'x11-apps/xkbcomp',
  'x11-apps/xlsatoms',
  'x11-apps/xlsclients',
  'x11-apps/xmodmap',
  'x11-apps/xprop',
  'x11-apps/xrandr',
  'x11-apps/xrdb',
  'x11-apps/xset',
  'x11-apps/xset-mini',
  'x11-apps/xwininfo',
  'x11-base/xorg-server',
  'x11-drivers/xf86-input-evdev',
  'x11-drivers/xf86-input-keyboard',
  'x11-drivers/xf86-input-mouse',
  'x11-drivers/xf86-input-synaptics',
  'x11-drivers/xf86-video-intel',
  'x11-drivers/xf86-video-vesa',
  'x11-drivers/xf86-video-vmware',
  'x11-libs/libICE',
  'x11-libs/libSM',
  'x11-libs/libX11',
  'x11-libs/libXScrnSaver',
  'x11-libs/libXau',
  'x11-libs/libXcomposite',
  'x11-libs/libXcursor',
  'x11-libs/libXdamage',
  'x11-libs/libXdmcp',
  'x11-libs/libXext',
  'x11-libs/libXfixes',
  'x11-libs/libXfont',
  'x11-libs/libXfontcache',
  'x11-libs/libXft',
  'x11-libs/libXi',
  'x11-libs/libXinerama',
  'x11-libs/libXmu',
  'x11-libs/libXp',
  'x11-libs/libXrandr',
  'x11-libs/libXrender',
  'x11-libs/libXres',
  'x11-libs/libXt',
  'x11-libs/libXtst',
  'x11-libs/libXv',
  'x11-libs/libXvMC',
  'x11-libs/libXxf86vm',
  'x11-libs/libdrm',
  'x11-libs/libfontenc',
  'x11-libs/libpciaccess',
  'x11-libs/libxkbfile',
  'x11-libs/libxkbui',
  'x11-libs/pixman',
  'x11-libs/xtrans',
  'x11-misc/util-macros',
  'x11-misc/xbitmaps',
  'x11-proto/bigreqsproto',
  'x11-proto/compositeproto',
  'x11-proto/damageproto',
  'x11-proto/dri2proto',
  'x11-proto/fixesproto',
  'x11-proto/fontcacheproto',
  'x11-proto/fontsproto',
  'x11-proto/inputproto',
  'x11-proto/kbproto',
  'x11-proto/printproto',
  'x11-proto/randrproto',
  'x11-proto/recordproto',
  'x11-proto/renderproto',
  'x11-proto/resourceproto',
  'x11-proto/scrnsaverproto',
  'x11-proto/trapproto',
  'x11-proto/videoproto',
  'x11-proto/xcmiscproto',
  'x11-proto/xextproto',
  'x11-proto/xf86bigfontproto',
  'x11-proto/xf86dgaproto',
  'x11-proto/xf86driproto',
  'x11-proto/xf86rushproto',
  'x11-proto/xf86vidmodeproto',
  'x11-proto/xineramaproto',
  'x11-proto/xproto',
]

ARCHIVE_SUFFIXES = [ '.tar.gz', '.tgz', '.tar.bz2', '.tbz2' ]

LICENSE_FILENAMES = [
  'COPYING',
  'COPYRIGHT',    # used by strace
  'LICENCE',      # used by openssh
  'LICENSE',
  'LICENSE.txt',  # used by NumPy, glew
  'LICENSE.TXT',  # used by hdparm
  'License',      # used by libcap
  'copyright',
  'IPA_Font_License_Agreement_v1.0.txt',  # used by ja-ipafonts
]

SKIPPED_LICENSE_FILENAME_COMPONENTS = [
  'third_party'
]

PACKAGE_LICENSES = {
  'app-admin/eselect-opengl': ['GPL-2'],
  'app-crypt/nss': ['MPL-1.1'],
  'app-editors/gentoo-editor': ['MIT-gentoo-editor'],
  'app-editors/vim': ['vim'],
  'app-i18n/ibus-mozc': ['BSD-Google'],
  'app-i18n/ibus-mozc-pinyin': ['BSD-google'],
  'dev-db/sqlite': ['sqlite'],
  'dev-libs/libevent': ['BSD-libevent'],
  'dev-libs/nspr': ['GPL-2'],
  'dev-libs/nss': ['GPL-2'],
  'dev-libs/protobuf': ['BSD-Google'],
  'dev-python/netifaces': ['netiface'],
  'dev-util/bsdiff': ['BSD-bsdiff'],
  'dev-util/quipper': ['BSD-Google'],
  'media-fonts/font-util': ['font-util'],  # COPYING file from git repo
  'media-libs/freeimage': ['GPL-2'],
  'media-libs/jpeg': ['jpeg'],
  'media-plugins/o3d': ['BSD-Google'],
  'net-dialup/ppp': ['ppp-2.4.4'],
  'net-dns/c-ares': ['MIT-MIT'],
  'net-misc/dhcpcd': ['BSD-dhcpcd'],
  'net-misc/iputils': ['BSD-iputils'],
  'net-wireless/wpa_supplicant': ['GPL-2'],
  'net-wireless/iwl1000-ucode': ['Intel-iwl1000'],
  'net-wireless/marvell_sd8787': ['Marvell'],
  'sys-apps/less': ['BSD-less'],
  'sys-libs/ncurses': ['ncurses'],
  'sys-libs/talloc': ['LGPL-3'],  # ebuild incorrectly says GPL-3
  'sys-libs/timezone-data': ['public-domain'],
  'sys-process/vixie-cron': ['vixie-cron'],
  'x11-drivers/xf86-input-cmt': ['BSD-Google'],
}

INVALID_STOCK_LICENSES = [
  'BSD',  # requires distribution of copyright notice
  'MIT',  # requires distribution of copyright notice
]

PACKAGE_HOMEPAGES = {
  'app-editors/vim': ['http://www.vim.org/'],
  'x11-proto/glproto': ['http://www.x.org/'],
}

TEMPLATE_FILE = 'about_credits.tmpl'
ENTRY_TEMPLATE_FILE = 'about_credits_entry.tmpl'

class PackageInfo:
  def __init__(self, category=None, name=None, version=None, revision=None):
    self.category = category
    self.name = name
    self.version = version
    if revision is not None:
      revision = str(revision).lstrip('r')
      if revision == '0':
        revision = None
    self.revision = revision
    self.description = None
    self.homepages = []
    self.license_names = []
    self.license_text = None

  # TODO, what is this doing?
  @property
  def cpvr(self):
    s = '%s-%s' % (self.fullname, self.version)
    if self.revision:
      s += '-r%s' % self.revision
    return s

  @property
  def fullname(self):
    return '%s/%s' % (self.category, self.name)

  def _RunEbuildPhases(self, path, *phases, **kwargs):
    """Receives something like:
      path = /mnt/host/source/src/
                 third_party/portage-stable/net-misc/rsync/rsync-3.0.8.ebuild
      phases = ['clean', 'fetch'] or ['unpack']."""

    logging.debug("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
    logging.debug('ebuild-%s | %s | %s', board, path, str(list(phases)))
    logging.debug("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
    subprocess.check_call(
      ['ebuild-%s' % board, path] + list(phases), **kwargs)

  def ExtractLicense(self):
    """Try to get a license from the package by unpacking it with ebuild
    and looking for license files in the unpacked tree.
    """

    # Some packages have hardcoded licenses and are generated in
    # GetPackageInfo, so we skip the license extraction and exit early.
    if self.fullname in PACKAGE_LICENSES:
      return False

    path = GetEbuildPath(board, self.cpvr)
    self._RunEbuildPhases(
      path, 'clean', 'fetch',
      stdout=open('/dev/null', 'wb'),
      stderr=subprocess.STDOUT)
    self._RunEbuildPhases(path, 'unpack')
    try:
      return self._ExtractLicense()
    finally:
      self._RunEbuildPhases(path, 'clean')

  def _ExtractLicense(self):
    """Scan the unpacked source code for what looks like license files
    as defined in LICENSE_FILENAMES.
    """
    p = subprocess.Popen(['portageq-%s' % board, 'envvar',
                      'PORTAGE_TMPDIR'], stdout=subprocess.PIPE)
    tmpdir = p.communicate()[0].strip()
    ret = p.wait()
    if ret != 0:
      raise AssertionError('exit code was not 0: got %s' % ret)

    # tmpdir gets something like /build/daisy/tmp/
    workdir = os.path.join(tmpdir, 'portage', self.cpvr, 'work')

    args = ['find', workdir + '/', '-maxdepth', '3',
        '-mindepth', '1', '-type', 'f']
    p = subprocess.Popen(args, stdout=subprocess.PIPE)
    files = p.communicate()[0].splitlines()
    ret = p.wait()
    if ret != 0:
      raise AssertionError('exit code was not 0: got %s' % ret)

    files = [x[len(workdir):].lstrip('/') for x in files]
    licenses = []
    for name in files:
      if os.path.basename(name) in LICENSE_FILENAMES and \
        (name.count('/') == 1 or (name.count('/') == 2 and
         name.split('/')[1] == 'doc')):
        has_skipped_component = False
        for comp in SKIPPED_LICENSE_FILENAME_COMPONENTS:
          if comp in name:
            has_skipped_component = True
            break
        if not has_skipped_component:
          licenses.append(name)

    if not licenses:
      logging.warn("%s: couldn't find license file in %s",
          self.fullname, workdir)
      return False

    logging.info('%s: using %s', self.fullname, ' '.join(licenses))
    license_file = licenses[0]
    self.license_text = open(os.path.join(workdir, license_file)).read()
    return True

  def GetStockLicense(self):
    if not self.license_names:
      logging.info('%s: no stock licenses from ebuild', self.fullname)
      return False

    logging.info('%s: using stock license %s',
        self.fullname, ','.join(self.license_names))

    license_texts = []
    for license_name in self.license_names:
      license_path = None
      for directory in STOCK_LICENSE_DIRS:
        path = '%s/%s' % (directory, license_name)
        if os.access(path, os.F_OK):
          license_path = path
          break
      if license_path:
        license_texts.append(open(license_path).read())
      else:
        # TODO: We should probably report failure if we're unable to
        # find one of the licenses from a dual-licensed package.
        logging.warning('%s: stock license %s does not exist',
            self.fullname, license_name)

    if not license_texts:
      logging.warning('%s: couldn\'t find any stock licenses', self.fullname)
      return False

    self.license_text = '\n'.join(license_texts)
    return True


def ListInstalledPackages(board):
  """Return a list of all packages installed for a particular board."""
  # FIXME(merlin): davidjames pointed out that this is
  # not the right way to get the package list as it does not apply
  # filters. This should change to ~/trunk/src/scripts/get_package_list
  args = [ EQUERY_BASE % board, 'list', '*' ]
  p = subprocess.Popen(args, stdout=subprocess.PIPE)
  return [s.strip() for s in p.stdout.readlines()]


def GetFakePackages():
  pkgs = []

  pkg = PackageInfo('x11-base', 'X.Org', '1.9.3')
  pkg.homepages = [ 'http://www.x.org/' ]
  pkg.license_names = [ 'X' ]
  pkgs.append(pkg)

  pkg = PackageInfo('sys-kernel', 'Linux', '2.6')
  pkg.homepages = [ 'http://www.kernel.org/' ]
  pkg.license_names = [ 'GPL-2' ]
  pkgs.append(pkg)

  for pkg in pkgs:
    pkg.GetStockLicense()
  return pkgs


def GetEbuildPath(board, name):
  p = subprocess.Popen(
    ['equery-%s' % board, 'which', name], stdout=subprocess.PIPE)
  stdout = p.communicate()[0]
  p.wait()
  path = stdout.strip()
  logging.debug("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
  logging.debug("equery-%s which %s", board, name)
  logging.debug("  -> %s", path)
  logging.debug("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
  if not path:
    raise AssertionError('GetEbuildPath for %s failed.\n'
                         'Is your tree clean? Delete /build/%s and rebuild' %
                         (name, board))
  return path


def GetPackageInfo(fullname):
  info = PackageInfo(*portage.versions.catpkgsplit(fullname))

  if info.category in SKIPPED_CATEGORIES or info.fullname in SKIPPED_PACKAGES:
    return None

  ebuild = GetEbuildPath(board, fullname)

  if not os.access(ebuild, os.F_OK):
    return info

  args = [
    'portageq',
    'metadata',
    '/build/%s' % board,
    'ebuild',
    fullname,
    'HOMEPAGE', 'LICENSE', 'DESCRIPTION',
  ]
  p = subprocess.Popen(args, stdout=subprocess.PIPE)
  lines = [s.strip() for s in p.stdout.readlines()]
  p.wait()
  if p.returncode != 0:
    return info

  if PACKAGE_HOMEPAGES.has_key(info.fullname):
    info.homepages = PACKAGE_HOMEPAGES[info.fullname]
  else:
    info.homepages = lines[0].split()

  if PACKAGE_LICENSES.has_key(info.fullname):
    licenses = PACKAGE_LICENSES[info.fullname]
  else:
    licenses = lines[1].split()

  info.license_names = []
  for license_name in licenses:
    if license_name in INVALID_STOCK_LICENSES:
      logging.info('%s: skipping invalid stock license %s',
                   info.fullname, license_name)
    else:
      info.license_names.append(license_name)

  info.description = '\n'.join(lines[2:])

  return info


def EvaluateTemplate(template, env, escape=True):
  """Expand a template with variables like {{foo}} using a
  dictionary of expansions."""
  for key, val in env.items():
    if escape:
      val = cgi.escape(val)
    template = template.replace('{{%s}}' % key, val)
  return template

def ProcessPkg(package):
  info = GetPackageInfo(package)
  if not info:
    return (package, None)
  if not info.ExtractLicense() and not info.GetStockLicense():
    raise RuntimeError("%s: unable to find license" % info.cpvr)
  return (package, info)


if __name__ == '__main__':
  singlethreaded = False
  debug = False
  if len(sys.argv) > 1 and sys.argv[1] == "--debug":
    singlethreaded = True
    debug = True
    sys.argv.pop(0)
    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
    logging.debug('>>> Debug enabled, using single threading <<<')
  else:
    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)


  if len(sys.argv) != 3:
    print >> sys.stderr, (__doc__)
    sys.exit(1)

  entry_template = open(ENTRY_TEMPLATE_FILE, 'rb').read()

  # We have a hardcoded list of skipped packages for various reasons, but we
  # also exclude any google platform package from needing a license since they
  # are covered by the top license in the tree.
  cmd = "cros_workon info --all --host | grep src/platform/ |"\
      "awk '{print $1}'"
  p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
  packages = p.communicate()[0].splitlines()
  ret = p.wait()
  if ret != 0:
    raise AssertionError('%s exit code was not 0: got %s' % (cmd, ret))
  SKIPPED_PACKAGES += packages

  board = sys.argv[1]
  packages = ListInstalledPackages(board)
  logging.debug("Package list to work through:")
  logging.debug("\n".join(packages))
  logging.debug("\n\nWill skip these packages:")
  logging.debug("\n".join(SKIPPED_PACKAGES))

  if singlethreaded:
    data = [ProcessPkg(x) for x in packages]
  else:
    data = multiprocessing.Pool().map(ProcessPkg, packages, 1)

  infos = [x[1] for x in data if x[1] is not None]

  infos += GetFakePackages()
  infos.sort(key=lambda x:(x.name, x.version, x.revision))

  entries = []
  seen_package_names = set()
  for info in infos:
    if info.name in seen_package_names:
      continue
    seen_package_names.add(info.name)
    env = {
      'name': info.name,
      'url': info.homepages[0] if info.homepages else '',
      'license': info.license_text or '',
    }
    entries.append(EvaluateTemplate(entry_template, env))

  file_template = open(TEMPLATE_FILE, 'rb').read()
  out_file = open(sys.argv[2], "w")
  out_file.write(EvaluateTemplate(file_template,
                                  { 'entries': '\n'.join(entries) },
                                  escape=False))