service/sysroot.py

# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Sysroot service."""

from __future__ import print_function

import os

from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import osutils
from chromite.lib import portage_util
from chromite.lib import sysroot_lib
from chromite.lib import workon_helper
from chromite.utils import metrics


class Error(Exception):
  """Base error class for the module."""


class InvalidArgumentsError(Error):
  """Invalid arguments."""


class NotInChrootError(Error):
  """When SetupBoard is run outside of the chroot."""


class UpdateChrootError(Error):
  """Error occurred when running update chroot."""


class SetupBoardRunConfig(object):
  """Value object for full setup board run configurations."""

  def __init__(self, set_default=False, force=False, usepkg=True, jobs=None,
               regen_configs=False, quiet=False, update_toolchain=True,
               upgrade_chroot=True, init_board_pkgs=True, local_build=False,
               toolchain_changed=False, package_indexes=None):
    """Initialize method.

    Args:
      set_default (bool): Whether to set the passed board as the default.
      force (bool): Force a new sysroot creation when it already exists.
      usepkg (bool): Whether to use binary packages to bootstrap.
      jobs (int): Max number of simultaneous packages to build.
      regen_configs (bool): Whether to only regen the configs.
      quiet (bool): Whether to print notification when sysroot exists.
      update_toolchain (bool): Update the toolchain?
      upgrade_chroot (bool): Upgrade the chroot before building?
      init_board_pkgs (bool): Emerging packages to sysroot?
      local_build (bool): Bootstrap only from local packages?
      toolchain_changed (bool): Has a toolchain change occurred? Implies
        'force'.
      package_indexes (list[PackageIndexInfo]): List of information about
        available prebuilts, youngest first, or None.
    """

    self.set_default = set_default
    self.force = force or toolchain_changed
    self.usepkg = usepkg
    self.jobs = jobs
    self.regen_configs = regen_configs
    self.quiet = quiet
    self.update_toolchain = update_toolchain
    self.update_chroot = upgrade_chroot
    self.init_board_pkgs = init_board_pkgs
    self.local_build = local_build
    self.package_indexes = package_indexes or []

  def GetUpdateChrootArgs(self):
    """Create a list containing the relevant update_chroot arguments.

    Returns:
      list[str]
    """
    args = []
    if self.usepkg:
      args += ['--usepkg']
    else:
      args += ['--nousepkg']

    if self.jobs:
      args += ['--jobs', str(self.jobs)]

    if not self.update_toolchain:
      args += ['--skip_toolchain_update']

    return args


class BuildPackagesRunConfig(object):
  """Value object to hold build packages run configs."""

  def __init__(self, usepkg=True, install_debug_symbols=False,
               packages=None, use_flags=None, use_goma=False,
               incremental_build=True, package_indexes=None):
    """Init method.

    Args:
      usepkg (bool): Whether to use binpkgs or build from source. False
        currently triggers a local build, which will enable local reuse.
      install_debug_symbols (bool): Whether to include the debug symbols for all
        packages.
      packages (list[str]|None): The list of packages to install, by default
        install all packages for the target.
      use_flags (list[str]|None): A list of use flags to set.
      use_goma (bool): Whether to enable goma.
      incremental_build (bool): Whether to treat the build as an incremental
        build or a fresh build. Always treating it as an incremental build is
        safe, but certain operations can be faster when we know we are doing
        a fresh build.
      package_indexes (list[PackageIndexInfo]): List of information about
        available prebuilts, youngest first, or None.
    """
    self.usepkg = usepkg
    self.install_debug_symbols = install_debug_symbols
    self.packages = packages
    self.use_flags = use_flags
    self.use_goma = use_goma
    self.is_incremental = incremental_build
    self.package_indexes = package_indexes or []

  def GetBuildPackagesArgs(self):
    """Get the build_packages script arguments."""
    # Defaults for the builder.
    # TODO(saklein): Parametrize/rework the defaults when build_packages is
    #   ported to chromite.
    args = [
        '--accept_licenses',
        '@CHROMEOS',
        '--skip_chroot_upgrade',
        '--nouse_any_chrome',
    ]

    if not self.usepkg:
      args.append('--nousepkg')

    if self.install_debug_symbols:
      args.append('--withdebugsymbols')

    if self.use_goma:
      args.append('--run_goma')

    if not self.is_incremental:
      args.append('--nowithrevdeps')

    if self.packages:
      args.extend(self.packages)

    return args

  def HasUseFlags(self):
    """Check if we have use flags."""
    return bool(self.use_flags)

  def GetUseFlags(self):
    """Get the use flags as a single string."""
    use_flags = self.use_flags
    if use_flags:
      # We have use flags to set, but we need to append them to any existing
      # use flags rather than overwrite them completely.
      # TODO(saklein) Add config for whether to extend or overwrite?
      existing_flags = os.environ.get('USE', '').split()
      existing_flags.extend(use_flags)
      use_flags = existing_flags

      return ' '.join(use_flags)

    return None

  def GetEnv(self):
    """Get the env from this config."""
    env = {}
    if self.HasUseFlags():
      env['USE'] = self.GetUseFlags()

    if self.use_goma:
      env['USE_GOMA'] = 'true'

    if self.package_indexes:
      env['PORTAGE_BINHOST'] = ' '.join(
          x.location for x in reversed(self.package_indexes))

    return env


def SetupBoard(target, accept_licenses=None, run_configs=None):
  """Run the full process to setup a board's sysroot.

  This is the entry point to run the setup_board script.

  Args:
    target (build_target_lib.BuildTarget): The build target configuration.
    accept_licenses (str|None): The additional licenses to accept.
    run_configs (SetupBoardRunConfig): The run configs.

  Raises:
    sysroot_lib.ToolchainInstallError when the toolchain fails to install.
  """
  if not cros_build_lib.IsInsideChroot():
    # TODO(saklein) switch to build out command and run inside chroot.
    raise NotInChrootError('SetupBoard must be run from inside the chroot')

  # Make sure we have valid run configs setup.
  run_configs = run_configs or SetupBoardRunConfig()

  sysroot = Create(target, run_configs, accept_licenses)

  if run_configs.regen_configs:
    # We're now done if we're only regenerating the configs.
    return

  InstallToolchain(target, sysroot, run_configs)


def Create(target, run_configs, accept_licenses):
  """Create a sysroot.

  This entry point is the subset of the full setup process that does the
  creation and configuration of a sysroot, including installing portage.

  Args:
    target (build_target.BuildTarget): The build target being installed in the
      sysroot being created.
    run_configs (SetupBoardRunConfig): The run configs.
    accept_licenses (str|None): The additional licenses to accept.
  """
  cros_build_lib.AssertInsideChroot()

  sysroot = sysroot_lib.Sysroot(target.root)

  if sysroot.Exists() and not run_configs.force and not run_configs.quiet:
    logging.warning('Board output directory already exists: %s\n'
                    'Use --force to clobber the board root and start again.',
                    sysroot.path)

  # Override regen_configs setting to force full setup run if the sysroot does
  # not exist.
  run_configs.regen_configs = run_configs.regen_configs and sysroot.Exists()

  # Make sure the chroot is fully up to date before we start unless the
  # chroot update is explicitly disabled.
  if run_configs.update_chroot:
    logging.info('Updating chroot.')
    update_chroot = [os.path.join(constants.CROSUTILS_DIR, 'update_chroot'),
                     '--toolchain_boards', target.name]
    update_chroot += run_configs.GetUpdateChrootArgs()
    try:
      cros_build_lib.run(update_chroot)
    except cros_build_lib.RunCommandError:
      raise UpdateChrootError('Error occurred while updating the chroot.'
                              'See the logs for more information.')

  # Delete old sysroot to force a fresh start if requested.
  if sysroot.Exists() and run_configs.force:
    sysroot.Delete(background=True)

  # Step 1: Create folders.
  # Dependencies: None.
  # Create the skeleton.
  logging.info('Creating sysroot directories.')
  _CreateSysrootSkeleton(sysroot)

  # Step 2: Standalone configurations.
  # Dependencies: Folders exist.
  # Install main, board setup, and user make.conf files.
  logging.info('Installing configurations into sysroot.')
  _InstallConfigs(sysroot, target)

  # Step 3: Portage configurations.
  # Dependencies: make.conf.board_setup.
  # Create the command wrappers, choose profile, and make.conf.board.
  # Refresh the workon symlinks to compensate for crbug.com/679831.
  logging.info('Setting up portage in the sysroot.')
  _InstallPortageConfigs(sysroot, target, accept_licenses,
                         run_configs.local_build,
                         package_indexes=run_configs.package_indexes)

  # Developer Experience Step: Set default board (if requested) to allow
  # running later commands without needing to pass the --board argument.
  if run_configs.set_default:
    cros_build_lib.SetDefaultBoard(target.name)

  return sysroot


def GenerateArchive(output_dir, build_target_name, pkg_list):
  """Generate a sysroot tarball for informational builders.

  Args:
    output_dir (string): Directory to contain the created the sysroot.
    build_target_name (string): The build target for the sysroot being created.
    pkg_list (list[string]|None): List of 'category/package' package strings.

  Returns:
    Path to the sysroot tar file.
  """
  cmd = ['cros_generate_sysroot',
         '--out-file', constants.TARGET_SYSROOT_TAR,
         '--out-dir', output_dir,
         '--board', build_target_name,
         '--package', ' '.join(pkg_list)]
  cros_build_lib.run(cmd, cwd=constants.SOURCE_ROOT)
  return os.path.join(output_dir, constants.TARGET_SYSROOT_TAR)

def CreateSimpleChromeSysroot(target, use_flags):
  """Create a sysroot for SimpleChrome to use.

  Args:
    target (build_target.BuildTarget): The build target being installed for the
      sysroot being created.
    use_flags (list[string]|None): Additional USE flags for building chrome.

  Returns:
    Path to the sysroot tar file.
  """
  extra_env = {}
  if use_flags:
    extra_env['USE'] = ' '.join(use_flags)
  with osutils.TempDir(delete=False) as tempdir:
    cmd = ['cros_generate_sysroot', '--out-dir', tempdir, '--board',
           target, '--deps-only', '--package', constants.CHROME_CP]
    cros_build_lib.run(cmd, cwd=constants.SOURCE_ROOT, enter_chroot=True,
                       extra_env=extra_env)
    sysroot_tar_path = os.path.join(tempdir, constants.CHROME_SYSROOT_TAR)
    return sysroot_tar_path

def InstallToolchain(target, sysroot, run_configs):
  """Update the toolchain to a sysroot.

  This entry point just installs the target's toolchain into the sysroot.
  Everything else must have been done already for this to be successful.

  Args:
    target (build_target_lib.BuildTarget): The target whose toolchain is being
      installed.
    sysroot (sysroot_lib.Sysroot): The sysroot where the toolchain is being
      installed.
    run_configs (SetupBoardRunConfig): The run configs.
  """
  cros_build_lib.AssertInsideChroot()
  if not sysroot.Exists():
    # Sanity check before we try installing anything.
    raise ValueError('The sysroot must exist, run Create first.')

  # Step 4: Install toolchain and packages.
  # Dependencies: Portage configs and wrappers have been installed.
  if run_configs.init_board_pkgs:
    logging.info('Updating toolchain.')
    # Use the local packages if we're doing a local only build or usepkg is set.
    local_init = run_configs.usepkg or run_configs.local_build
    _InstallToolchain(sysroot, target, local_init=local_init)


def BuildPackages(target, sysroot, run_configs):
  """Build and install packages into a sysroot.

  Args:
    target (build_target_lib.BuildTarget): The target whose packages are being
      installed.
    sysroot (sysroot_lib.Sysroot): The sysroot where the packages are being
      installed.
    run_configs (BuildPackagesRunConfig): The run configs.
  """
  cros_build_lib.AssertInsideChroot()

  cmd = [os.path.join(constants.CROSUTILS_DIR, 'build_packages'),
         '--board', target.name, '--board_root', sysroot.path]
  cmd += run_configs.GetBuildPackagesArgs()

  extra_env = run_configs.GetEnv()
  extra_env['USE_NEW_PARALLEL_EMERGE'] = '1'
  with osutils.TempDir() as tempdir:
    extra_env[constants.CROS_METRICS_DIR_ENVVAR] = tempdir

    try:
      # REVIEW: discuss which dimensions to flatten into the metric
      # name other than target.name...
      with metrics.timer('service.sysroot.BuildPackages.RunCommand'):
        cros_build_lib.run(cmd, extra_env=extra_env)
    except cros_build_lib.RunCommandError as e:
      failed_pkgs = portage_util.ParseDieHookStatusFile(tempdir)
      raise sysroot_lib.PackageInstallError(
          str(e), e.result, exception=e, packages=failed_pkgs)


def _CreateSysrootSkeleton(sysroot):
  """Create the sysroot skeleton.

  Dependencies: None.
  Creates the sysroot directory structure and installs the portage hooks.

  Args:
    sysroot (sysroot_lib.Sysroot): The sysroot.
  """
  sysroot.CreateSkeleton()


def _InstallConfigs(sysroot, target):
  """Install standalone configuration files into the sysroot.

  Dependencies: The sysroot exists (i.e. CreateSysrootSkeleton).
  Installs the main, board setup, and user make.conf files.

  Args:
    sysroot (sysroot_lib.Sysroot): The sysroot.
    target (build_target.BuildTarget): The build target being setup in
      the sysroot.
  """
  sysroot.InstallMakeConf()
  sysroot.InstallMakeConfBoardSetup(target.name)
  sysroot.InstallMakeConfUser()


def _InstallPortageConfigs(sysroot, target, accept_licenses, local_build,
                           package_indexes=None):
  """Install portage wrappers and configurations.

  Dependencies: make.conf.board_setup (InstallConfigs).
  Create the command wrappers, choose profile, and generate make.conf.board.
  Refresh the workon symlinks to compensate for crbug.com/679831.

  Args:
    sysroot (sysroot_lib.Sysroot): The sysroot.
    target (build_target.BuildTarget): The build target being installed
      in the sysroot.
    accept_licenses (str): Additional accepted licenses as a string.
    local_build (bool): If the build is a local only build.
    package_indexes (list[PackageIndexInfo]): List of information about
      available prebuilts, youngest first, or None.
  """
  sysroot.CreateAllWrappers(friendly_name=target.name)
  _ChooseProfile(target, sysroot)
  _RefreshWorkonSymlinks(target.name, sysroot)
  # Must be done after the profile is chosen or binhosts may be incomplete.
  sysroot.InstallMakeConfBoard(accepted_licenses=accept_licenses,
                               local_only=local_build,
                               package_indexes=package_indexes)


def _InstallToolchain(sysroot, target, local_init=True):
  """Install toolchain and packages.

  Dependencies: Portage configs and wrappers have been installed
    (InstallPortageConfigs).
  Install the toolchain and the implicit dependencies.

  Args:
    sysroot (sysroot_lib.Sysroot): The sysroot to install to.
    target (build_target.BuildTarget): The build target whose toolchain is
      being installed.
    local_init (bool): Whether to use local packages to bootstrap implicit
      dependencies.
  """
  sysroot.UpdateToolchain(target.name, local_init=local_init)


def _RefreshWorkonSymlinks(target, sysroot):
  """Force refresh the workon symlinks.

  Create an instance of the WorkonHelper, which will recreate all symlinks
  to masked/unmasked packages currently worked on in case the sysroot was
  recreated (crbug.com/679831).

  This was done with a call to `cros_workon list` in the bash version of
  the script, but all we actually need is for the WorkonHelper to be
  instantiated since it refreshes the symlinks in its __init__.

  Args:
    target (str): The build target name.
    sysroot (sysroot_lib.Sysroot): The board's sysroot.
  """
  workon_helper.WorkonHelper(sysroot.path, friendly_name=target)


def _ChooseProfile(target, sysroot):
  """Helper function to execute cros_choose_profile.

  TODO(saklein) Refactor cros_choose_profile to avoid needing the run
  call here, and by extension this method all together.

  Args:
    target (build_target_lib.BuildTarget): The build target whose profile is
      being chosen.
    sysroot (sysroot_lib.Sysroot): The sysroot for which the profile is
      being chosen.
  """
  choose_profile = ['cros_choose_profile', '--board', target.name,
                    '--board-root', sysroot.path]
  if target.profile:
    # Chooses base by default, only override when we have a passed param.
    choose_profile += ['--profile', target.profile]
  try:
    cros_build_lib.run(choose_profile, print_cmd=False)
  except cros_build_lib.RunCommandError as e:
    logging.error('Selecting profile failed, removing incomplete board '
                  'directory!')
    sysroot.Delete()
    raise e