blob: f483a056a0db9bfaad01ab945fe4038f5c4b778a [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Sysroot service."""
from __future__ import print_function
import os
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import osutils
from chromite.lib import portage_util
from chromite.lib import sysroot_lib
from chromite.lib import workon_helper
from chromite.utils import metrics
class Error(Exception):
"""Base error class for the module."""
class InvalidArgumentsError(Error):
"""Invalid arguments."""
class NotInChrootError(Error):
"""When SetupBoard is run outside of the chroot."""
class UpdateChrootError(Error):
"""Error occurred when running update chroot."""
class SetupBoardRunConfig(object):
"""Value object for full setup board run configurations."""
def __init__(self, set_default=False, force=False, usepkg=True, jobs=None,
regen_configs=False, quiet=False, update_toolchain=True,
upgrade_chroot=True, init_board_pkgs=True, local_build=False):
"""Initialize method.
set_default (bool): Whether to set the passed board as the default.
force (bool): Force a new sysroot creation when it already exists.
usepkg (bool): Whether to use binary packages to bootstrap.
jobs (int): Max number of simultaneous packages to build.
regen_configs (bool): Whether to only regen the configs.
quiet (bool): Whether to print notification when sysroot exists.
update_toolchain (bool): Update the toolchain?
upgrade_chroot (bool): Upgrade the chroot before building?
init_board_pkgs (bool): Emerging packages to sysroot?
local_build (bool): Bootstrap only from local packages?
self.set_default = set_default
self.force = force
self.usepkg = usepkg = jobs
self.regen_configs = regen_configs
self.quiet = quiet
self.update_toolchain = update_toolchain
self.update_chroot = upgrade_chroot
self.init_board_pkgs = init_board_pkgs
self.local_build = local_build
def GetUpdateChrootArgs(self):
"""Create a list containing the relevant update_chroot arguments.
args = []
if self.usepkg:
args += ['--usepkg']
args += ['--nousepkg']
if > 0:
args += ['--jobs', str(]
if not self.update_toolchain:
args += ['--skip_toolchain_update']
return args
class BuildPackagesRunConfig(object):
"""Value object to hold build packages run configs."""
def __init__(self, event_file=None, usepkg=True, install_debug_symbols=False,
packages=None, use_flags=None):
"""Init method.
event_file (str): The event file location, enables events.
usepkg (bool): Whether to use binpkgs or build from source. False
currently triggers a local build, which will enable local reuse.
install_debug_symbols (bool): Whether to include the debug symbols for all
packages (list[str]|None): The list of packages to install, by default
install all packages for the target.
use_flags (list[str]|None): A list of use flags to set.
self.event_file = event_file
self.usepkg = usepkg
self.install_debug_symbols = install_debug_symbols
self.packages = packages
self.use_flags = use_flags
def GetBuildPackagesArgs(self):
"""Get the build_packages script arguments."""
# Defaults for the builder.
# TODO(saklein): Parametrize/rework the defaults when build_packages is
# ported to chromite.
args = ['--accept_licenses', '@CHROMEOS', '--skip_chroot_upgrade']
if self.event_file:
args.extend(['--eventfile', self.event_file])
if not self.usepkg:
if self.install_debug_symbols:
if self.packages:
return args
def HasUseFlags(self):
"""Check if we have use flags."""
return len(self.use_flags) > 0
def GetUseFlags(self):
"""Get the use flags as a single string."""
use_flags = self.use_flags
if use_flags:
# We have use flags to set, but we need to append them to any existing
# use flags rather than overwrite them completely.
# TODO(saklein) Add config for whether to extend or overwrite?
existing_flags = os.environ.get('USE', '').split()
use_flags = existing_flags
return ' '.join(use_flags)
return None
def SetupBoard(target, accept_licenses=None, run_configs=None):
"""Run the full process to setup a board's sysroot.
This is the entry point to run the setup_board script.
target (build_target_util.BuildTarget): The build target configuration.
accept_licenses (str|None): The additional licenses to accept.
run_configs (SetupBoardRunConfig): The run configs.
sysroot_lib.ToolchainInstallError when the toolchain fails to install.
if not cros_build_lib.IsInsideChroot():
# TODO(saklein) switch to build out command and run inside chroot.
raise NotInChrootError('SetupBoard must be run from inside the chroot')
# Make sure we have valid run configs setup.
run_configs = run_configs or SetupBoardRunConfig()
sysroot = Create(target, run_configs, accept_licenses)
if run_configs.regen_configs:
# We're now done if we're only regenerating the configs.
InstallToolchain(target, sysroot, run_configs)
def Create(target, run_configs, accept_licenses):
"""Create a sysroot.
This entry point is the subset of the full setup process that does the
creation and configuration of a sysroot, including installing portage.
target (build_target.BuildTarget): The build target being installed in the
sysroot being created.
run_configs (SetupBoardRunConfig): The run configs.
accept_licenses (str|None): The additional licenses to accept.
sysroot = sysroot_lib.Sysroot(target.root)
if sysroot.Exists() and not run_configs.force and not run_configs.quiet:
logging.warning('Board output directory already exists: %s\n'
'Use --force to clobber the board root and start again.',
# Override regen_configs setting to force full setup run if the sysroot does
# not exist.
run_configs.regen_configs = run_configs.regen_configs and sysroot.Exists()
# Make sure the chroot is fully up to date before we start unless the
# chroot update is explicitly disabled.
if run_configs.update_chroot:'Updating chroot.')
update_chroot = [os.path.join(constants.CROSUTILS_DIR, 'update_chroot'),
update_chroot += run_configs.GetUpdateChrootArgs()
except cros_build_lib.RunCommandError:
raise UpdateChrootError('Error occurred while updating the chroot.'
'See the logs for more information.')
# Delete old sysroot to force a fresh start if requested.
if sysroot.Exists() and run_configs.force:
# Step 1: Create folders.
# Dependencies: None.
# Create the skeleton.'Creating sysroot directories.')
# Step 2: Standalone configurations.
# Dependencies: Folders exist.
# Install main, board setup, and user make.conf files.'Installing configurations into sysroot.')
_InstallConfigs(sysroot, target)
# Step 3: Portage configurations.
# Dependencies: make.conf.board_setup.
# Create the command wrappers, choose profile, and make.conf.board.
# Refresh the workon symlinks to compensate for'Setting up portage in the sysroot.')
_InstallPortageConfigs(sysroot, target, accept_licenses,
# Developer Experience Step: Set default board (if requested) to allow
# running later commands without needing to pass the --board argument.
if run_configs.set_default:
return sysroot
def InstallToolchain(target, sysroot, run_configs):
"""Update the toolchain to a sysroot.
This entry point just installs the target's toolchain into the sysroot.
Everything else must have been done already for this to be successful.
target (build_target_util.BuildTarget): The target whose toolchain is being
sysroot (sysroot_lib.Sysroot): The sysroot where the toolchain is being
run_configs (SetupBoardRunConfig): The run configs.
if not sysroot.Exists():
# Sanity check before we try installing anything.
raise ValueError('The sysroot must exist, run Create first.')
# Step 4: Install toolchain and packages.
# Dependencies: Portage configs and wrappers have been installed.
if run_configs.init_board_pkgs:'Updating toolchain.')
# Use the local packages if we're doing a local only build or usepkg is set.
local_init = run_configs.usepkg or run_configs.local_build
_InstallToolchain(sysroot, target, local_init=local_init)
def BuildPackages(target, sysroot, run_configs):
"""Build and install packages into a sysroot.
target (build_target_util.BuildTarget): The target whose packages are being
sysroot (sysroot_lib.Sysroot): The sysroot where the packages are being
run_configs (BuildPackagesRunConfig): The run configs.
cmd = [os.path.join(constants.CROSUTILS_DIR, 'build_packages'),
'--board',, '--board_root', sysroot.path]
cmd += run_configs.GetBuildPackagesArgs()
with osutils.TempDir(base_dir='/tmp') as tempdir:
extra_env = {
constants.CROS_METRICS_DIR_ENVVAR: tempdir,
if run_configs.use_flags:
extra_env['USE'] = run_configs.GetUseFlags()
# REVIEW: discuss which dimensions to flatten into the metric
# name other than
with metrics.timer('service.sysroot.BuildPackages.RunCommand'):
cros_build_lib.RunCommand(cmd, extra_env=extra_env)
except cros_build_lib.RunCommandError as e:
failed_pkgs = portage_util.ParseDieHookStatusFile(tempdir)
raise sysroot_lib.PackageInstallError(
e.message, e.result, exception=e, packages=failed_pkgs)
def _CreateSysrootSkeleton(sysroot):
"""Create the sysroot skeleton.
Dependencies: None.
Creates the sysroot directory structure and installs the portage hooks.
sysroot (sysroot_lib.Sysroot): The sysroot.
def _InstallConfigs(sysroot, target):
"""Install standalone configuration files into the sysroot.
Dependencies: The sysroot exists (i.e. CreateSysrootSkeleton).
Installs the main, board setup, and user make.conf files.
sysroot (sysroot_lib.Sysroot): The sysroot.
target (build_target.BuildTarget): The build target being setup in
the sysroot.
def _InstallPortageConfigs(sysroot, target, accept_licenses, local_build):
"""Install portage wrappers and configurations.
Dependencies: make.conf.board_setup (InstallConfigs).
Create the command wrappers, choose profile, and generate make.conf.board.
Refresh the workon symlinks to compensate for
sysroot (sysroot_lib.Sysroot): The sysroot.
target (build_target.BuildTarget): The build target being installed
in the sysroot.
accept_licenses (str): Additional accepted licenses as a string.
local_build (bool): If the build is a local only build.
_ChooseProfile(target, sysroot)
_RefreshWorkonSymlinks(, sysroot)
# Must be done after the profile is chosen or binhosts may be incomplete.
def _InstallToolchain(sysroot, target, local_init=True):
"""Install toolchain and packages.
Dependencies: Portage configs and wrappers have been installed
Install the toolchain and the implicit dependencies.
sysroot (sysroot_lib.Sysroot): The sysroot to install to.
target (build_target.BuildTarget): The build target whose toolchain is
being installed.
local_init (bool): Whether to use local packages to bootstrap implicit
sysroot.UpdateToolchain(, local_init=local_init)
def _RefreshWorkonSymlinks(target, sysroot):
"""Force refresh the workon symlinks.
Create an instance of the WorkonHelper, which will recreate all symlinks
to masked/unmasked packages currently worked on in case the sysroot was
recreated (
This was done with a call to `cros_workon list` in the bash version of
the script, but all we actually need is for the WorkonHelper to be
instantiated since it refreshes the symlinks in its __init__.
target (str): The build target name.
sysroot (sysroot_lib.Sysroot): The board's sysroot.
workon_helper.WorkonHelper(sysroot.path, friendly_name=target)
def _ChooseProfile(target, sysroot):
"""Helper function to execute cros_choose_profile.
TODO(saklein) Refactor cros_choose_profile to avoid needing the RunCommand
call here, and by extension this method all together.
target (build_target_util.BuildTarget): The build target whose profile is
being chosen.
sysroot (sysroot_lib.Sysroot): The sysroot for which the profile is
being chosen.
choose_profile = ['cros_choose_profile', '--board',,
'--board-root', sysroot.path]
if target.profile:
# Chooses base by default, only override when we have a passed param.
choose_profile += ['--profile', target.profile]
cros_build_lib.RunCommand(choose_profile, print_cmd=False)
except cros_build_lib.RunCommandError as e:
logging.error('Selecting profile failed, removing incomplete board '
raise e