blob: 9273360860ac17cd67e97361d3337c4b238eb44d [file] [log] [blame]
# Copyright 2023 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""cros build-packages updates the set of binary packages needed by ChromiumOS.
The build-packages process cross compiles all packages that have been
updated into the given sysroot and builds binary packages as a side-effect.
The output packages will be used by `cros build-image` to create a bootable
ChromiumOS image.
If packages are specified in the command line, only build those specific
packages and any dependencies they might need.
"""
import argparse
import logging
import os
import urllib.error
import urllib.request
from chromite.cli import command
from chromite.lib import build_target_lib
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import sysroot_lib
from chromite.lib.telemetry import trace
from chromite.service import sysroot
from chromite.utils import timer
tracer = trace.get_tracer(__name__)
@command.command_decorator("build-packages")
class BuildPackagesCommand(command.CliCommand):
"""Update the set of binary packages used by ChromiumOS."""
use_telemetry = True
@classmethod
def AddParser(cls, parser: commandline.ArgumentParser):
"""Build the parser.
Args:
parser: The argument parser.
"""
super().AddParser(parser)
parser.add_argument(
"-b",
"--board",
"--build-target",
dest="board",
default=cros_build_lib.GetDefaultBoard(),
help="The board to build packages for.",
)
parser.add_argument(
"--profile",
help="The portage configuration profile to use. Profile "
"must be located in overlay-board/profiles.",
)
parser.add_bool_argument(
"--usepkg",
True,
"Use binary packages when available.",
"Don't use binary packages.",
)
parser.add_bool_argument(
"--usepkgonly",
False,
"Only use binary packages; abort if any are missing.",
"Prefer binary packages, but build from source if needed.",
)
parser.add_bool_argument(
"--workon",
True,
"Force-build workon packages.",
"Don't force-build workon packages.",
)
parser.add_bool_argument(
"--withrevdeps",
True,
"Calculate reverse dependencies on changed ebuilds.",
"Don't calculate reverse dependencies.",
)
parser.add_bool_argument(
"--cleanbuild",
False,
"Delete sysroot if it exists before building.",
"Re-use existing sysroot state.",
)
parser.add_bool_argument(
"--pretend",
False,
"Only display which packages would be installed.",
"Build packages like normal.",
)
# The --sysroot flag specifies the environment variables ROOT and
# PKGDIR. This allows fetching and emerging of all packages to
# specified sysroot. Note that --sysroot will setup the board normally
# in /build/$BOARD, if it's not setup yet. It also expects the toolchain
# to already be installed in the sysroot.
# --usepkgonly and --norebuild are required, because building is not
# supported when board_root is set.
parser.add_argument(
"--sysroot", type="str_path", help="Emerge packages to sysroot."
)
# CPU Governor related options.
group = parser.add_argument_group("CPU Governor Options")
group.add_bool_argument(
"--autosetgov",
False,
"Automatically set cpu governor to 'performance' when building.",
"Do not change the cpu governor.",
)
group.add_bool_argument(
"--autosetgov-sticky",
False,
"Remember --autosetgov setting for future runs.",
"Only change the cpu governor for this run.",
)
# Chrome building related options.
group = parser.add_argument_group(
"Chrome Options",
description="By default, the build will use any available "
"chromeos-chrome binpkg, and build Chromium if no usable binpkg "
"can be found. These options alter that behavior.",
)
exclusive_chrome_group = group.add_mutually_exclusive_group()
exclusive_chrome_group.add_argument(
"--chrome",
action="store_true",
help="Ensure Chrome is installed, building from source if "
"necessary.",
)
exclusive_chrome_group.add_argument(
"--chromium",
action="store_true",
help="Ensure Chromium is installed, building from source if "
"necessary.",
)
# Legacy Chrome arguments.
group.add_argument(
"--internal", action="store_true", help=argparse.SUPPRESS
)
group.add_argument(
"--no-internal",
"--nointernal",
dest="internal",
action="store_false",
help=argparse.SUPPRESS,
)
group.add_argument(
"--use-any-chrome",
action="store_true",
default=True,
help=argparse.SUPPRESS,
)
group.add_argument(
"--no-use-any-chrome",
"--nouse-any-chrome",
action="store_false",
help=argparse.SUPPRESS,
)
# Setup board related options.
group = parser.add_argument_group("Setup Board Config Options")
group.add_bool_argument(
"--skip-chroot-upgrade",
True,
"Skip the automatic chroot upgrade; use with care.",
"Upgrade the chroot first.",
)
group.add_bool_argument(
"--skip-toolchain-update",
False,
"Deprecated (flag is ignored if passed).",
"Deprecated (flag is ignored if passed).",
)
group.add_bool_argument(
"--skip-setup-board",
False,
"Skip running setup_board. Implies --skip-chroot-upgrade.",
"Automatically setup the board before building.",
)
# Image Type selection related options.
group = parser.add_argument_group("Image Type Options")
group.add_bool_argument(
"--withdev",
True,
"Build useful developer friendly utilities.",
"Omit extra dev image related packages.",
)
group.add_bool_argument(
"--withdebug",
True,
"Build debug versions of CrOS-specific packages. "
"Enables DCHECK, USE=cros-debug, etc...",
"Build release versions of CrOS-specific packages.",
)
group.add_bool_argument(
"--withfactory",
True,
"Build factory installer.",
"Omit factory related packages.",
)
group.add_bool_argument(
"--withtest",
True,
"Build packages required for testing.",
"Omit test image related packages.",
)
group.add_bool_argument(
"--withautotest",
True,
"Build autotest client code.",
"Omit autotest related packages.",
)
group.add_bool_argument(
"--withdebugsymbols",
False,
"Install the debug symbols for all packages.",
"Skip debug symbol install -- faster, but debugging is difficult.",
)
# Advanced Options.
group = parser.add_argument_group("Advanced Options")
group.add_argument(
"--accept-licenses", help="Licenses to append to the accept list."
)
group.add_bool_argument(
"--eclean",
True,
"Run eclean to delete old binpkgs.",
"Don't run eclean.",
)
group.add_argument(
"--jobs",
type=int,
default=os.cpu_count(),
help=(
"Number of packages to build in parallel. (Default: "
"%(default)s)"
),
)
group.add_bool_argument(
"--expandedbinhosts",
True,
"Allow expanded binhost inheritance.",
"Don't expand binhosts.",
)
group.add_argument(
"--backtrack",
type=int,
default=sysroot.BACKTRACK_DEFAULT,
help="See emerge --backtrack.",
)
# The --reuse-pkgs-from-local-boards flag tells Portage to share binary
# packages between boards that are built locally, so that the total time
# required to build several boards is reduced. This flag is only useful
# when you are not able to use remote binary packages, since remote
# binary packages are usually more up to date than anything you have
# locally.
group.add_bool_argument(
"--reuse-pkgs-from-local-boards",
False,
"Bootstrap from local packages instead of remote packages.",
"Only pull remote binpkgs.",
)
# --run-goma option is designed to be used on bots.
# If you're trying to build packages with goma in your local dev env,
# this is *not* the option you're looking for. Please see comments
# below. This option; 1) starts goma, 2) builds packages (expecting
# that goma is used), then 3) stops goma explicitly.
# 4) is a request from the goma team, so that stats/logs can be taken.
# Note: GOMA_DIR is expected to be passed via env var.
#
# In local dev env cases, compiler_proxy is expected to keep running.
# In such a case;
# $ python ${GOMA_DIR}/goma_ctl.py ensure_start
# $ cros build-image (... and options without --run-goma ...)
# is an expected commandline sequence. If you set --run-goma flag while
# compiler_proxy is already running, the existing compiler_proxy will be
# stopped.
group.add_bool_argument(
"--run-goma",
False,
"(Re)start goma, build packages, and then stop goma.",
"Don't use goma to build.",
)
# This option is for building chrome remotely.
# 1) starts reproxy 2) builds chrome with reproxy and 3) stops reproxy
# so logs/stats can be collected.
# Note: RECLIENT_DIR and REPROXY_CFG env var will be deprecated July
# 2022. Use --reclient-dir and --reproxy-cfg input options instead.
group.add_bool_argument(
"--run-remoteexec",
False,
"Start RBE reproxy, build packages, and then stop reproxy.",
"Don't use RBE to build.",
)
deprecated_note = "Flag will be removed Jan 2025. Use %s instead."
group.add_argument(
"--run_remoteexec",
action="store_true",
deprecated=deprecated_note % "--run-remoteexec",
help=argparse.SUPPRESS,
)
group.add_argument(
"--no-run_remoteexec",
action="store_false",
deprecated=deprecated_note % "--no-run-remoteexec",
help=argparse.SUPPRESS,
)
group.add_bool_argument(
"--bazel",
False,
"Use Bazel to build packages.",
"Use portage (emerge) to build packages.",
)
group.add_bool_argument(
"--bazel-lite",
False,
"Perform lite build with a limited set of packages.",
"Build all packages.",
)
group.add_argument(
"--bazel_lite",
action="store_true",
deprecated=deprecated_note % "--bazel-lite",
help=argparse.SUPPRESS,
)
group.add_argument(
"--no-bazel_lite",
action="store_false",
deprecated=deprecated_note % "--no-bazel-lite",
help=argparse.SUPPRESS,
)
group.add_bool_argument(
"--bazel-use-remote-execution",
False,
"Execute Bazel actions remotely.",
"Execute Bazel actions locally.",
)
parser.add_argument("packages", nargs="*", help="Packages to build.")
return parser
@classmethod
def ProcessOptions(
cls,
parser: commandline.ArgumentParser,
options: commandline.ArgumentNamespace,
) -> None:
if not options.board:
# Not supplied and no default set.
parser.error("--board is required")
if options.chrome:
options.internal = True
options.use_any_chrome = False
elif options.chromium:
options.internal = False
options.use_any_chrome = False
if options.cleanbuild:
# Turn off incremental builds when force replacing the sysroot since
# they can't be incremental.
options.withrevdeps = False
options.setup_board_run_config = sysroot.SetupBoardRunConfig(
force=options.cleanbuild,
usepkg=options.usepkg,
jobs=options.jobs,
quiet=True,
upgrade_chroot=not options.skip_chroot_upgrade,
local_build=options.reuse_pkgs_from_local_boards,
expanded_binhost_inheritance=options.expandedbinhosts,
use_cq_prebuilts=options.usepkg,
backtrack=options.backtrack,
)
options.build_run_config = sysroot.BuildPackagesRunConfig(
usepkg=options.usepkg,
install_debug_symbols=options.withdebugsymbols,
packages=options.packages,
use_goma=options.run_goma,
use_remoteexec=options.run_remoteexec,
incremental_build=options.withrevdeps,
dryrun=options.pretend,
usepkgonly=options.usepkgonly,
workon=options.workon,
install_auto_test=options.withautotest,
autosetgov=options.autosetgov,
autosetgov_sticky=options.autosetgov_sticky,
use_any_chrome=options.use_any_chrome,
internal_chrome=options.internal,
eclean=options.eclean,
jobs=options.jobs,
local_pkg=options.reuse_pkgs_from_local_boards,
dev_image=options.withdev,
factory_image=options.withfactory,
test_image=options.withtest,
debug_version=options.withdebug,
backtrack=options.backtrack,
bazel=options.bazel,
bazel_lite=options.bazel_lite,
bazel_use_remote_execution=options.bazel_use_remote_execution,
)
@timer.timed("Elapsed time (cros build-packages)")
def Run(self) -> None:
commandline.RunInsideChroot()
try:
build_packages(self.options)
except sysroot_lib.PackageInstallError as e:
try:
with urllib.request.urlopen(
"https://chromiumos-status.appspot.com/current?format=raw"
) as request:
logging.notice("Tree Status: %s", request.read().decode())
except urllib.error.HTTPError:
pass
cros_build_lib.Die(e)
@tracer.start_as_current_span("cli.cros.cros_build_packages.build_packages")
def build_packages(opts: commandline.ArgumentNamespace) -> None:
span = trace.get_current_span()
build_target = build_target_lib.BuildTarget(
opts.board,
build_root=opts.sysroot,
profile=opts.profile,
)
board_root = sysroot_lib.Sysroot(build_target.root)
if not board_root.Exists():
# Disable incremental builds when the sysroot doesn't exist.
opts.build_run_config.is_incremental = False
span.set_attributes(
{
"board": build_target.name,
"packages": opts.packages or [],
"is_complete": not opts.packages,
"is_incremental": opts.build_run_config.is_incremental,
"workon": opts.workon is True,
"bazel": opts.bazel is True,
}
)
# TODO(xcl): Update run_configs to have a common base set of configs for
# setup_board and cros build-packages.
if not opts.skip_setup_board:
sysroot.SetupBoard(
build_target,
accept_licenses=opts.accept_licenses,
run_configs=opts.setup_board_run_config,
)
sysroot.BuildPackages(build_target, board_root, opts.build_run_config)