# Copyright 2018 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Sysroot service."""

import contextlib
import datetime
import glob
import logging
import multiprocessing
import os
from pathlib import Path
import re
import shutil
import tempfile
from typing import (
    Dict,
    Generator,
    Iterator,
    List,
    NamedTuple,
    Optional,
    TYPE_CHECKING,
    Union,
)
import urllib

from chromite.third_party.opentelemetry import trace

from chromite.lib import cache
from chromite.lib import chromite_config
from chromite.lib import constants
from chromite.lib import cpupower_helper
from chromite.lib import cros_build_lib
from chromite.lib import goma_lib
from chromite.lib import gs
from chromite.lib import metrics_lib
from chromite.lib import osutils
from chromite.lib import portage_util
from chromite.lib import remoteexec_util
from chromite.lib import sysroot_lib
from chromite.lib import workon_helper
from chromite.service import sdk as sdk_service


if TYPE_CHECKING:
    from chromite.lib import binpkg
    from chromite.lib import build_target_lib
    from chromite.lib import chroot_lib


tracer = trace.get_tracer(__name__)

# TODO(xcl): Revisit/remove this after the Lacros launch if no longer needed
_CHROME_PACKAGES = ("chromeos-base/chromeos-chrome", "chromeos-base/chrome-icu")

# A list of critical system packages that should never be incidentally
# reinstalled as a side effect of build_packages. All packages in this list
# are special cased to prefer matching installed versions, overriding the
# typical logic of upgrading to the newest available version.
#
# This list can't include any package that gets installed to a board!
# Packages such as LLVM or binutils must not be in this list as the normal
# rebuild logic must still apply to them for board targets.
#
# TODO(crbug/1050752): Remove this list once we figure out how to exclude
# toolchain packages from being upgraded transitively via BDEPEND relations.
_CRITICAL_SDK_PACKAGES = (
    "dev-embedded/hps-sdk",
    "dev-lang/rust",
    "dev-lang/go",
    "sys-libs/glibc",
    "sys-devel/gcc",
)

_PACKAGE_LIST = List[Optional[str]]

# The default to use for --backtrack everywhere. Must be manually changed in
# update_chroot.
BACKTRACK_DEFAULT = 10

SYSROOT_ARCHIVE_FILE = "sysroot.tar.zst"


class Error(Exception):
    """Base error class for the module."""


class NoFilesError(Error):
    """When there are no files to archive."""


class InvalidArgumentsError(Error):
    """Invalid arguments."""


class NotInChrootError(Error):
    """When SetupBoard is run outside of the chroot."""


class UpdateChrootError(Error):
    """Error occurred when running update chroot."""


class SetupBoardRunConfig:
    """Value object for full setup board run configurations."""

    def __init__(
        self,
        set_default: bool = False,
        force: bool = False,
        usepkg: bool = True,
        jobs: Optional[int] = None,
        regen_configs: bool = False,
        quiet: bool = False,
        update_toolchain: bool = True,
        upgrade_chroot: bool = True,
        init_board_pkgs: bool = True,
        local_build: bool = False,
        toolchain_changed: bool = False,
        package_indexes: Optional[List["binpkg.PackageIndexInfo"]] = None,
        expanded_binhost_inheritance: bool = False,
        use_cq_prebuilts: bool = False,
        backtrack: int = BACKTRACK_DEFAULT,
    ):
        """Initialize method.

        Args:
            set_default: Whether to set the passed board as the default.
            force: Force a new sysroot creation when it already exists.
            usepkg: Whether to use binary packages to bootstrap.
            jobs: Max number of simultaneous packages to build.
            regen_configs: Whether to only regen the configs.
            quiet: Whether to print notification when sysroot exists.
            update_toolchain: Update the toolchain?
            upgrade_chroot: Upgrade the chroot before building?
            init_board_pkgs: Emerging packages to sysroot?
            local_build: Bootstrap only from local packages?
            toolchain_changed: Has a toolchain change occurred? Implies 'force'.
            package_indexes: List of information about available prebuilts,
                youngest first, or None.
            expanded_binhost_inheritance: Allow expanded binhost inheritance.
            use_cq_prebuilts: Whether to use the prebuilts generated by CQ.
            backtrack: emerge --backtrack value.
        """
        self.set_default = set_default
        self.force = force or toolchain_changed
        self.usepkg = usepkg
        self.jobs = jobs
        self.regen_configs = regen_configs
        self.quiet = quiet
        self.update_toolchain = update_toolchain
        self.update_chroot = upgrade_chroot
        self.init_board_pkgs = init_board_pkgs
        self.local_build = local_build
        self.package_indexes = package_indexes or []
        self.expanded_binhost_inheritance = expanded_binhost_inheritance
        self.use_cq_prebuilts = use_cq_prebuilts
        self.backtrack = backtrack

    def GetUpdateChrootArgs(
        self, toolchain_target: str
    ) -> sdk_service.UpdateArguments:
        """Create a list containing the relevant update_chroot arguments.

        Returns:
            The list of arguments
        """
        return sdk_service.UpdateArguments(
            build_source=not self.usepkg,
            toolchain_targets=[toolchain_target],
            jobs=self.jobs,
            backtrack=self.backtrack,
            update_toolchain=self.update_toolchain,
        )


class BuildPackagesRunConfig:
    """Value object to hold build packages run configs."""

    def __init__(
        self,
        usepkg: bool = True,
        install_debug_symbols: bool = False,
        packages: Optional[List[str]] = None,
        use_flags: Optional[List[str]] = None,
        use_goma: bool = False,
        use_remoteexec: bool = False,
        incremental_build: bool = True,
        package_indexes: Optional[List["binpkg.PackageIndexInfo"]] = None,
        dryrun: bool = False,
        usepkgonly: bool = False,
        workon: bool = True,
        install_auto_test: bool = True,
        autosetgov: bool = False,
        autosetgov_sticky: bool = False,
        use_any_chrome: bool = True,
        internal_chrome: bool = False,
        clean_build: bool = False,
        eclean: bool = True,
        rebuild_dep: bool = True,
        jobs: Optional[int] = None,
        local_pkg: bool = False,
        dev_image: bool = True,
        factory_image: bool = True,
        test_image: bool = True,
        debug_version: bool = True,
        backtrack: int = BACKTRACK_DEFAULT,
        bazel: bool = False,
    ):
        """Init method.

        Args:
            usepkg: Whether to use binpkgs or build from source. False currently
                triggers a local build, which will enable local reuse.
            install_debug_symbols: Whether to include the debug symbols for all
                packages.
            packages: The list of packages to install, by default install all
                packages for the target.
            use_flags: A list of use flags to set.
            use_goma: Whether to enable goma.
            use_remoteexec: Whether to use RBE for remoteexec.
            incremental_build: Whether to treat the build as an incremental
                build or a fresh build. Always treating it as an incremental
                build is safe, but certain operations can be faster when we know
                we are doing a fresh build.
            package_indexes: List of information about available prebuilts,
                youngest first, or None.  Deprecated.
            dryrun: Whether to do a dryrun and not actually build any packages.
            usepkgonly: Only use binary packages to bootstrap; abort if any are
                missing.
            workon: Force-build workon packages.
            install_auto_test: Build autotest client code.
            autosetgov: Automatically set cpu governor to 'performance'.
            autosetgov_sticky: Remember --autosetgov setting for future runs.
            use_any_chrome: Use any Chrome prebuilt available, even if the
                prebuilt doesn't match exactly.
            internal_chrome: Build the internal version of chrome.
            clean_build: Perform a clean build; delete sysroot if it exists
                before building.
            eclean: Run eclean to delete old binpkgs.
            rebuild_dep: Rebuild dependencies.
            jobs: How many packages to build in parallel at maximum.
            local_pkg: Bootstrap from local packages instead of remote packages.
            dev_image: Build useful developer friendly utilities.
            factory_image: Build factory installer.
            test_image: Build packages required for testing.
            debug_version: Build debug versions of Chromium-OS-specific
                packages.
            backtrack: emerge --backtrack value.
            bazel: Whether to use Bazel to build packages.
        """
        self.usepkg = usepkg
        self.install_debug_symbols = install_debug_symbols
        self.packages = packages
        self.use_flags = use_flags
        self.use_goma = use_goma
        self.use_remoteexec = use_remoteexec
        self.is_incremental = incremental_build
        self.package_indexes = package_indexes or []
        self.dryrun = dryrun
        self.usepkgonly = usepkgonly
        self.workon = workon
        self.install_auto_test = install_auto_test
        self.autosetgov = autosetgov
        self.autosetgov_sticky = autosetgov_sticky
        self.use_any_chrome = use_any_chrome
        self.internal_chrome = internal_chrome
        self.clean_build = clean_build
        self.eclean = eclean
        self.rebuild_dep = rebuild_dep
        self.jobs = jobs
        self.local_pkg = local_pkg
        self.dev_image = dev_image
        self.factory_image = factory_image
        self.test_image = test_image
        self.debug_version = debug_version
        self.backtrack = backtrack
        self.bazel = bazel

    def GetUseFlags(self) -> Optional[str]:
        """Get the use flags as a single string."""
        use_flags = os.environ.get("USE", "").split()

        if self.use_flags:
            use_flags.extend(self.use_flags)

        if self.internal_chrome:
            use_flags.append("chrome_internal")

        if not self.debug_version:
            use_flags.append("-cros-debug")

        return " ".join(use_flags) if use_flags else None

    def GetExtraEnv(self) -> Dict[str, str]:
        """Get the extra env for this config."""
        env = {}

        use_flags = self.GetUseFlags()
        if use_flags:
            env["USE"] = use_flags

        if self.use_goma:
            env["USE_GOMA"] = "true"

        if self.use_remoteexec:
            env["USE_REMOTEEXEC"] = "true"

        return env

    def GetPackages(self) -> List[str]:
        """Get the set of packages to build for this config."""
        if self.packages:
            return self.packages

        packages = ["virtual/target-os"]

        if self.dev_image:
            packages.append("virtual/target-os-dev")

        if self.factory_image:
            packages.extend(
                ["virtual/target-os-factory", "virtual/target-os-factory-shim"]
            )

        if self.test_image:
            packages.append("virtual/target-os-test")

        if self.install_auto_test:
            packages.append("chromeos-base/autotest-all")

        return packages

    @metrics_lib.timed("service.sysroot.GetForceLocalBuildPackages")
    def GetForceLocalBuildPackages(
        self, sysroot: sysroot_lib.Sysroot
    ) -> _PACKAGE_LIST:
        """Get the set of force local build packages for this config.

        This includes:
            1. Getting packages for a test image.
            2. Getting packages and reverse dependencies for cros workon
                packages.
            3. Getting packages and reverse dependencies for base install
                packages.

        Args:
            sysroot: The sysroot to get packages for.

        Returns:
            A list of packages to build from source.

        Raises:
            cros_build_lib.RunCommandError
        """
        sysroot_path = Path(sysroot.path)
        force_local_build_packages = set()
        packages = self.GetPackages()
        metrics_prefix = "service.sysroot.GetForceLocalBuildPackages"

        cros_workon_packages = None
        if self.workon:
            cros_workon_packages = _GetCrosWorkonPackages(sysroot_path)

            # Any package that directly depends on an active cros_workon package
            # also needs to be rebuilt in order to be correctly built against
            # the current set of changes a user may have made to the cros_workon
            # package.
            if cros_workon_packages:
                force_local_build_packages.update(cros_workon_packages)

                with metrics_lib.timer(
                    f"{metrics_prefix}.CrosWorkonReverseDependencies"
                ):
                    reverse_dependencies = [
                        x.atom
                        for x in portage_util.GetReverseDependencies(
                            cros_workon_packages, sysroot_path
                        )
                    ]
                logging.info(
                    "The following packages depend directly on an active "
                    "cros_workon package and will be rebuilt: %s",
                    " ".join(reverse_dependencies),
                )
                force_local_build_packages.update(reverse_dependencies)

        # Determine base install packages and reverse dependencies if
        # incremental build (--withdevdeps) and clean build (--cleanbuild) is
        # not specified or the sysroot path exists.
        if self.is_incremental and (
            not self.clean_build or sysroot_path.exists()
        ):
            logging.info("Starting reverse dependency calculations...")

            # Temporarily modify the emerge flags so we can calculate the
            # revdeps on the modified packages.
            sim_emerge_flags = self.GetEmergeFlags()
            sim_emerge_flags.extend(
                [
                    "--pretend",
                    "--columns",
                    f'--reinstall-atoms={" ".join(packages)}',
                    f'--usepkg-exclude={" ".join(packages)}',
                ]
            )

            # cros-workon packages are always going to be force reinstalled, so
            # we add the forced reinstall behavior to the modified package
            # calculation. This is necessary to include when a user has already
            # installed a 9999 ebuild and is now reinstalling that package with
            # additional local changes, because otherwise the modified package
            # calculation would not see that a 'new' package is being installed.
            if cros_workon_packages:
                sim_emerge_flags.extend(
                    [
                        f'--reinstall-atoms={" ".join(cros_workon_packages)}',
                        f'--usepkg-exclude={" ".join(cros_workon_packages)}',
                    ]
                )

            revdeps_packages = _GetBaseInstallPackages(
                sysroot_path, sim_emerge_flags, packages
            )
            if revdeps_packages:
                force_local_build_packages.update(revdeps_packages)
                logging.info(
                    "Calculating reverse dependencies on packages: %s",
                    " ".join(revdeps_packages),
                )
                with metrics_lib.timer(f"{metrics_prefix}.ReverseDependencies"):
                    r_revdeps_packages = portage_util.GetReverseDependencies(
                        revdeps_packages, sysroot_path, indirect=True
                    )

                exclude_patterns = ["virtual/"]
                exclude_patterns.extend(_CHROME_PACKAGES)
                reverse_dependencies = [
                    x.atom
                    for x in r_revdeps_packages
                    if not any(p in x.atom for p in exclude_patterns)
                ]
                logging.info(
                    "Final reverse dependencies that will be rebuilt: %s",
                    " ".join(reverse_dependencies),
                )
                force_local_build_packages.update(reverse_dependencies)

        return list(force_local_build_packages)

    def GetEmergeFlags(self) -> List[str]:
        """Get the emerge flags for this config."""
        flags = [
            "-uDNv",
            f"--backtrack={self.backtrack}",
            "--newrepo",
            "--with-test-deps",
            "y",
        ]

        if self.use_any_chrome:
            for pkg in _CHROME_PACKAGES:
                flags.append(f"--force-remote-binary={pkg}")

        extra_board_flags = os.environ.get("EXTRA_BOARD_FLAGS", "").split()
        if extra_board_flags:
            flags.extend(extra_board_flags)

        if self.dryrun:
            flags.append("--pretend")

        if self.usepkg or self.local_pkg or self.usepkgonly:
            # Use binary packages. Include all build-time dependencies, so as to
            # avoid unnecessary differences between source and binary builds.
            flags.extend(["--getbinpkg", "--with-bdeps", "y"])
            if self.usepkgonly:
                flags.append("--usepkgonly")
            else:
                flags.append("--usepkg")

        if self.jobs:
            flags.append(f"--jobs={self.jobs}")

        if self.rebuild_dep:
            # TODO(vapier): Purge the rebuild_dep logic and --rebuild options
            # from various scripts & APIs.
            logging.warning("Ignore rebuild dependency request")

        return flags


@tracer.start_as_current_span("service.sysroot.SetupBoard")
def SetupBoard(
    target: "build_target_lib.BuildTarget",
    accept_licenses: Optional[str] = None,
    run_configs: Optional[SetupBoardRunConfig] = None,
) -> None:
    """Run the full process to setup a board's sysroot.

    This is the entry point to run the setup_board script.

    Args:
        target: The build target configuration.
        accept_licenses: The additional licenses to accept.
        run_configs: The run configs.

    Raises:
        sysroot_lib.ToolchainInstallError when the toolchain fails to install.
    """
    if not cros_build_lib.IsInsideChroot():
        raise NotInChrootError("SetupBoard must be run from inside the chroot")

    # Make sure we have valid run configs setup.
    run_configs = run_configs or SetupBoardRunConfig()

    sysroot = Create(target, run_configs, accept_licenses)

    if run_configs.regen_configs:
        # We're now done if we're only regenerating the configs.
        return

    InstallToolchain(target, sysroot, run_configs)


@tracer.start_as_current_span("service.sysroot.Create")
def Create(
    target: "build_target_lib.BuildTarget",
    run_configs: SetupBoardRunConfig,
    accept_licenses: Optional[str],
) -> sysroot_lib.Sysroot:
    """Create a sysroot.

    This entry point is the subset of the full setup process that does the
    creation and configuration of a sysroot, including installing portage.

    Args:
        target: The build target being installed in the sysroot being created.
        run_configs: The run configs.
        accept_licenses: The additional licenses to accept.
    """
    cros_build_lib.AssertInsideChroot()

    sysroot = sysroot_lib.Sysroot(target.root)

    if sysroot.Exists() and not run_configs.force and not run_configs.quiet:
        logging.warning(
            "Board output directory already exists: %s\n"
            "Use --force to clobber the board root and start again.",
            sysroot.path,
        )

    # Override regen_configs setting to force full setup run if the sysroot does
    # not exist.
    run_configs.regen_configs = run_configs.regen_configs and sysroot.Exists()

    # Make sure the chroot is fully up to date before we start unless the
    # chroot update is explicitly disabled.
    if run_configs.update_chroot:
        with tracer.start_as_current_span(
            "service.sysroot.Create.update_chroot"
        ):
            result = sdk_service.Update(
                run_configs.GetUpdateChrootArgs(target.name)
            )
        if not result.success:
            raise UpdateChrootError(
                "Error occurred while updating the chroot. "
                "See the logs for more information."
            )

    # Delete old sysroot to force a fresh start if requested.
    if sysroot.Exists() and run_configs.force:
        sysroot.Delete(background=True)

    # Step 1: Create folders.
    # Dependencies: None.
    # Create the skeleton.
    logging.info("Creating sysroot directories.")
    _CreateSysrootSkeleton(sysroot)

    # Step 2: Standalone configurations.
    # Dependencies: Folders exist.
    # Install main, board setup, and user make.conf files.
    logging.info("Installing configurations into sysroot.")
    _InstallConfigs(sysroot, target)

    # Step 3: Portage configurations.
    # Dependencies: make.conf.board_setup.
    # Create the command wrappers, choose profile, and make.conf.board.
    # Refresh the workon symlinks to compensate for crbug.com/679831.
    logging.info("Setting up portage in the sysroot.")
    _InstallPortageConfigs(
        sysroot,
        target,
        accept_licenses,
        run_configs.local_build,
        use_cq_prebuilts=run_configs.use_cq_prebuilts,
        package_indexes=run_configs.package_indexes,
        expanded_binhost_inheritance=run_configs.expanded_binhost_inheritance,
    )

    # Developer Experience Step: Set default board (if requested) to allow
    # running later commands without needing to pass the --board argument.
    if run_configs.set_default:
        cros_build_lib.SetDefaultBoard(target.name)

    return sysroot


def GenerateArchive(
    output_dir: str, build_target_name: str, pkg_list: List[str]
) -> str:
    """Generate a sysroot tarball for informational builders.

    Args:
        output_dir: Directory to contain the created the sysroot.
        build_target_name: The build target for the sysroot being created.
        pkg_list: List of 'category/package' package strings.

    Returns:
        Path to the sysroot tar file.
    """
    cmd = [
        "cros_generate_sysroot",
        "--out-file",
        constants.TARGET_SYSROOT_TAR,
        "--out-dir",
        output_dir,
        "--board",
        build_target_name,
        "--package",
        " ".join(pkg_list),
    ]
    cros_build_lib.run(cmd, cwd=constants.SOURCE_ROOT)
    return os.path.join(output_dir, constants.TARGET_SYSROOT_TAR)


def _create_sysroot(
    chroot: "chroot_lib.Chroot",
    _sysroot_class,
    build_target: "build_target_lib.BuildTarget",
    output_dir: str,
    package_list: List[str],
    output_file: str,
    deps_only: bool = True,
) -> str:
    """Create a sysroot to use.

    Args:
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot class used for these artifacts.
        build_target: The build target used for these artifacts.
        output_dir: The path to write artifacts to.
        package_list: List of packages to use.
        output_file: Name of the archive to output.
        deps_only: Whether to pass --deps-only.

    Returns:
        Path to the sysroot tar file.
    """
    with chroot.tempdir() as tempdir:
        outdir = chroot.chroot_path(tempdir)
        cmd = [
            "cros_generate_sysroot",
            "--out-dir",
            outdir,
            "--board",
            build_target.name,
            "--package",
            " ".join(package_list),
            "--out-file",
            output_file,
        ]
        if deps_only:
            cmd.append("--deps-only")
        chroot.run(cmd, cwd=constants.SOURCE_ROOT)

        # Move the artifact out of the chroot.
        sysroot_tar_path = os.path.join(tempdir, output_file)
        shutil.copy(sysroot_tar_path, output_dir)
        return os.path.join(output_dir, output_file)


def CreateSimpleChromeSysroot(
    chroot: "chroot_lib.Chroot",
    _sysroot_class,
    build_target: "build_target_lib.BuildTarget",
    output_dir: str,
) -> str:
    """Create a sysroot for SimpleChrome to use.

    Args:
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot class used for these artifacts.
        build_target: The build target used for these artifacts.
        output_dir: The path to write artifacts to.

    Returns:
        Path to the sysroot tar file.
    """
    return _create_sysroot(
        chroot,
        _sysroot_class,
        build_target,
        output_dir,
        [constants.CHROME_CP],
        output_file=constants.CHROME_SYSROOT_TAR,
    )


def CreateFuzzerSysroot(
    chroot: "chroot_lib.Chroot",
    _sysroot_class,
    build_target: "build_target_lib.BuildTarget",
    output_dir: str,
) -> str:
    """Create a sysroot for fuzzer builders.

    Args:
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot class used for these artifacts.
        build_target: The build target used for these artifacts.
        output_dir: The path to write artifacts to.

    Returns:
        Path to the sysroot tar file.
    """
    return _create_sysroot(
        chroot,
        _sysroot_class,
        build_target,
        output_dir,
        ["virtual/target-fuzzers"],
        output_file="sysroot_virtual_target-os.tar.xz",
        deps_only=False,
    )


def CreateChromeEbuildEnv(
    chroot: "chroot_lib.Chroot",
    sysroot_class: sysroot_lib.Sysroot,
    _build_target,
    output_dir: str,
) -> Optional[str]:
    """Generate Chrome ebuild environment.

    Args:
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot where the original environment archive can be
        found.
        output_dir: Where the result should be stored.

    Returns:
        The path to the archive, or None.
    """
    pkg_dir = chroot.full_path(sysroot_class.path, portage_util.VDB_PATH)
    files = glob.glob(os.path.join(pkg_dir, constants.CHROME_CP) + "-*")
    if not files:
        logging.warning("No package found for %s", constants.CHROME_CP)
        return None

    if len(files) > 1:
        logging.warning(
            "Expected one package for %s, found %d",
            constants.CHROME_CP,
            len(files),
        )

    chrome_dir = sorted(files)[-1]
    env_bzip = os.path.join(chrome_dir, "environment.bz2")
    result_path = os.path.join(output_dir, constants.CHROME_ENV_TAR)
    with osutils.TempDir() as tempdir:
        # Convert from bzip2 to tar format.
        bzip2 = cros_build_lib.FindCompressor(
            cros_build_lib.CompressionType.BZIP2
        )
        tempdir_tar_path = os.path.join(tempdir, constants.CHROME_ENV_FILE)
        cros_build_lib.run(
            [bzip2, "-d", env_bzip, "-c"], stdout=tempdir_tar_path
        )

        cros_build_lib.CreateTarball(result_path, tempdir)

    return result_path


@tracer.start_as_current_span("service.sysroot.InstallToolchain")
def InstallToolchain(
    target: "build_target_lib.BuildTarget",
    sysroot: sysroot_lib.Sysroot,
    run_configs: SetupBoardRunConfig,
) -> None:
    """Update the toolchain to a sysroot.

    This entry point just installs the target's toolchain into the sysroot.
    Everything else must have been done already for this to be successful.

    Args:
        target: The target whose toolchain is being installed.
        sysroot: The sysroot where the toolchain is being installed.
        run_configs: The run configs.
    """
    cros_build_lib.AssertInsideChroot()
    if not sysroot.Exists():
        # Sanity check before we try installing anything.
        raise ValueError("The sysroot must exist, run Create first.")

    # Step 4: Install toolchain and packages.
    # Dependencies: Portage configs and wrappers have been installed.
    if run_configs.init_board_pkgs:
        logging.info("Updating toolchain.")
        # Use the local packages if we're doing a local only build or usepkg is
        # set.
        local_init = run_configs.usepkg or run_configs.local_build
        _InstallToolchain(sysroot, target, local_init=local_init)


@tracer.start_as_current_span("service.sysroot.BuildPackages")
@metrics_lib.timed("service.sysroot.BuildPackages")
def BuildPackages(
    target: "build_target_lib.BuildTarget",
    sysroot: sysroot_lib.Sysroot,
    run_configs: BuildPackagesRunConfig,
) -> None:
    """Build and install packages into a sysroot.

    Args:
        target: The target whose packages are being installed.
        sysroot: The sysroot where the packages are being installed.
        run_configs: The run configs.

    Raises:
        sysroot_lib.PackageInstallError when packages fail to install.
    """
    cros_build_lib.AssertInsideChroot()
    cros_build_lib.AssertNonRootUser()
    metrics_prefix = "service.sysroot.BuildPackages"

    if (
        not chromite_config.AUTO_COP_CONFIG_OFF.is_file()
        and os.environ.get("CROS_CLEAN_OUTDATED_PKGS") != "0"
    ):
        logging.debug(
            "clean-outdated-pkgs config does not exist: %s",
            chromite_config.AUTO_COP_CONFIG_OFF,
        )
        cop_command = [
            "cros",
            "clean-outdated-pkgs",
            f"--board={target.name}",
        ]
        # Set check=False to allow cop to fail.
        cros_build_lib.sudo_run(
            cop_command,
            preserve_env=True,
            check=False,
        )

    extra_env = run_configs.GetExtraEnv()
    extra_env["PKGDIR"] = f"{sysroot.path}/packages"
    with osutils.TempDir() as tempdir, cpupower_helper.ModifyCpuGovernor(
        run_configs.autosetgov, run_configs.autosetgov_sticky
    ):
        extra_env[constants.CROS_METRICS_DIR_ENVVAR] = tempdir

        cros_build_lib.ClearShadowLocks(sysroot.path)

        portage_binhost = portage_util.PortageqEnvvar(
            "PORTAGE_BINHOST", target.name
        )
        logging.info("PORTAGE_BINHOST: %s", portage_binhost)

        binhosts = portage_binhost.strip().split()
        _LogBinhostAge(binhosts, date_threshold=30)

        # Before running any emerge operations, regenerate the Portage
        # dependency cache in parallel.
        logging.info("Rebuilding Portage cache.")
        with metrics_lib.timer(f"{metrics_prefix}.RegenPortageCache"):
            portage_util.RegenDependencyCache(
                sysroot=sysroot.path, jobs=run_configs.jobs
            )

        # Clean out any stale binpkgs we've accumulated. This is done
        # immediately after regenerating the cache in case ebuilds have been
        # removed (e.g. from a revert).
        if run_configs.eclean:
            _CleanStaleBinpkgs(sysroot.path)

        emerge_cmd = _GetEmergeCommand(sysroot.path)
        emerge_flags = run_configs.GetEmergeFlags()
        rebuild_pkgs = " ".join(run_configs.GetForceLocalBuildPackages(sysroot))
        if rebuild_pkgs:
            emerge_flags.extend(
                [
                    f"--reinstall-atoms={rebuild_pkgs}",
                    f"--usepkg-exclude={rebuild_pkgs}",
                ]
            )

        sdk_pkgs = " ".join(_CRITICAL_SDK_PACKAGES)
        emerge_flags.extend(
            [
                f"--useoldpkg-atoms={sdk_pkgs}",
                f"--rebuild-exclude={sdk_pkgs}",
            ]
        )
        with RemoteExecution(run_configs.use_goma, run_configs.use_remoteexec):
            logging.info("Merging board packages now.")
            try:
                with metrics_lib.timer(f"{metrics_prefix}.emerge"):
                    packages = run_configs.GetPackages()
                    span = trace.get_current_span()
                    span.set_attributes(
                        {
                            "board": target.name,
                            "packages": packages,
                            "bazel": run_configs.bazel,
                        }
                    )

                    if run_configs.bazel:
                        # Bazel needs amd64-host sysroot with sdk/bootstrap
                        # profile.
                        cros_build_lib.run(
                            [
                                constants.CROSUTILS_DIR
                                / "create_sdk_board_root",
                                "--board",
                                "amd64-host",
                                "--profile",
                                "sdk/bootstrap",
                            ]
                        )

                        bazel_extra_env = {"BOARD": target.name}
                        bazel_cmd = "/mnt/host/source/chromite/bin/bazel"
                        # Generate an exec log for a single package, to help us
                        # debug cache misses. We may eventually want to account
                        # for the possibility that sys-lib/zlib isn't in
                        # packages and so this means we're doing extra work,
                        # but we won't worry about that for now.
                        cros_build_lib.run(
                            [
                                bazel_cmd,
                                "build",
                                (
                                    "--execution_log_binary_file="
                                    "/tmp/bazel_build_appcryptnss_exec.log"
                                ),
                                "--execution_log_sort=false",
                                "@portage//app-crypt/nss:package_set",
                            ],
                            extra_env=bazel_extra_env,
                        )

                        cros_build_lib.run(
                            [
                                constants.SOURCE_ROOT
                                / "src/bazel/portage/tools"
                                / "install_packages_to_sysroot.py",
                                "--board",
                                target.name,
                            ]
                            + packages
                        )
                    else:
                        cros_build_lib.sudo_run(
                            emerge_cmd + emerge_flags + packages,
                            preserve_env=True,
                            extra_env=extra_env,
                        )
                logging.info("Builds complete.")
            except cros_build_lib.RunCommandError as e:
                failed_pkgs = portage_util.ParseDieHookStatusFile(tempdir)
                raise sysroot_lib.PackageInstallError(
                    "Merging board packages failed",
                    e.result,
                    exception=e,
                    packages=failed_pkgs,
                )

        if run_configs.install_debug_symbols:
            logging.info("Fetching the debug symbols.")
            try:
                # TODO(xcl): Convert to directly importing and calling a Python
                #   lib instead of calling a binary.
                cros_build_lib.run(
                    [
                        constants.CHROMITE_BIN_DIR / "cros_install_debug_syms",
                        f"--board={sysroot.build_target_name}",
                        "--all",
                    ]
                )
            except cros_build_lib.RunCommandError as e:
                logging.error("Unable to install debug symbols: %s", e)

        # Remove any broken or outdated binpkgs.
        if run_configs.eclean:
            portage_util.CleanOutdatedBinaryPackages(sysroot.path, deep=True)


def _LogBinhostAge(binhosts: List[str], date_threshold: int) -> None:
    """Log the age of the binhosts and suggest remediation steps if necessary.

    Args:
        binhosts: The list of binhost gs urls.
        date_threshold: The maximum number of days considered as acceptable
            before logging a warning.
    """
    today = datetime.datetime.now()

    for binhost in binhosts:
        # In some cases the binhost url ends with a trailing slash, the rstrip
        # handles those urls.
        package_index = binhost.rstrip("/") + "/Packages"

        try:
            binhost_age = gs.GSContext().GetCreationTimeSince(
                path=package_index, since_date=today
            )
        # Using the catch-all gs.GSContextException to record and suppress gs
        # errors since this is a function for logging and should not halt
        # program execution.
        except gs.GSContextException as e:
            logging.info("PORTAGE_BINHOST %s", binhost)
            logging.warning("Error getting the binhost age: %s", e)
            return

        logging.info(
            "PORTAGE_BINHOST %s was created %d days ago.",
            binhost,
            binhost_age.days,
        )

        if binhost_age.days >= date_threshold:
            logging.warning(
                "PORTAGE_BINHOST %s was created more than 30 days ago. "
                "Please repo sync for the latest build artifacts.",
                binhost,
            )


def _CleanStaleBinpkgs(sysroot: Union[str, os.PathLike]) -> None:
    """Clean any accumulated stale binpkgs.

    Args:
        sysroot: The sysroot to clean stale binpkgs for.

    Raises:
        cros_build_lib.RunCommandError
    """
    logging.info("Cleaning stale binpkgs.")
    exclude_pkgs = [
        x.package_info.atom
        for x in portage_util.PortageDB().InstalledPackages()
        if x.category.startswith("cross-")
    ]
    with tempfile.NamedTemporaryFile(mode="w") as f:
        f.write("\n".join(exclude_pkgs))
        f.flush()
        portage_util.CleanOutdatedBinaryPackages(
            sysroot, deep=True, exclusion_file=f.name
        )


def _GetCrosWorkonPackages(sysroot: Union[str, os.PathLike]) -> _PACKAGE_LIST:
    """Get cros_workon packages.

    Args:
        sysroot: The sysroot to get cros_workon packages for.

    Raises:
        cros_build_lib.RunCommandError
    """
    # TODO(xcl): Migrate this to calling an imported Python lib
    cmd = ["cros_list_modified_packages", "--sysroot", sysroot]
    result = cros_build_lib.run(
        cmd, print_cmd=False, capture_output=True, encoding="utf-8"
    )
    logging.info(
        "Detected cros_workon modified packages: %s", result.stdout.rstrip()
    )
    packages = result.stdout.split()

    if os.environ.get("CHROME_ORIGIN"):
        packages.extend(_CHROME_PACKAGES)

    return packages


def _GetBaseInstallPackages(
    sysroot: Union[str, os.PathLike], emerge_flags: str, packages: List[str]
) -> List[Optional[str]]:
    """Get packages to determine reverse dependencies for.

    Args:
        sysroot: The sysroot to get packages for.
        emerge_flags: Emerge flags to run the command with.
        packages: The packages to get dependencies for.

    Raises:
        cros_build_lib.RunCommandError
    """
    # Do a pretend `emerge` command to get a list of what would be built.
    # Sample output:
    # [binary N] dev-go/containerd ... to /build/eve/ USE="..."
    # [ebuild r U] chromeos-base/tast-build-deps ... to /build/eve/ USE="..."
    # [binary U] chromeos-base/chromeos-chrome ... to /build/eve/ USE="..."
    cmd = _GetEmergeCommand(sysroot)
    result = cros_build_lib.sudo_run(
        cmd + emerge_flags + packages, capture_output=True, encoding="utf-8"
    )

    # Filter to a heuristic set of packages known to have incorrectly specified
    # dependencies that will be installed to the board sysroot.
    # Sample output is filtered to:
    # [ebuild r U] chromeos-base/tast-build-deps ... to /build/eve/ USE="..."
    include_patterns = ["coreboot-private-files", "tast-build-deps"]

    # Pattern used to rewrite the line from Portage's full output to only
    # $CATEGORY/$PACKAGE.
    pattern = re.compile(r"\[ebuild(.*?)\]\s(.*?)\s")

    # Filter and sort the output and remove any duplicate entries.
    packages = set()
    for line in result.stdout.splitlines():
        if "to /build/" in line and any(x in line for x in include_patterns):
            # Use regex to get substrings that matches a
            # '[ebuild ...] <some characters> ' pattern. The second matching
            # group returns the $CATEGORY/$PACKAGE from a line of the emerge
            # output.
            m = pattern.search(line)
            if m:
                packages.add(m.group(2))
    return sorted(packages)


def _GetEmergeCommand(
    sysroot: Optional[Union[str, os.PathLike]] = None
) -> List[Union[str, os.PathLike]]:
    """Get the emerge command to use with `cros build-packages`."""
    # TODO(xcl): Convert to directly importing and calling a Python lib instead
    # of calling a binary.
    cmd = [constants.CHROMITE_BIN_DIR / "parallel_emerge"]
    if sysroot:
        cmd.extend(
            [
                f"--sysroot={sysroot}",
                f"--root={sysroot}",
            ]
        )
    return cmd


def _CreateSysrootSkeleton(sysroot: sysroot_lib.Sysroot) -> None:
    """Create the sysroot skeleton.

    Dependencies: None.
    Creates the sysroot directory structure and installs the portage hooks.

    Args:
        sysroot: The sysroot.
    """
    sysroot.CreateSkeleton()


def _InstallConfigs(
    sysroot: sysroot_lib.Sysroot, target: "build_target_lib.BuildTarget"
) -> None:
    """Install standalone configuration files into the sysroot.

    Dependencies: The sysroot exists (i.e. CreateSysrootSkeleton).
    Installs the main, board setup, and user make.conf files.

    Args:
        sysroot: The sysroot.
        target: The build target being setup in the sysroot.
    """
    sysroot.InstallMakeConf()
    sysroot.InstallMakeConfBoardSetup(target.name)
    sysroot.InstallMakeConfUser()


def _InstallPortageConfigs(
    sysroot: sysroot_lib.Sysroot,
    target: "build_target_lib.BuildTarget",
    accept_licenses: Optional[str],
    local_build: bool,
    package_indexes: List["binpkg.PackageIndexInfo"] = None,
    use_cq_prebuilts: bool = False,
    expanded_binhost_inheritance: bool = False,
) -> None:
    """Install portage wrappers and configurations.

    Dependencies: make.conf.board_setup (InstallConfigs).
    Create the command wrappers, choose profile, and generate make.conf.board.
    Refresh the workon symlinks to compensate for crbug.com/679831.

    Args:
        sysroot: The sysroot.
        target: The build target being installed in the sysroot.
        accept_licenses: Additional accepted licenses as a string.
        local_build: If the build is a local only build.
        package_indexes: List of information about available prebuilts, youngest
            first, or None.
        use_cq_prebuilts: Whether to use the prebuilts generated by CQ.
        expanded_binhost_inheritance: Whether to allow expanded binhost
            inheritance.
    """
    sysroot.CreateAllWrappers(friendly_name=target.name)
    _ChooseProfile(target, sysroot)
    _RefreshWorkonSymlinks(target.name, sysroot)
    # Must be done after the profile is chosen or binhosts may be incomplete.
    sysroot.InstallMakeConfBoard(
        accepted_licenses=accept_licenses,
        local_only=local_build,
        package_indexes=package_indexes,
        use_cq_prebuilts=use_cq_prebuilts,
        expanded_binhost_inheritance=expanded_binhost_inheritance,
    )


def _InstallToolchain(
    sysroot: sysroot_lib.Sysroot,
    target: "build_target_lib.BuildTarget",
    local_init: bool = True,
) -> None:
    """Install toolchain and packages.

    Dependencies: Portage configs and wrappers have been installed
        (InstallPortageConfigs).
    Install the toolchain and the implicit dependencies.

    Args:
        sysroot: The sysroot to install to.
        target: The build target whose toolchain is being installed.
        local_init: Whether to use local packages to bootstrap implicit
            dependencies.
    """
    sysroot.UpdateToolchain(target.name, local_init=local_init)


def _RefreshWorkonSymlinks(target: str, sysroot: sysroot_lib.Sysroot) -> None:
    """Force refresh the workon symlinks.

    Create an instance of the WorkonHelper, which will recreate all symlinks
    to masked/unmasked packages currently worked on in case the sysroot was
    recreated (crbug.com/679831).

    This was done with a call to `cros_workon list` in the bash version of
    the script, but all we actually need is for the WorkonHelper to be
    instantiated since it refreshes the symlinks in its __init__.

    Args:
        target: The build target name.
        sysroot: The board's sysroot.
    """
    workon_helper.WorkonHelper(sysroot.path, friendly_name=target)


def _ChooseProfile(
    target: "build_target_lib.BuildTarget", sysroot: sysroot_lib.Sysroot
) -> None:
    """Helper function to execute cros_choose_profile.

    TODO(saklein) Refactor cros_choose_profile to avoid needing the run
    call here, and by extension this method all together.

    Args:
        target: The build target whose profile is being chosen.
        sysroot: The sysroot for which the profile is being chosen.
    """
    choose_profile = [
        "cros_choose_profile",
        "--board",
        target.name,
        "--board-root",
        sysroot.path,
    ]
    if target.profile:
        # Chooses base by default, only override when we have a passed param.
        choose_profile += ["--profile", target.profile]
    try:
        cros_build_lib.run(choose_profile, print_cmd=False)
    except cros_build_lib.RunCommandError as e:
        logging.error(
            "Selecting profile failed, removing incomplete board " "directory!"
        )
        sysroot.Delete()
        raise e


def BundleDebugSymbols(
    chroot: "chroot_lib.Chroot",
    sysroot_class: sysroot_lib.Sysroot,
    _build_target: "build_target_lib.BuildTarget",
    output_dir: str,
) -> Optional[str]:
    """Bundle debug symbols into a tarball for importing into GCE.

    Bundle the debug symbols found in the sysroot into a .tgz. This assumes
    these files are present.

    Args:
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot class used for these artifacts.
        build_target: The build target used for these artifacts.
        output_dir: The path to write artifacts to.

    Returns:
        A string path to the output debug.tgz artifact, or None.
    """
    base_path = chroot.full_path(sysroot_class.path)
    debug_dir = os.path.join(base_path, "usr/lib/debug")

    if not os.path.isdir(debug_dir):
        logging.error("No debug directory found at %s.", debug_dir)
        return None

    # Create tarball from destination_tmp, then copy it...
    tarball_path = os.path.join(output_dir, constants.DEBUG_SYMBOLS_TAR)
    exclude_breakpad_tar_arg = "--exclude=%s" % os.path.join(
        debug_dir, "breakpad"
    )
    exclude_vmlinux_tar_arg = "--exclude=%s" % os.path.join(
        debug_dir, "boot/vmlinux.debug"
    )
    result = None
    try:
        result = cros_build_lib.CreateTarball(
            tarball_path,
            debug_dir,
            compression=cros_build_lib.CompressionType.GZIP,
            sudo=True,
            extra_args=[exclude_breakpad_tar_arg, exclude_vmlinux_tar_arg],
        )
    except cros_build_lib.TarballError:
        pass
    if not result or result.returncode:
        # We don't abort here, because the tar may still be somewhat intact.
        err = result.return_code if result else "TarballError"
        logging.error(
            "Error (%s) when creating tarball %s from %s",
            err,
            tarball_path,
            debug_dir,
        )
    if os.path.exists(tarball_path):
        return tarball_path
    else:
        return None


def BundleBreakpadSymbols(
    chroot: "chroot_lib.Chroot",
    sysroot_class: sysroot_lib.Sysroot,
    build_target: "build_target_lib.BuildTarget",
    output_dir: str,
    ignore_generation_errors: bool,
    ignore_generation_expected_files: List[str],
) -> Optional[str]:
    """Bundle breakpad debug symbols into a tarball for importing into GCE.

    Call the GenerateBreakpadSymbols function and archive this into a tar.gz.

    Args:
        chroot: The chroot class used for these artifacts.
        sysroot_class: The sysroot class used for these artifacts.
        build_target: The build target used for these artifacts.
        output_dir: The path to write artifacts to.
        ignore_generation_errors: If True, ignore errors during symbol
            generation.
        ignore_generation_expected_files: A list of files (like "ASH_CHROME" or
            "LIBC") that symbol generation normally expects to generate symbols
            for; the generate symbols program will not generate errors if it
            doesn't generate symbols for a file in the list. See
            cros_generate_breakpad_symbols.py's ExpectedFiles enum for the list
            of valid values.

    Returns:
        A string path to the output debug_breakpad.tar.gz artifact, or None.
    """
    base_path = chroot.full_path(sysroot_class.path)

    result = GenerateBreakpadSymbols(
        chroot,
        build_target,
        debug=True,
        ignore_errors=ignore_generation_errors,
        ignore_expected_files=ignore_generation_expected_files,
    )

    # Verify breakpad symbol generation before gathering the sym files.
    if result.returncode:
        logging.error(
            "Error (%d) when generating breakpad symbols", result.returncode
        )
        return None
    with chroot.tempdir() as symbol_tmpdir, chroot.tempdir() as dest_tmpdir:
        breakpad_dir = os.path.join(base_path, "usr/lib/debug/breakpad")
        # Call list on the atifacts.GatherSymbolFiles generator function to
        # materialize and consume all entries so that all are copied to
        # dest dir and complete list of all symbol files is returned.
        sym_file_list = list(
            GatherSymbolFiles(
                tempdir=symbol_tmpdir, destdir=dest_tmpdir, paths=[breakpad_dir]
            )
        )

        if not sym_file_list:
            logging.warning("No sym files found in %s.", breakpad_dir)
        # Create tarball from destination_tmp, then copy it...
        tarball_path = os.path.join(
            output_dir, constants.BREAKPAD_DEBUG_SYMBOLS_TAR
        )
        result = cros_build_lib.CreateTarball(tarball_path, dest_tmpdir)
        if result.returncode != 0:
            logging.error(
                "Error (%d) when creating tarball %s from %s",
                result.returncode,
                tarball_path,
                dest_tmpdir,
            )
            return None
    return tarball_path


# A SymbolFileTuple is a data object that contains:
#  relative_path (str): Relative path to the file based on initial search path.
#  source_file_name (str): Full path to where the SymbolFile was found.
# For example, if the search path for symbol files is '/some/bot/path/'
# and a symbol file is found at '/some/bot/path/a/b/c/file1.sym',
# then the relative_path would be 'a/b/c/file1.sym' and the source_file_name
# would be '/some/bot/path/a/b/c/file1.sym'.
# The source_file_name is informational for two reasons:
# 1) They are typically copied off a machine (such as a build bot) where
#    that path will disappear, which is why when we find them they get
#    copied to a destination directory.
# 2) For tar files, the source_file_name is not a full path that can be
#    opened, since it is the path the tar file plus the relative path of
#    the file when we untar it.
class SymbolFileTuple(NamedTuple):
    """Contain a relative and full path to a SymbolFile."""

    relative_path: str
    source_file_name: str


def GenerateBreakpadSymbols(
    chroot: "chroot_lib.Chroot",
    build_target: "build_target_lib.BuildTarget",
    debug: bool,
    ignore_errors: bool,
    ignore_expected_files: List[str],
) -> cros_build_lib.CompletedProcess:
    """Generate breakpad (go/breakpad) symbols for debugging.

    This function generates .sym files to /build/<board>/usr/lib/debug/breakpad
    from .debug files found in /build/<board>/usr/lib/debug by calling
    cros_generate_breakpad_symbols.

    Args:
        chroot: The chroot in which the sysroot should be built.
        build_target: The sysroot's build target.
        debug: Include extra debugging output.
        ignore_errors: If True, ignore errors and generate symbols best effort.
        ignore_expected_files: A list of files (like "ASH_CHROME" or "LIBC")
            that we tell cros_generate_breakpad_symbols it should not expect to
            generate symbols for. See cros_generate_breakpad_symbols.py's
            ExpectedFiles enum for the list of valid values.
    """
    # The firmware directory contains elf symbols that we have trouble parsing
    # and that don't help with breakpad debugging (see crbug.com/213670).
    exclude_dirs = ["firmware"]

    cmd = ["cros_generate_breakpad_symbols"]
    if debug:
        cmd += ["--debug"]
    if ignore_errors:
        cmd += ["--ignore_errors"]
    for ignore_expected_file in ignore_expected_files:
        cmd += ["--ignore_expected_file=" + ignore_expected_file]

    # Execute for board in parallel with half # of cpus available to avoid
    # starving other parallel processes on the same machine.
    cmd += [
        "--board=%s" % build_target.name,
        "--jobs",
        str(max(1, multiprocessing.cpu_count() // 2)),
    ]
    cmd += ["--exclude-dir=%s" % x for x in exclude_dirs]

    logging.info("Generating breakpad symbols: %s.", cmd)
    result = chroot.run(cmd)
    return result


def GatherSymbolFiles(
    tempdir: str, destdir: str, paths: List[str]
) -> Generator[SymbolFileTuple, None, None]:
    """Locate symbol files in |paths|

    This generator function searches all paths for .sym files and copies them to
    destdir. A path to a tarball will result in the tarball being unpacked and
    examined. A path to a directory will result in the directory being searched
    for .sym files. The generator yields SymbolFileTuple objects that contain
    symbol file references which are valid after this exits. Those files may
    exist externally, or be created in the tempdir (when expanding tarballs).
    Typical usage in the BuildAPI will be for the .sym files to exist under a
    directory such as /build/<board>/usr/lib/debug/breakpad so that the path to
    a sym file will always be unique.
    Note: the caller must clean up the tempdir.
    Note: this function is recursive for tar files.

    Args:
        tempdir: Path to use for temporary files.
        destdir: All .sym files are copied to this path. Tarfiles are opened
            inside a tempdir and any .sym files within them are copied to
            destdir from within that temp path.
        paths: A list of input paths to walk. Files are returned based on .sym
            name w/out any checking internal symbol file format. Dirs are
            searched for files that end in ".sym". Urls are not handled.
            Tarballs are unpacked and walked.

    Yields:
        A SymbolFileTuple for every symbol file found in paths.
    """
    logging.info(
        "GatherSymbolFiles tempdir %s destdir %s paths %s",
        tempdir,
        destdir,
        paths,
    )
    for p in paths:
        o = urllib.parse.urlparse(p)
        if o.scheme:
            raise NotImplementedError("URL paths are not expected/handled: ", p)
        elif not os.path.exists(p):
            raise NoFilesError("The path did not exist: ", p)
        elif os.path.isdir(p):
            for root, _, files in os.walk(p):
                for f in files:
                    if f.endswith(".sym"):
                        # If p is '/tmp/foo' and filename is
                        # '/tmp/foo/bar/bar.sym', relative_path = 'bar/bar.sym'
                        filename = os.path.join(root, f)
                        relative_path = filename[len(p) :].lstrip("/")
                        try:
                            shutil.copy(
                                filename, os.path.join(destdir, relative_path)
                            )
                        except IOError:
                            # Handles pre-3.3 Python where we may need to make
                            # the target path's dirname before copying.
                            os.makedirs(
                                os.path.join(
                                    destdir, os.path.dirname(relative_path)
                                )
                            )
                            shutil.copy(
                                filename, os.path.join(destdir, relative_path)
                            )
                        yield SymbolFileTuple(
                            relative_path=relative_path,
                            source_file_name=filename,
                        )

        elif cros_build_lib.IsTarball(p):
            tardir = tempfile.mkdtemp(dir=tempdir)
            cache.Untar(os.path.realpath(p), tardir)
            for sym in GatherSymbolFiles(tardir, destdir, [tardir]):
                # The SymbolFileTuple is generated from [tardir], but we want
                # the source_file_name (which informational) to reflect the tar
                # path plus the relative path after the file is untarred. Thus,
                # something like /botpath/some/path/tmp22dl33sa/dir1/fileB.sym
                # (where the tardir is /botpath/some/path/tmp22dl33sa) has a
                # resulting path /botpath/some/path/symfiles.tar/dir1/fileB.sym
                # When we call GatherSymbolFiles with [tardir] as the argument,
                # the os.path.isdir case above will walk the tar contents,
                # processing only .sym. Non-sym files within the tar file will
                # be ignored (even tar files within tar files, which we don't
                # expect).
                new_source_file_name = sym.source_file_name.replace(tardir, p)
                yield SymbolFileTuple(
                    relative_path=sym.relative_path,
                    source_file_name=new_source_file_name,
                )

        elif os.path.isfile(p):
            # Path p is a file. This code path is only executed when a full file
            # path is one of the elements in the 'paths' argument. When a
            # directory is an element of the 'paths' argument, we walk the tree
            # (above) and process each file. When a tarball is an element of the
            # 'paths' argument, we untar it into a directory and recurse with
            # the temp tardir as the directory, so that tarfile contents are
            # processed (above) in the os.walk of the directory.
            if p.endswith(".sym"):
                shutil.copy(p, destdir)
                yield SymbolFileTuple(
                    relative_path=os.path.basename(p), source_file_name=p
                )
        else:
            raise ValueError("Unexpected input to GatherSymbolFiles: ", p)


@contextlib.contextmanager
def RemoteExecution(use_goma: bool, use_remoteexec: bool) -> Iterator[None]:
    """A context manager to start goma or remoteexec instance.

    The context manager depending on the input argument will decide to start
    either the goma or remote exec instance.

    Args:
        use_goma: If true, start the goma instance.
        use_remoteexec: If true, start the remoteexec instance.

    Yields:
        Iterator.
    """
    goma_dir = Path(os.environ.get("GOMA_DIR", Path.home() / "goma"))
    goma_tmp_dir = os.environ.get("GOMA_TMP_DIR")
    glog_log_dir = os.environ.get("GLOG_log_dir")
    reclient_dir = os.environ.get("RECLIENT_DIR")
    reproxy_cfg_file = os.environ.get("REPROXY_CFG")
    remoteexec_instance = None
    goma_instance = None

    try:
        if use_remoteexec and reclient_dir and reproxy_cfg_file:
            logging.info("Starting RBE reproxy.")
            remoteexec_instance = remoteexec_util.Remoteexec(
                reclient_dir, reproxy_cfg_file
            )
        elif use_goma:
            logging.info("Starting goma compiler_proxy.")
            goma_instance = goma_lib.Goma(
                goma_dir,
                goma_tmp_dir,
                stage_name="BuildPackages",
                log_dir=glog_log_dir,
            )
    except ValueError:
        logging.warning("Remote execution initialization error.")

    try:
        if remoteexec_instance:
            remoteexec_instance.Start()
        elif goma_instance:
            goma_instance.Restart()
        yield
    finally:
        if remoteexec_instance:
            logging.info("Stopping RBE reproxy.")
            remoteexec_instance.Stop()
        elif goma_instance:
            logging.info("Stopping goma compiler_proxy.")
            goma_instance.Stop()


def ArchiveSysroot(
    chroot: "chroot_lib.Chroot",
    sysroot: "sysroot_lib.Sysroot",
    _build_target: "build_target_lib.BuildTarget",
    output_dir: os.PathLike,
) -> Optional[os.PathLike]:
    """Archive the given sysroot.

    Args:
        chroot: The chroot in which the sysroot exists.
        sysroot: Sysroot that needs to be archived.
        output_dir: Directory in which the generated archive will be placed.

    Returns:
        The archive file path or None if the archive directory doesnt exists.
    """
    sysroot_path = Path(chroot.full_path(sysroot.path))
    if not sysroot_path.is_dir():
        return None

    osutils.SafeMakedirs(output_dir)
    archive_path = Path(output_dir) / SYSROOT_ARCHIVE_FILE

    compression_type = cros_build_lib.CompressionExtToType(SYSROOT_ARCHIVE_FILE)
    cros_build_lib.CreateTarball(
        archive_path,
        sysroot_path,
        compression=compression_type,
        chroot=chroot.path,
        sudo=True,
    )

    return archive_path