build_packages.sh

#!/bin/bash

# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script is being deprecated and moved to chromite/ by 2022-06-30.
# Do not add any references to this script.
# Please reach out if you have any questions.

. "$(dirname "$0")/common.sh" || exit 1

if [[ "$1" != "--script-is-run-only-by-chromite-and-not-users" ]]; then
  die_notrace 'This script must not be run by users.' \
    'Please run `build_packages` from $PATH (in chromite/bin/) instead.'
fi
# Discard the magic marker flag.
shift

# Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \
  "The board to build packages for."
DEFINE_boolean usepkg "${FLAGS_TRUE}" \
  "Use binary packages to bootstrap when possible."
DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
  "Only use binary packages to bootstrap; abort if any are missing."
DEFINE_boolean workon "${FLAGS_TRUE}" \
  "Force-build workon packages."
DEFINE_boolean withrevdeps "${FLAGS_TRUE}" \
  "Calculate reverse dependencies on changed ebuilds."
DEFINE_boolean use_any_chrome "${FLAGS_TRUE}" \
  "Use any Chrome prebuilt available, even if the prebuilt doesn't match exactly."
DEFINE_boolean cleanbuild "${FLAGS_FALSE}" \
  "Perform a clean build; delete sysroot if it exists before building."
DEFINE_boolean pretend "${FLAGS_FALSE}" \
  "Don't build packages, just display which packages would have been installed."

# The --board_root flag specifies the environment variables ROOT and PKGDIR.
# This allows fetching and emerging of all packages to specified board_root.
# Note that --board_root will setup the board normally in /build/$BOARD, if it's
# not setup yet. It also expects the toolchain to already be installed in the
# board_root. --usepkgonly and --norebuild are required, because building is not
# supported when board_root is set.
# enforce this)."
DEFINE_string board_root "" \
  "Emerge packages to board_root."

# The following options are advanced options, only available to those willing
# to read the source code. They are not shown in help output, since they are
# not needed for the typical developer workflow.
DEFINE_integer jobs -1 \
  "How many packages to build in parallel at maximum."
DEFINE_boolean norebuild "${FLAGS_FALSE}" \
  "Don't automatically rebuild dependencies."

# The --reuse_pkgs_from_local_boards flag tells Portage to share binary
# packages between boards that are built locally, so that the total time
# required to build several boards is reduced. This flag is only useful
# when you are not able to use remote binary packages, since remote binary
# packages are usually more up to date than anything you have locally.
DEFINE_boolean reuse_pkgs_from_local_boards "${FLAGS_FALSE}" \
  "Bootstrap from local packages instead of remote packages."

# --run_goma option is designed to be used on bots.
# If you're trying to build pacakges with goma in your local dev env, this is
# *not* the option you're looking for. Please see comments below.
# This option; 1) starts goma, 2) builds packages (expecting that goma is
# used), then 3) stops goma explicitly.
# 3) is a request from the goma team, so that stats/logs can be taken.
# Note: GOMA_DIR and GOMA_SERVICE_ACCOUNT_JSON_FILE are expected to be passed
# via env var.
#
# In local dev env cases, compiler_proxy is expected to keep running.
# In such a case;
#   $ python ${GOMA_DIR}/goma_ctl.py ensure_start
#   $ ./build_packages (... and options without --run_goma ...)
# is an expected commandline sequence. If you set --run_goma flag while
# compiler_proxy is already running, the existing compiler_proxy will be
# stopped.
DEFINE_boolean run_goma "${FLAGS_FALSE}" \
  "If set to true, (re)starts goma, builds packages, and then stops goma."

# This option is for building chrome remotely.
#1) starts reproxy 2) builds chrome with reproxy and 3) stops reproxy so
# logs/stats can be collected.
# Note: RECLIENT_DIR and REPROXY_CFG are expected to be passed via env var.
DEFINE_boolean run_remoteexec "${FLAGS_FALSE}" \
  "If set to true, starts RBE reproxy, builds packages, and then stops reproxy."

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Die on any errors.
switch_to_strict_mode

# Chrome packages that need to be treated the same. These are the chrome and
# chrome follow-on packages that share the same version as chrome and are
# updated in lock step.
CHROME_PACKAGES=(
  "chromeos-base/chromeos-chrome"
  "chromeos-base/chrome-icu"
)

BOARD_ROOT="${FLAGS_board_root:-/build/${FLAGS_board}}"

# Skip revdeps when we don't already have a built sysroot.
if [[ "${FLAGS_cleanbuild}" -eq "${FLAGS_TRUE}" || ! -d "${BOARD_ROOT}" ]]; then
  SKIP_REVDEPS="${FLAGS_TRUE}"
else
  SKIP_REVDEPS="${FLAGS_FALSE}"
fi

# Setup all the emerge command/flags.
EMERGE_FLAGS=( -uDNv --backtrack=30 --newrepo --with-test-deps y )

EMERGE_CMD=(
  "${CHROMITE_BIN}/parallel_emerge"
  --board=${FLAGS_board}
)

if [[ "${FLAGS_use_any_chrome}" -eq "${FLAGS_TRUE}" ]]; then
  for pkg in "${CHROME_PACKAGES[@]}"; do
    EMERGE_CMD+=( "--force-remote-binary=${pkg}" )
  done
fi

EMERGE_CMD+=( ${EXTRA_BOARD_FLAGS} )

if [[ "${FLAGS_pretend}" -eq "${FLAGS_TRUE}" ]]; then
  EMERGE_FLAGS+=( "--pretend" )
fi

if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ||
      "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ||
      "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
  # Use binary packages. Include all build-time dependencies,
  # so as to avoid unnecessary differences between source
  # and binary builds.
  EMERGE_FLAGS+=( --getbinpkg --with-bdeps y )
  if [[ ${FLAGS_usepkgonly} -eq ${FLAGS_TRUE} ]]; then
    EMERGE_FLAGS+=( --usepkgonly )
  else
    EMERGE_FLAGS+=( --usepkg )
  fi
fi

if [[ "${FLAGS_jobs}" -ne -1 ]]; then
  EMERGE_FLAGS+=( --jobs=${FLAGS_jobs} )
fi

if [[ "${FLAGS_norebuild}" -eq "${FLAGS_FALSE}" ]]; then
  EMERGE_FLAGS+=( --rebuild-if-new-rev )
fi

# Figure out which packages we should be building.
PACKAGES=( "$@" )
FORCE_LOCAL_BUILD_PKGS=()
TEST_PACKAGES=( $(printf '%s\n' "${PACKAGES[@]}" | \
  grep "virtual/target-os-test" || :) )
if [[ ${#TEST_PACKAGES[@]} -gt 0 ]]; then
  # chromeos-ssh-testkeys may generate ssh keys if the right USE flag is set.
  # We force rebuilding this package from source every time, so that
  # consecutive builds don't share ssh keys.
  FORCE_LOCAL_BUILD_PKGS+=( chromeos-base/chromeos-ssh-testkeys )
fi

# Build cros_workon packages when they are changed.
CROS_WORKON_PKGS=()
if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then
  LIST_MODIFIED_PACKAGES="${CHROMITE_BIN}/cros_list_modified_packages"
  MODIFIED_PACKAGES=( $("${LIST_MODIFIED_PACKAGES}" --board=${FLAGS_board}) )
  info "cros_workon modified packages '${MODIFIED_PACKAGES[*]}' detected"
  CROS_WORKON_PKGS+=( "${MODIFIED_PACKAGES[@]}" )

  # TODO(anush): Make chrome a fake cros-workon package.
  if [[ -n "${CHROME_ORIGIN}" ]]; then
    CROS_WORKON_PKGS+=( "${CHROME_PACKAGES[@]}" )
  fi
fi

# cros_workon packages always have to be rebuilt.
FORCE_LOCAL_BUILD_PKGS+=( "${CROS_WORKON_PKGS[@]}" )

# Any package that directly depends on an active cros_workon package also needs
# to be rebuilt in order to be correctly built against the current set of
# changes a user may have made to the cros_workon package.
if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then
  # Collect all installed packages that depend on active cros_workon packages.
  WORKON_PKG_CONSUMERS=()
  mapfile -t WORKON_PKG_CONSUMERS < <( \
    equery-${FLAGS_board} -q depends "${CROS_WORKON_PKGS[@]}" | \
    sort -u | \
    grep -Ev "^\s*$" )

  # Transform this list of packages with versions in to a list of just
  # $CATEGORY/$NAME entries, since we don't want to pass packages with explicit
  # version numbers as arguments to `emerge`.
  if [[ ${#WORKON_PKG_CONSUMERS[@]} -gt 0 ]]; then
    WORKON_REBUILD_PKGS=()
    mapfile -t WORKON_REBUILD_PKGS < <( \
      equery-${FLAGS_board} list -p -o --format='$category/$name' \
        "${WORKON_PKG_CONSUMERS[@]}" | sort -u )

    info "The following packages depend directly on an active" \
      "cros_workon package and will be rebuilt: ${WORKON_REBUILD_PKGS[*]}"

    FORCE_LOCAL_BUILD_PKGS+=( "${WORKON_REBUILD_PKGS[@]}" )
  fi
fi

if [[ -n "${FLAGS_board_root}" ]]; then
  export ROOT="${FLAGS_board_root}"
  export PORTAGE_CONFIGROOT="${ROOT}"
  export SYSROOT="${ROOT}"
  export PKGDIR="${ROOT}"/packages
fi

# Temporarily modify the emerge flags so we can calculate the revdeps
# on the modified packages.
if [[ "${FLAGS_withrevdeps}" -eq "${FLAGS_TRUE}" &&
      "${SKIP_REVDEPS}" -eq "${FLAGS_FALSE}" ]]; then
  info "starting reverse dependency calculations ..."
  SIM_EMERGE_FLAGS=( "${EMERGE_FLAGS[@]}" --pretend --columns )

  if [[ ${#PACKAGES[@]} -gt 0 ]]; then
    SIM_EMERGE_FLAGS+=(
      --reinstall-atoms="${PACKAGES[*]}"
      --usepkg-exclude="${PACKAGES[*]}"
    )
  fi

  # cros-workon packages are always going to be force reinstalled, so we add
  # the forced reinstall behavior to the modified package calculation. This is
  # necessary to include when a user has already installed a 9999 ebuild and is
  # now reinstalling that package with additional local changes, because
  # otherwise the modified package calculation would not see that a 'new'
  # package is being installed.
  if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then
    SIM_EMERGE_FLAGS+=(
      --reinstall-atoms="${CROS_WORKON_PKGS[*]}"
      --usepkg-exclude="${CROS_WORKON_PKGS[*]}"
    )
  fi

  # Calculate only the ebuild changes from the emerge simulation ignoring
  # the virtual packages and the forced rebuild of autotest-all package.
  # The lines of the following block do the following operations:
  # 1. Do a pretend `emerge` command to get a list of what would be built.
  # 2. Filter to only packages that will be installed to the board sysroot.
  # 3. Filter to only packages that would be built from source and rewrite the
  #    line from Portage's full output to only $CATEGORY/$PACKAGE
  # 4. Filter the list of packages to a heuristic set of packages known to have
  #    incorrectly specified dependencies.
  # 5. Sort the output and remove any duplicate entries.
  BASE_INSTALL_PKGS=( $( \
    sudo -E "${EMERGE_CMD[@]}" "${SIM_EMERGE_FLAGS[@]}" "${PACKAGES[@]}" | \
    grep -e 'to /build/' | \
    sed -n -E '/^\[ebuild /{s:^[^]]+\] +::;s: .*::;p}' | \
    grep -E '/(coreboot-private-files.*|tast-build-deps)$' | \
    sort -u ) )

  MOD_PKGS=()
  if [[ "${#BASE_INSTALL_PKGS[@]}" -gt 0 ]]; then
    info "Forced rebuild packages detected: ${BASE_INSTALL_PKGS[*]}."
    # Convert specific versions into base package names
    MOD_PKGS+=( $(\
    equery-${FLAGS_board} list -p -o --format='$category/$name' \
      "${BASE_INSTALL_PKGS[@]}" | sort -u ) )
    # Remove Chrome as rebuilding it is expensive and almost never makes sense.
    # Ignore grep exit status in case chromeos-chrome is the only package.
    grep_cmd=( grep -v )
    for pkg in "${CHROME_PACKAGES[@]}"; do
      grep_cmd+=( -e "${pkg}" )
    done
    MOD_PKGS=( $(printf '%s\n' "${MOD_PKGS[@]}" | "${grep_cmd[@]}" || :) )
  fi

  FORCE_LOCAL_BUILD_PKGS+=( "${MOD_PKGS[@]}" )

  if [[ "${#MOD_PKGS[@]}" -gt 0 ]]; then
    info "calculating reverse dependencies on packages: ${MOD_PKGS[*]}"
    REV_DEPS=( $(\
      equery-${FLAGS_board} -q depends --indirect "${MOD_PKGS[@]}" |\
      awk '{print $1}' | grep -v ^virtual/ | sort -u) )
    if [[ "${#REV_DEPS[@]}" -gt 0 ]]; then
      # Convert specific versions into base package names
      RMOD_PKGS=( $(\
        equery-${FLAGS_board} -q list -p -o --format='$category/$name' \
        "${REV_DEPS[@]}" | sort -u ) )
      # Remove Chrome as rebuilding it is expensive and almost never makes
      # sense.  Ignore grep exit status in case chromeos-chrome is the only
      # package.
      grep_cmd=( grep -v )
      for pkg in "${CHROME_PACKAGES[@]}"; do
        grep_cmd+=( -e "${pkg}" )
      done
      RMOD_PKGS=( $(printf '%s\n' "${RMOD_PKGS[@]}" | "${grep_cmd[@]}" || :) )
      info "final reverse dependencies that will be rebuilt: ${RMOD_PKGS[*]}"
      FORCE_LOCAL_BUILD_PKGS+=( "${RMOD_PKGS[@]}" )
    fi
  fi
fi # end FLAGS_withrevdeps

if [[ ${#FORCE_LOCAL_BUILD_PKGS[@]} -gt 0 ]]; then
  EMERGE_FLAGS+=(
    --reinstall-atoms="${FORCE_LOCAL_BUILD_PKGS[*]}"
    --usepkg-exclude="${FORCE_LOCAL_BUILD_PKGS[*]}"
  )
fi

# A list of critical system packages that should never be incidentally
# reinstalled as a side effect of build_packages. All packages in this list
# are special cased to prefer matching installed versions, overriding the
# typical logic of upgrading to the newest available version.
#
# This list can't include any package that gets installed to a board!
# Packages such as LLVM or binutils must not be in this list as the normal
# rebuild logic must still apply to them for board targets.
#
# TODO(crbug/1050752): Remove this list and the corresponding arguments
# to `emerge` below once we figure out how to exclude toolchain packages from
# being upgraded transitively via BDEPEND relations.
CRITICAL_SDK_PACKAGES=(
  "dev-embedded/hps-sdk"
  "dev-lang/rust"
  "dev-lang/go"
  "sys-libs/glibc"
  "sys-devel/gcc"
)

info "Merging board packages now"
(

  # Start reproxy for remote execution of building chrome.
  if [[ "${FLAGS_run_remoteexec}" -eq "${FLAGS_TRUE}" ]]; then
    info "Starting RBE reproxy."
    bootstrap="${RECLIENT_DIR}/bootstrap --cfg=${REPROXY_CFG} \
      --re_proxy=${RECLIENT_DIR}/reproxy"
    ${bootstrap}
    trap "${bootstrap} --shutdown" EXIT
  # Support goma on bots. This has to run in subshell, otherwise EXIT trap
  # handler is overwritten.
  elif [[ "${FLAGS_run_goma}" -eq "${FLAGS_TRUE}" ]]; then
    info "Starting goma compiler_proxy."
    goma_ctl="${GOMA_DIR:-${HOME}/goma}/goma_ctl.py"
    "${goma_ctl}" restart
    trap "'${goma_ctl}' stop" EXIT
  fi

  info_run sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" \
    --useoldpkg-atoms="${CRITICAL_SDK_PACKAGES[*]}" \
    --rebuild-exclude="${CRITICAL_SDK_PACKAGES[*]}"
)

echo "Builds complete"

command_completed
echo "Done"