Merge "Merge cos-sdk into ToT" into main
diff --git a/OWNERS.kernel b/OWNERS.kernel
index 0dd4ce5..5cf9ce5 100644
--- a/OWNERS.kernel
+++ b/OWNERS.kernel
@@ -2,7 +2,4 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-# Kernel folks, mostly for update_kernel.sh.
-dianders@chromium.org
-groeck@chromium.org
-briannorris@chromium.org
+file:chromiumos/owners:v1:/kernel/OWNERS.build
diff --git a/build_image b/build_image
index edeb5ce..100ed27 100755
--- a/build_image
+++ b/build_image
@@ -1,274 +1,17 @@
#!/bin/bash
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-# Script to build a bootable keyfob-based chromeos system image from within
-# a chromiumos setup. This assumes that all needed packages have been built into
-# the given target's root with binary packages turned on. This script will
-# build the Chrome OS image using only pre-built binary packages.
+. "$(dirname "$0")/common.sh" || exit 1
-SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")"
-# shellcheck source=build_library/build_common.sh
-. "${SCRIPT_ROOT}/build_library/build_common.sh" || exit 1
-
-# Make sure we run with network disabled to prevent leakage.
-if [[ -z ${UNSHARE} ]]; then
- if [[ $(id -u) -ne 0 ]]; then
- exec sudo -E env PATH="${PATH}" "$0" "$@"
- fi
- exec unshare -n -- sudo -E UNSHARE=true -u "${SUDO_USER}" -- "$0" "$@"
-fi
-
-# Developer-visible flags.
-DEFINE_string adjust_part "" \
- "Adjustments to apply to partition table (LABEL:[+-=]SIZE) e.g. ROOT-A:+1G"
-DEFINE_string board "${DEFAULT_BOARD}" \
- "The board to build an image for."
-DEFINE_string boot_args "noinitrd" \
- "Additional boot arguments to pass to the commandline"
-DEFINE_boolean enable_bootcache ${FLAGS_FALSE} \
- "Default all bootloaders to NOT use boot cache."
-DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
- "Default all bootloaders to use kernel-based root fs integrity checking." \
- r
-DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
- "Directory in which to place image result directories (named by version)"
-DEFINE_string disk_layout "default" \
- "The disk layout type to use for this image."
-DEFINE_string enable_serial "" \
- "Enable serial port for printks. Example values: ttyS0"
-DEFINE_integer loglevel 7 \
- "The loglevel to add to the kernel command line."
-DEFINE_string builder_path "" \
- "The build_name to be installed on DUT during hwtest."
-
-
-FLAGS_HELP="USAGE: build_image [flags] [list of images to build].
-This script is used to build a Chromium OS image. Chromium OS comes in many
-different forms. This scripts can be used to build the following:
-
-base - Pristine Chromium OS image. As similar to Chrome OS as possible.
-dev [default] - Developer image. Like base but with additional dev packages.
-test - Like dev, but with additional test specific packages and can be easily
- used for automated testing using scripts like test_that, etc.
-factory_install - Install shim for bootstrapping the factory test process.
- Cannot be built along with any other image.
-
-Examples:
-
-build_image --board=<board> dev test - builds developer and test images.
-build_image --board=<board> factory_install - builds a factory install shim.
-
-Note if you want to build an image with custom size partitions, either consider
-adding a new disk layout in build_library/legacy_disk_layout.json OR use
-adjust_part. See the help above but here are a few examples:
-
-adjust_part='STATE:+1G' -- add one GB to the size the stateful partition
-adjust_part='ROOT-A:-1G' -- remove one GB from the primary rootfs partition
-adjust_part='STATE:=1G' -- make the stateful partition 1 GB
-...
-"
-
-# The following options are advanced options, only available to those willing
-# to read the source code. They are not shown in help output, since they are
-# not needed for the typical developer workflow.
-DEFINE_integer build_attempt 1 \
- "The build attempt for this image build."
-DEFINE_string build_root "${DEFAULT_BUILD_ROOT}/images" \
- "Directory in which to compose the image, before copying it to output_root."
-DEFINE_integer jobs -1 \
- "How many packages to build in parallel at maximum."
-DEFINE_boolean replace ${FLAGS_FALSE} \
- "Overwrite existing output, if any."
-DEFINE_string symlink "latest" \
- "Symlink name to use for this image."
-DEFINE_string version "" \
- "Overrides version number in name to this version."
-DEFINE_string output_suffix "" \
- "Add custom suffix to output directory."
-DEFINE_boolean eclean ${FLAGS_TRUE} \
- "Do NOT call eclean before building the image (default is to call eclean)."
-
-# Parse command line.
-FLAGS "$@" || exit 1
-
-# See if we want to default the bootcache flag before we clobber
-# the user's command line. We want to default to false if the
-# user explicitly disabled rootfs verification otherwise they
-# have to manually specify both.
-FLAGS_bootcache_use_board_default=${FLAGS_enable_rootfs_verification}
-case " $* " in
- *" --enable_bootcache "*|\
- *" --noenable_bootcache "*)
- FLAGS_bootcache_use_board_default=${FLAGS_FALSE}
- ;;
-esac
-
-eval set -- "${FLAGS_ARGV}"
-
-# Only now can we die on error. shflags functions leak non-zero error codes,
-# so will die prematurely if 'switch_to_strict_mode' is specified before now.
-switch_to_strict_mode
-
-# Determine build version.
-OVERLAY_CHROMEOS_DIR="${SRC_ROOT}/third_party/chromiumos-overlay/chromeos"
-# shellcheck source=../third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh
-. "${OVERLAY_CHROMEOS_DIR}/config/chromeos_version.sh" || exit 1
-# N.B. Ordering matters for some of the libraries below, because
-# some of the files contain initialization used by later files.
-# shellcheck source=build_library/board_options.sh
-. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
-# shellcheck source=build_library/disk_layout_util.sh
-. "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1
-# shellcheck source=build_library/mount_gpt_util.sh
-. "${BUILD_LIBRARY_DIR}/mount_gpt_util.sh" || exit 1
-# shellcheck source=build_library/build_image_util.sh
-. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
-# shellcheck source=build_library/base_image_util.sh
-. "${BUILD_LIBRARY_DIR}/base_image_util.sh" || exit 1
-# shellcheck source=build_library/dev_image_util.sh
-. "${BUILD_LIBRARY_DIR}/dev_image_util.sh" || exit 1
-# shellcheck source=build_library/test_image_util.sh
-. "${BUILD_LIBRARY_DIR}/test_image_util.sh" || exit 1
-# shellcheck source=build_library/selinux_util.sh
-. "${BUILD_LIBRARY_DIR}/selinux_util.sh" || exit 1
-
-parse_build_image_args
-
-load_board_specific_script "board_specific_setup.sh"
-
-sudo_clear_shadow_locks "/build/${FLAGS_board}"
-
-# TODO: <prebuild hook>
-
-BASE_PACKAGE="virtual/target-os"
-# Tweak flags, configure extra USE flags, and set base packages for the factory
-# install shim.
-if should_build_image "${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}"; then
- # TODO: Build a separated ebuild for the install shim to reduce size.
- INSTALL_MASK="${FACTORY_SHIM_INSTALL_MASK}"
-
- # Add the cros_factory_install boot arg.
- FLAGS_boot_args="${FLAGS_boot_args} cros_factory_install"
-
- BASE_PACKAGE="virtual/target-os-factory-shim"
-
- export USE="${USE} fbconsole vtconsole factory_shim_ramfs i2cdev vfat"
-fi
-
-# Mask systemd directories if this is not a systemd image.
-if ! has "systemd" "$("portageq-${FLAGS_board}" envvar USE)"; then
- COMMON_INSTALL_MASK+=" ${SYSTEMD_INSTALL_MASK}"
- DEFAULT_INSTALL_MASK+=" ${SYSTEMD_INSTALL_MASK}"
- FACTORY_SHIM_INSTALL_MASK+=" ${SYSTEMD_INSTALL_MASK}"
- INSTALL_MASK+=" ${SYSTEMD_INSTALL_MASK}"
-fi
-
-# TODO: </prebuild hook>
-
-# If we are creating a developer image, also create a pristine image with a
-# different name.
-PRISTINE_IMAGE_NAME=
-if should_build_image "${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}"; then
- PRISTINE_IMAGE_NAME="${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}"
-else
- PRISTINE_IMAGE_NAME="${CHROMEOS_BASE_IMAGE_NAME}"
-fi
-
-if [[ ${FLAGS_eclean} -eq ${FLAGS_TRUE} ]]; then
- "${BOARD_ROOT}/build/bin/eclean" -d packages
-fi
-
-# Handle existing directory.
-if [[ -e "${BUILD_DIR}" ]]; then
- if [[ ${FLAGS_replace} -eq ${FLAGS_TRUE} ]]; then
- sudo rm -rf "${BUILD_DIR}"
- else
- error "Directory ${BUILD_DIR} already exists."
- error "Use --build_attempt option to specify an unused attempt."
- error "Or use --replace if you want to overwrite this directory."
- die "Unwilling to overwrite ${BUILD_DIR}."
- fi
-fi
-
-# Create the output directory and temporary mount points.
-mkdir -p "${BUILD_DIR}"
-
-# Create the base image.
-create_base_image "${PRISTINE_IMAGE_NAME}" \
- "${FLAGS_enable_rootfs_verification}" "${FLAGS_enable_bootcache}"
-
-# Running board-specific setup if any exists.
-if type board_setup &>/dev/null; then
- board_setup "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
-fi
-
-# Create a developer image if an image that is based on it is requested.
-if should_build_image "${CHROMEOS_DEVELOPER_IMAGE_NAME}" \
- "${CHROMEOS_TEST_IMAGE_NAME}"; then
- copy_image "${CHROMEOS_BASE_IMAGE_NAME}" "${CHROMEOS_DEVELOPER_IMAGE_NAME}"
- install_dev_packages "${CHROMEOS_DEVELOPER_IMAGE_NAME}"
-fi
-
-# From a developer image create a test image.
-if should_build_image "${CHROMEOS_TEST_IMAGE_NAME}"; then
- copy_image "${CHROMEOS_DEVELOPER_IMAGE_NAME}" "${CHROMEOS_TEST_IMAGE_NAME}"
- mod_image_for_test "${CHROMEOS_TEST_IMAGE_NAME}"
-fi
-
-# Move the completed image to the output_root.
-move_image "${BUILD_DIR}" "${OUTPUT_DIR}"
-
-# Copy DLC images to the output_root directory.
-build_dlc --sysroot="${BOARD_ROOT}" --install-root-dir="${OUTPUT_DIR}/dlc" \
- --board="${BOARD}"
-
-# Create a named symlink.
-LINK_NAME="${FLAGS_output_root}/${BOARD}/${FLAGS_symlink}"
-ln -sfT "$(basename "${OUTPUT_DIR}")" "${LINK_NAME}"
-
-info "Done. Image(s) created in ${OUTPUT_DIR}"
-echo
-
-# Print out the images we generated.
-summarize() {
- local name="$1" img="$2"
- local dir_path
-
- # __flags_output_root_default defined in shflags.
- # shellcheck disable=2154
- if [[ "${FLAGS_output_root}" == "${__flags_output_root_default}" ]]; then
- dir_path=${OUTSIDE_OUTPUT_DIR}
- else
- dir_path="${OUTPUT_DIR}"
- fi
-
- info "${name} image created as ${img}"
- info "To copy the image to a USB key, use:"
- info " cros flash usb:// ${dir_path}/${img}"
- info "To flash the image to a Chrome OS device, use:"
- info " cros flash YOUR_DEVICE_IP ${dir_path}/${img}"
- info "Note that the device must be accessible over the network."
- info "A base image will not work in this mode, but a test or dev image will."
- if [[ $# -ge 3 ]]; then
- info "To run the image in a virtual machine, use:"
- info " cros_vm --start --image-path=${dir_path}/${img} --board=${BOARD}"
- fi
- echo
-}
-if should_build_image "${CHROMEOS_BASE_IMAGE_NAME}"; then
- summarize "Non-developer Chromium OS" "${PRISTINE_IMAGE_NAME}"
-fi
-if should_build_image "${CHROMEOS_FACTORY_SHIM_NAME}"; then
- summarize "Chromium OS Factory install shim" "${PRISTINE_IMAGE_NAME}"
-fi
-if should_build_image "${CHROMEOS_DEVELOPER_IMAGE_NAME}"; then
- summarize "Developer" "${CHROMEOS_DEVELOPER_IMAGE_NAME}" ""
-fi
-if should_build_image "${CHROMEOS_TEST_IMAGE_NAME}"; then
- summarize "Test" "${CHROMEOS_TEST_IMAGE_NAME}" "--test"
-fi
-
-command_completed
+new_script="build_image"
+warn "$0: This script is deprecated and will be removed."
+warn "All users must migrate to ${new_script} in chromite/bin."
+warn "You can simply change all references of $0 to \`${new_script}\`" \
+ "from \$PATH (in chromite/bin/)."
+warn "This old script will be removed by January 2023."
+warn "If you have questions or found code that needs updating, please" \
+ "contact chromium-os-dev@, or file a bug at go/cros-build-bug."
+exec "${CHROMITE_BIN}/${new_script}" "$@"
diff --git a/build_image.sh b/build_image.sh
new file mode 100755
index 0000000..c966a79
--- /dev/null
+++ b/build_image.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Script to build a bootable keyfob-based chromeos system image from within
+# a chromiumos setup. This assumes that all needed packages have been built into
+# the given target's root with binary packages turned on. This script will
+# build the Chrome OS image using only pre-built binary packages.
+
+SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")"
+# shellcheck source=build_library/build_common.sh
+. "${SCRIPT_ROOT}/build_library/build_common.sh" || exit 1
+
+if [[ "$1" != "--script-is-run-only-by-chromite-and-not-users" ]]; then
+ die_notrace 'This script must not be run by users.' \
+ 'Please run `build_images` from $PATH (in chromite/bin/) instead.'
+fi
+
+# Make sure we run with network disabled to prevent leakage.
+if [[ -z ${UNSHARE} ]]; then
+ if [[ $(id -u) -ne 0 ]]; then
+ exec sudo -E env PATH="${PATH}" "$0" "$@"
+ fi
+ exec unshare -n -- sudo -E UNSHARE=true -u "${SUDO_USER}" -- "$0" "$@"
+fi
+
+# Discard the 'script-is-run-only-by-chromite-and-not-users' flag.
+shift
+
+# Developer-visible flags.
+DEFINE_string adjust_part "" \
+ "Adjustments to apply to partition table (LABEL:[+-=]SIZE) e.g. ROOT-A:+1G"
+DEFINE_string board "${DEFAULT_BOARD}" \
+ "The board to build an image for."
+DEFINE_string boot_args "noinitrd" \
+ "Additional boot arguments to pass to the commandline"
+DEFINE_boolean enable_bootcache ${FLAGS_FALSE} \
+ "Default all bootloaders to NOT use boot cache."
+DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
+ "Default all bootloaders to use kernel-based root fs integrity checking." \
+ r
+DEFINE_string disk_layout "default" \
+ "The disk layout type to use for this image."
+DEFINE_string enable_serial "" \
+ "Enable serial port for printks. Example values: ttyS0"
+DEFINE_integer loglevel 7 \
+ "The loglevel to add to the kernel command line."
+DEFINE_string builder_path "" \
+ "The build_name to be installed on DUT during hwtest."
+
+
+FLAGS_HELP="USAGE: build_image [flags] [list of images to build].
+This script is used to build a Chromium OS image. Chromium OS comes in many
+different forms. This scripts can be used to build the following:
+
+base - Pristine Chromium OS image. As similar to Chrome OS as possible.
+dev [default] - Developer image. Like base but with additional dev packages.
+test - Like dev, but with additional test specific packages and can be easily
+ used for automated testing using scripts like test_that, etc.
+factory_install - Install shim for bootstrapping the factory test process.
+ Cannot be built along with any other image.
+
+Examples:
+
+build_image --board=<board> dev test - builds developer and test images.
+build_image --board=<board> factory_install - builds a factory install shim.
+
+Note if you want to build an image with custom size partitions, either consider
+adding a new disk layout in build_library/legacy_disk_layout.json OR use
+adjust_part. See the help above but here are a few examples:
+
+adjust_part='STATE:+1G' -- add one GB to the size the stateful partition
+adjust_part='ROOT-A:-1G' -- remove one GB from the primary rootfs partition
+adjust_part='STATE:=1G' -- make the stateful partition 1 GB
+...
+"
+
+# The following options are advanced options, only available to those willing
+# to read the source code. They are not shown in help output, since they are
+# not needed for the typical developer workflow.
+DEFINE_integer jobs -1 \
+ "How many packages to build in parallel at maximum."
+
+# Parse command line.
+FLAGS "$@" || exit 1
+
+eval set -- "${FLAGS_ARGV}"
+
+# Only now can we die on error. shflags functions leak non-zero error codes,
+# so will die prematurely if 'switch_to_strict_mode' is specified before now.
+switch_to_strict_mode
+
+# N.B. Ordering matters for some of the libraries below, because
+# some of the files contain initialization used by later files.
+# shellcheck source=build_library/board_options.sh
+. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+# shellcheck source=build_library/disk_layout_util.sh
+. "${BUILD_LIBRARY_DIR}/disk_layout_util.sh" || exit 1
+# shellcheck source=build_library/mount_gpt_util.sh
+. "${BUILD_LIBRARY_DIR}/mount_gpt_util.sh" || exit 1
+# shellcheck source=build_library/build_image_util.sh
+. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+# shellcheck source=build_library/base_image_util.sh
+. "${BUILD_LIBRARY_DIR}/base_image_util.sh" || exit 1
+# shellcheck source=build_library/dev_image_util.sh
+. "${BUILD_LIBRARY_DIR}/dev_image_util.sh" || exit 1
+# shellcheck source=build_library/test_image_util.sh
+. "${BUILD_LIBRARY_DIR}/test_image_util.sh" || exit 1
+# shellcheck source=build_library/selinux_util.sh
+. "${BUILD_LIBRARY_DIR}/selinux_util.sh" || exit 1
+
+IMAGES_TO_BUILD="$*"
+
+load_board_specific_script "board_specific_setup.sh"
+
+sudo_clear_shadow_locks "/build/${FLAGS_board}"
+
+# TODO: <prebuild hook>
+
+# Create the base image.
+create_base_image "${PRISTINE_IMAGE_NAME}" \
+ "${FLAGS_enable_rootfs_verification}" "${FLAGS_enable_bootcache}"
+
+# Running board-specific setup if any exists.
+if type board_setup &>/dev/null; then
+ board_setup "${BUILD_DIR}/${PRISTINE_IMAGE_NAME}"
+fi
+
+# Create a developer image if an image that is based on it is requested.
+if should_build_image "${CHROMEOS_DEVELOPER_IMAGE_NAME}" \
+ "${CHROMEOS_TEST_IMAGE_NAME}"; then
+ copy_image "${CHROMEOS_BASE_IMAGE_NAME}" "${CHROMEOS_DEVELOPER_IMAGE_NAME}"
+ install_dev_packages "${CHROMEOS_DEVELOPER_IMAGE_NAME}"
+fi
+
+# From a developer image create a test image.
+if should_build_image "${CHROMEOS_TEST_IMAGE_NAME}"; then
+ copy_image "${CHROMEOS_DEVELOPER_IMAGE_NAME}" "${CHROMEOS_TEST_IMAGE_NAME}"
+ mod_image_for_test "${CHROMEOS_TEST_IMAGE_NAME}"
+fi
diff --git a/build_library/base_image_util.sh b/build_library/base_image_util.sh
index ef44879..849a3e3 100755
--- a/build_library/base_image_util.sh
+++ b/build_library/base_image_util.sh
@@ -147,8 +147,11 @@
sudo mkdir -p "${BOARD_ROOT}/build/dev-install"
sudo mv "${pkgs_out}/package.installable" "${BOARD_ROOT}/build/dev-install/"
+ local package_provided_dir="${root_fs_dir}/usr/share/dev-install/"
+ package_provided_dir+="portage/make.profile/package.provided"
+
sudo mkdir -p \
- "${root_fs_dir}/usr/share/dev-install/portage/make.profile/package.provided" \
+ "${package_provided_dir}" \
"${root_fs_dir}/usr/share/dev-install/rootfs.provided"
sudo cp "${pkgs_out}/bootstrap.packages" \
"${root_fs_dir}/usr/share/dev-install/"
@@ -157,13 +160,14 @@
# Copy the toolchain settings which are fixed at build_image time.
sudo cp "${BOARD_ROOT}/etc/portage/profile/package.provided" \
- "${root_fs_dir}/usr/share/dev-install/portage/make.profile/package.provided/"
+ "${package_provided_dir}/toolchain.conf"
# Copy the profile stubbed packages which are always disabled for the board.
# TODO(vapier): This doesn't currently respect the profile or its parents.
+ local overlay="${CHROOT_TRUNK_DIR}/src/third_party/chromiumos-overlay"
sudo cp \
- "/usr/local/portage/chromiumos/profiles/targets/chromeos/package.provided" \
- "${root_fs_dir}/usr/share/dev-install/portage/make.profile/package.provided/"
+ "${overlay}/profiles/targets/chromeos/package.provided" \
+ "${package_provided_dir}/board-profile.conf"
rm -r "${pkgs_out}"
}
@@ -187,38 +191,93 @@
fi
}
-install_libc() {
- root_fs_dir="$1"
+install_libc_for_abi() {
# We need to install libc manually from the cross toolchain.
# TODO: Improve this? It would be ideal to use emerge to do this.
- libc_version="$(_get_variable "${BOARD_ROOT}/${SYSROOT_SETTINGS_FILE}" \
- "LIBC_VERSION")"
+ local root_fs_dir="$1"
+ local abi="$2"
PKGDIR="/var/lib/portage/pkgs"
- local libc_atom="cross-${CHOST}/glibc-${libc_version}"
- LIBC_PATH="${PKGDIR}/${libc_atom}.tbz2"
- if [[ ! -e ${LIBC_PATH} ]]; then
- sudo emerge --nodeps -gf "=${libc_atom}"
+ local pkgs=( "glibc" )
+ local pkg_ver_str=( "LIBC_VERSION" )
+ local exclude_gconv=false
+ if [[ "$3" == "--copy-cxx-runtime" ]]; then
+ # Normally only glibc gets installed here since it is not built per board
+ # while other "standard" libraries like the C++ libraries get installed by
+ # ebuilds. In the case of NaCl on Arm64, we need to install a second set of
+ # "standard" libraries for Arm32 to run NaCl binaries in Arm32 mode.
+ # So we'll install glibc and C++ libraries here using prebuilts for host
+ # cross-<abi>/<library> packages and not rely on ebuilds.
+ # This is a bit hacky but is only for a "short" while.
+ pkgs+=( "llvm-libunwind" "libcxx" )
+ pkg_ver_str+=("LIBGCC_VERSION" "LIBCXX_VERSION")
+ # gconv_strip.py does not like two set of gconv files but for
+ # arm32 + aarch64 multilib for NaCl, we have two set of files.
+ # As long as the second set for extra ABI is not needed for NaCl, we can
+ # just skip installing them for the arm32 abi while keeping aarch64.
+ exclude_gconv=true
fi
- # Strip out files we don't need in the final image at runtime.
- local libc_excludes=(
- # Compile-time headers.
- 'usr/include' 'sys-include'
- # Link-time objects.
- '*.[ao]'
- # Debug commands not used by normal runtime code.
- 'usr/bin/'{getent,ldd}
- # LD_PRELOAD objects for debugging.
- 'lib*/lib'{memusage,pcprofile,SegFault}.so 'usr/lib*/audit'
- # We only use files & dns with nsswitch, so throw away the others.
- 'lib*/libnss_'{compat,db,hesiod,nis,nisplus}'*.so*'
- # This is only for very old packages which we don't have.
- 'lib*/libBrokenLocale*.so*'
- )
- pbzip2 -dc --ignore-trailing-garbage=1 "${LIBC_PATH}" | \
- sudo tar xpf - -C "${root_fs_dir}" ./usr/${CHOST} \
- --strip-components=3 "${libc_excludes[@]/#/--exclude=}"
+ local idx pkg pkg_version pkg_atom pkg_path pkg_excludes pkg_comp
+ for (( idx = 0; idx < ${#pkgs[@]}; idx++ )); do
+ pkg="${pkgs[${idx}]}"
+ pkg_version="$(_get_variable "${BOARD_ROOT}/${SYSROOT_SETTINGS_FILE}" \
+ "${pkg_ver_str[${idx}]}")"
+ pkg_atom="cross-${abi}/${pkg}-${pkg_version}"
+ pkg_path="${PKGDIR}/${pkg_atom}.tbz2"
+ pkg_decompressor="$(detect_decompression_tool "${pkg_path}")"
+ if [[ "${pkg}" == "glibc" ]]; then
+ # LIBC_PATH needs to be export for later use in dev_image_util.sh
+ # to extract debug symbols.
+ # TODO(b/223285139): Change to export an array of packages instead
+ # since we should have really extract debug info for all of them.
+ export LIBC_PATH="${pkg_path}"
+ export LIBC_DECOMPRESSOR="${pkg_decompressor}"
+ fi
+
+ if [[ ! -e "${pkg_path}" ]]; then
+ sudo emerge --nodeps -gf "=${pkg_atom}"
+ fi
+
+ # Strip out files we don't need in the final image at runtime.
+ pkg_excludes=(
+ # Compile-time headers.
+ 'usr/include' 'sys-include'
+ # Link-time objects.
+ '*.[ao]'
+ # Debug commands not used by normal runtime code.
+ 'usr/bin/'{getent,ldd}
+ # LD_PRELOAD objects for debugging.
+ 'lib*/lib'{memusage,pcprofile,SegFault}.so 'usr/lib*/audit'
+ # We only use files & dns with nsswitch, so throw away the others.
+ 'lib*/libnss_'{compat,db,hesiod,nis,nisplus}'*.so*'
+ # This is only for very old packages which we don't have.
+ 'lib*/libBrokenLocale*.so*'
+ )
+ if [[ "${exclude_gconv}" == true ]]; then
+ pkg_excludes+=(
+ 'usr/lib/gconv'
+ 'usr/lib64/gconv'
+ )
+ fi
+ info_run sudo tar -I"${pkg_decompressor}" -xpf "${pkg_path}" \
+ -C "${root_fs_dir}" "./usr/${abi}" \
+ --strip-components=3 "${pkg_excludes[@]/#/--exclude=}"
+ done
+}
+
+install_libc() {
+ local root_fs_dir="$1"
+
+ # Until nacl goes away completely, we actually need to run it in arm32
+ # mode on arm64 hosts. This will get the basic C/C++ libraries installed
+ # onto the device.
+ if [[ "${CHOST}" == "aarch64-cros-linux-gnu" ]]; then
+ install_libc_for_abi "${root_fs_dir}" "armv7a-cros-linux-gnueabihf" \
+ "--copy-cxx-runtime"
+ fi
+
+ install_libc_for_abi "${root_fs_dir}" "${CHOST}"
}
create_base_image() {
@@ -288,7 +347,7 @@
info "Generating license credits page. Time:"
sudo mkdir -p "${root_fs_dir}/opt/google/chrome/resources"
local license_path="${root_fs_dir}/opt/google/chrome/resources/about_os_credits.html"
- time sudo "${GCLIENT_ROOT}/chromite/licensing/licenses" \
+ time info_run sudo "${GCLIENT_ROOT}/chromite/licensing/licenses" \
--board="${BOARD}" \
--log-level error \
--generate-licenses \
@@ -305,7 +364,7 @@
# and we don't known which ones will be used until all the applications are
# installed. This script looks for the charset names on all the binaries
# installed on the the ${root_fs_dir} and removes the unreferenced ones.
- sudo "${CHROMITE_BIN}/gconv_strip" "${root_fs_dir}"
+ info_run sudo "${CHROMITE_BIN}/gconv_strip" "${root_fs_dir}"
# Run ldconfig to create /etc/ld.so.cache.
run_ldconfig "${root_fs_dir}"
@@ -339,19 +398,25 @@
# into /usr/local later on. Create symlinks in the rootfs so python still
# works even when not in the rootfs. This is needed because Gentoo creates
# wrappers with hardcoded paths to the rootfs (e.g. python-exec).
+ local pyversions=(
+ # Querying versions is a bit fun. We don't know precisely what will be
+ # installed in /usr/local, so just query what is available in the sysroot.
+ # The qlist output is: dev-lang/python:3.6
+ $("qlist-${BOARD}" -ICSe 'dev-lang/python' | cut -d: -f2)
+ )
local path python_paths=(
"/etc/env.d/python"
+ "${pyversions[@]/#//usr/lib/python}"
"/usr/lib/python-exec"
"/usr/lib/portage"
"/usr/bin/python"
- "/usr/bin/python2"
"/usr/bin/python3"
- # Querying versions is a bit fun. We don't know precisely what will be
- # installed in /usr/local, so just query what is available in the sysroot.
- # The qlist output is: dev-lang/python:2.7
- $("qlist-${BOARD}" -ICSe dev-lang/python | \
- tr -d ':' | sed 's:dev-lang:/usr/bin:')
+ "/usr/bin/python-exec2c"
+ "${pyversions[@]/#//usr/bin/python}"
)
+ if [[ -d "${root_fs_dir}/lib64" ]]; then
+ python_paths+=( "${pyversions[@]/#//usr/lib64/python}" )
+ fi
for path in "${python_paths[@]}"; do
if [[ ! -e "${root_fs_dir}${path}" && ! -L "${root_fs_dir}${path}" ]]; then
sudo ln -sf "/usr/local${path}" "${root_fs_dir}${path}"
@@ -387,7 +452,7 @@
unibuild_flag="--unibuild"
fi
- "${CHROMITE_BIN}/cros_set_lsb_release" \
+ info_run "${CHROMITE_BIN}/cros_set_lsb_release" \
--sysroot="${root_fs_dir}" \
--board="${BOARD}" \
${unibuild_flag} \
@@ -416,7 +481,7 @@
# Note: fields in /etc/os-release can come from different places:
# * /etc/os-release itself with docrashid
# * /etc/os-release.d for fields created with do_osrelease_field
- sudo "${CHROMITE_BIN}/cros_generate_os_release" \
+ info_run sudo "${CHROMITE_BIN}/cros_generate_os_release" \
--root="${root_fs_dir}" \
--version="${CHROME_BRANCH}" \
--build_id="${CHROMEOS_VERSION_STRING}"
@@ -460,7 +525,7 @@
fi
if [[ "${has_bootable_kernel}" -eq 1 ]]; then
- ${BUILD_LIBRARY_DIR}/create_legacy_bootloader_templates.sh \
+ info_run "${BUILD_LIBRARY_DIR}/create_legacy_bootloader_templates.sh" \
--arch=${ARCH} \
--board=${BOARD} \
--image_type="${image_type}" \
@@ -503,7 +568,7 @@
# stateful since we created a symlink for those. We ignore the
# symlink in this package since the directory /usr/local/var
# exists in the target image when dev_install runs.
- sudo tar -cf "${BOARD_ROOT}/packages/dev-only-extras.tar.xz" \
+ info_run sudo tar -cf "${BOARD_ROOT}/packages/dev-only-extras.tar.xz" \
-I 'xz -9 -T0' --exclude=var -C "${root_fs_dir}/usr/local" .
create_dev_install_lists "${root_fs_dir}"
@@ -514,7 +579,7 @@
if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
info "Skipping DLC copying for factory shim images."
else
- build_dlc --sysroot="${BOARD_ROOT}" --rootfs="${root_fs_dir}" \
+ info_run build_dlc --sysroot="${BOARD_ROOT}" --rootfs="${root_fs_dir}" \
--stateful="${stateful_fs_dir}" --board="${BOARD}" --factory-install
fi
@@ -586,13 +651,14 @@
if [[ "${skip_kernelblock_install}" -ne 1 \
&& "${has_bootable_kernel}" -eq 1 ]]; then
# Place flags before positional args.
- ${SCRIPTS_DIR}/bin/cros_make_image_bootable "${BUILD_DIR}" \
+ info_run "${SCRIPTS_DIR}/bin/cros_make_image_bootable" "${BUILD_DIR}" \
${image_name} ${USE_DEV_KEYS} --adjust_part="${FLAGS_adjust_part}"
fi
# Build minios kernel and put it in the MINIOS-A partition of the image.
if has "minios" "$(portageq-"${BOARD}" envvar USE)"; then
- build_minios --board "${BOARD}" --image "${BUILD_DIR}/${image_name}" \
+ info_run build_minios --board "${BOARD}" \
+ --image "${BUILD_DIR}/${image_name}" \
--version "${CHROMEOS_VERSION_STRING}"
fi
}
diff --git a/build_library/build_common.sh b/build_library/build_common.sh
index 38eb1ec..87ef6b2 100644
--- a/build_library/build_common.sh
+++ b/build_library/build_common.sh
@@ -103,3 +103,18 @@
sudo -E PKGDIR="${tmp_pkgdir}" ${EMERGE_BOARD_CMD} --usepkgonly \
--root=${install_root} ${kernel} || die "Cannot emerge kernel to root"
}
+
+# Detect the decompression tool to use for |file|.
+detect_decompression_tool() {
+ local file="$1"
+
+ if [[ "$(od -An -tx1 -N4 "${file}")" == " 28 b5 2f fd" ]]; then
+ # Since the Gentoo binpkg has trailing garbage, tell zstd to ignore it.
+ echo "zstd -f"
+ elif [[ "$(od -An -tx1 -N3 "${file}")" == " 42 5a 68" ]]; then
+ echo "lbzip2"
+ else
+ echo "$1: unknown compression type" >&2
+ return 1
+ fi
+}
diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh
index 5b2d72f..53c735d 100755
--- a/build_library/build_image_util.sh
+++ b/build_library/build_image_util.sh
@@ -9,86 +9,13 @@
# one file aside from its lack of anywhere else to go. Probably,
# this file should get broken up or otherwise reorganized.
-# Use canonical path since some tools (e.g. mount) do not like symlinks.
-# Append build attempt to output directory.
-IMAGE_SUBDIR="R${CHROME_BRANCH}"
-if [ -z "${FLAGS_version}" ]; then
- IMAGE_SUBDIR="${IMAGE_SUBDIR}-${CHROMEOS_VERSION_STRING}-a\
-${FLAGS_build_attempt}"
-else
- IMAGE_SUBDIR="${IMAGE_SUBDIR}-${FLAGS_version}"
-fi
-
-if [ -n "${FLAGS_output_suffix}" ]; then
- IMAGE_SUBDIR="${IMAGE_SUBDIR}-${FLAGS_output_suffix}"
-fi
-
-BUILD_DIR="${FLAGS_build_root}/${BOARD}/${IMAGE_SUBDIR}"
-OUTPUT_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}"
-OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}"
-IMAGES_TO_BUILD=
-
EMERGE_BOARD_CMD="${CHROMITE_BIN}/parallel_emerge"
EMERGE_BOARD_CMD="$EMERGE_BOARD_CMD --board=$BOARD"
-export INSTALL_MASK="${DEFAULT_INSTALL_MASK}"
-
if [[ $FLAGS_jobs -ne -1 ]]; then
EMERGE_JOBS="--jobs=$FLAGS_jobs"
fi
-# Populates list of IMAGES_TO_BUILD from args passed in.
-# Arguments should be the shortnames of images we want to build.
-get_images_to_build() {
- local image_to_build
- for image_to_build in $*; do
- # Shflags leaves "'"s around ARGV.
- case ${image_to_build} in
- \'base\' )
- IMAGES_TO_BUILD="${IMAGES_TO_BUILD} ${CHROMEOS_BASE_IMAGE_NAME}"
- ;;
- \'dev\' )
- IMAGES_TO_BUILD="${IMAGES_TO_BUILD} ${CHROMEOS_DEVELOPER_IMAGE_NAME}"
- ;;
- \'test\' )
- IMAGES_TO_BUILD="${IMAGES_TO_BUILD} ${CHROMEOS_TEST_IMAGE_NAME}"
- ;;
- \'factory_install\' )
- IMAGES_TO_BUILD="${IMAGES_TO_BUILD} \
- ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}"
- ;;
- * )
- die "${image_to_build} is not an image specification."
- ;;
- esac
- done
-
- # Set default if none specified.
- if [ -z "${IMAGES_TO_BUILD}" ]; then
- IMAGES_TO_BUILD=${CHROMEOS_DEVELOPER_IMAGE_NAME}
- fi
-
- info "The following images will be built ${IMAGES_TO_BUILD}."
-}
-
-# Look at flags to determine which image types we should build.
-parse_build_image_args() {
- get_images_to_build ${FLAGS_ARGV}
- if should_build_image ${CHROMEOS_BASE_IMAGE_NAME} \
- ${CHROMEOS_DEVELOPER_IMAGE_NAME} ${CHROMEOS_TEST_IMAGE_NAME} && \
- should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
- die_notrace \
- "Can't build ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME} with any other" \
- "image."
- fi
- if should_build_image ${CHROMEOS_FACTORY_INSTALL_SHIM_NAME}; then
- # For factory, force rootfs verification and bootcache off
- FLAGS_enable_rootfs_verification=${FLAGS_FALSE}
- FLAGS_enable_bootcache=${FLAGS_FALSE}
- FLAGS_bootcache_use_board_default=${FLAGS_FALSE}
- fi
-}
-
make_salt() {
# It is not important that the salt be cryptographically strong; it just needs
# to be different for each release. The purpose of the salt is just to ensure
@@ -196,8 +123,7 @@
emerge_to_image() {
set -- ${EMERGE_BOARD_CMD} --root-deps=rdeps --usepkgonly -v --with-bdeps=n \
"$@" ${EMERGE_JOBS}
- info "$*"
- sudo -E "$@"
+ info_run sudo -E "$@"
}
# Create the /etc/shadow file with all the right entries.
@@ -296,7 +222,8 @@
local system_map="${kernel_out_dir}/System.map"
if [[ -r "${system_map}" ]]; then
- sudo depmod -ae -F "${system_map}" -b "${root_fs_dir}" "${kernel_release}"
+ info_run sudo depmod -ae -F "${system_map}" -b "${root_fs_dir}" \
+ "${kernel_release}"
fi
done
}
diff --git a/build_library/cgpt.py b/build_library/cgpt.py
index afcba26..984d3ec 100755
--- a/build_library/cgpt.py
+++ b/build_library/cgpt.py
@@ -741,12 +741,12 @@
return size
-def WriteLayoutFunction(options, sfile, func, image_type, config):
+def WriteLayoutFunction(options, slines, func, image_type, config):
"""Writes a shell script function to write out a given partition table.
Args:
options: Flags passed to the script
- sfile: File handle we're writing to
+ slines: lines to write to the script
func: function of the layout:
for removable storage device: 'partition',
for the fixed storage device: 'base'
@@ -917,20 +917,22 @@
if _HasExternalGpt(partitions):
lines += ['flashrom -w -iRW_GPT:${gptfile} --noverify-all']
- sfile.write('%s\n}\n' % '\n '.join(lines))
+ slines += '%s\n}\n\n' % '\n '.join(lines)
-def WritePartitionSizesFunction(options, sfile, func, image_type, config):
+def WritePartitionSizesFunction(options, slines, func, image_type, config,
+ data):
"""Writes out the partition size variable that can be extracted by a caller.
Args:
options: Flags passed to the script
- sfile: File handle we're writing to
+ slines: lines to write to the script file
func: function of the layout:
for removable storage device: 'partition',
for the fixed storage device: 'base'
image_type: Type of image eg base/test/dev/factory_install
config: Partition configuration file object
+ data: data dict we will write to a json file
"""
func_name = 'load_%s_vars' % func
lines = [
@@ -938,6 +940,11 @@
'DEFAULT_ROOTDEV="%s"' % config['metadata'].get('rootdev_%s' % func, ''),
]
+ data[func_name] = {}
+ data[func_name]['DEFAULT_ROOTDEV'] = (
+ '%s' % config['metadata'].get('rootdev_%s' % func, '')
+ )
+
partitions = GetPartitionTable(options, config, image_type)
for partition in partitions:
if partition.get('num') == 'metadata':
@@ -952,17 +959,22 @@
fs_format = partition.get('fs_format', '')
fs_options = partition.get('fs_options', '')
partition_num = partition.get('num', '')
- lines += [
- 'PARTITION_SIZE_%s=%s' % (shell_label, part_bytes),
- ' RESERVED_EBS_%s=%s' % (shell_label, reserved_ebs),
- ' DATA_SIZE_%s=%s' % (shell_label, fs_bytes),
- ' FORMAT_%s=%s' % (shell_label, part_format),
- ' FS_FORMAT_%s=%s' % (shell_label, fs_format),
- ' FS_OPTIONS_%s="%s"' % (shell_label, fs_options),
- ' PARTITION_NUM_%s="%s"' % (shell_label, partition_num),
- ]
-
- sfile.write('%s\n}\n' % '\n '.join(lines))
+ args = [('PARTITION_SIZE_', part_bytes),
+ ('RESERVED_EBS_', reserved_ebs),
+ ('DATA_SIZE_', fs_bytes),
+ ('FORMAT_', part_format),
+ ('FS_FORMAT_', fs_format)]
+ sargs = [('FS_OPTIONS_', fs_options),
+ ('PARTITION_NUM_', partition_num)]
+ for arg, value in args:
+ label = arg + shell_label
+ lines += ['%s=%s' % (label, value),]
+ data[func_name][label] = '%s' % value
+ for arg, value in sargs:
+ label = arg + shell_label
+ lines += ['%s="%s"' % (label, value),]
+ data[func_name][label] = '"%s"' % value
+ slines += '%s\n}\n\n' % '\n '.join(lines)
def GetPartitionByNumber(partitions, num):
@@ -1036,7 +1048,8 @@
raise PartitionNotFound('Partition "%s" not found' % label)
-def WritePartitionScript(options, image_type, layout_filename, sfilename):
+def WritePartitionScript(options, image_type, layout_filename, sfilename,
+ vfilename):
"""Writes a shell script with functions for the base and requested layouts.
Args:
@@ -1044,16 +1057,22 @@
image_type: Type of image eg base/test/dev/factory_install
layout_filename: Path to partition configuration file
sfilename: Filename to write the finished script to
+ vfilename: Filename to write the partition variables json data to
"""
config = LoadPartitionConfig(layout_filename)
- with open(sfilename, 'w') as f:
+ with open(sfilename, 'w') as f, open(vfilename, 'w') as jFile:
script_shell = GetScriptShell()
f.write(script_shell)
+ data = {}
+ slines = []
for func, layout in (('base', BASE_LAYOUT), ('partition', image_type)):
- WriteLayoutFunction(options, f, func, layout, config)
- WritePartitionSizesFunction(options, f, func, layout, config)
+ WriteLayoutFunction(options, slines, func, layout, config)
+ WritePartitionSizesFunction(options, slines, func, layout, config, data)
+
+ f.write(''.join(slines))
+ json.dump(data, jFile)
# TODO: Backwards compat. Should be killed off once we update
# cros_generate_update_payload to use the new code.
diff --git a/build_library/create_legacy_bootloader_templates.sh b/build_library/create_legacy_bootloader_templates.sh
index b3179c2..e3c47c3 100755
--- a/build_library/create_legacy_bootloader_templates.sh
+++ b/build_library/create_legacy_bootloader_templates.sh
@@ -60,7 +60,7 @@
# TODO: This code to support modifying kernel command line by boards is very
# similar to the one in build_kernel_image.sh. This could be refactored into a
# common place. Until then it needs to be kept consistent.
-config_file="$(mktemp legacy_config_XXXXXXXXXX.txt)"
+config_file="$(mktemp --tmpdir legacy_config_XXXXXXXXXX.txt)"
cleanup() {
rm -f "${config_file}"
}
diff --git a/build_library/dev_image_util.sh b/build_library/dev_image_util.sh
index 06d1d56..9397fad 100755
--- a/build_library/dev_image_util.sh
+++ b/build_library/dev_image_util.sh
@@ -40,9 +40,9 @@
# Copy over the libc debug info so that gdb
# works with threads and also for a better debugging experience.
sudo mkdir -p "${root_fs_dir}/usr/local/usr/lib/debug"
- pbzip2 -dc --ignore-trailing-garbage=1 "${LIBC_PATH}" | \
- sudo tar xpf - -C "${root_fs_dir}/usr/local/usr/lib/debug" \
- ./usr/lib/debug/usr/${CHOST} --strip-components=6
+ info_run sudo tar -I"${LIBC_DECOMPRESSOR}" -xpf "${LIBC_PATH}" \
+ -C "${root_fs_dir}/usr/local/usr/lib/debug" \
+ ./usr/lib/debug/usr/${CHOST} --strip-components=6
# Since gdb only looks in /usr/lib/debug, symlink the /usr/local
# path so that it is found automatically.
sudo ln -sfT /usr/local/usr/lib/debug "${root_fs_dir}/usr/lib/debug"
diff --git a/build_library/disk_layout_util.sh b/build_library/disk_layout_util.sh
index 661be49..2f5d464 100644
--- a/build_library/disk_layout_util.sh
+++ b/build_library/disk_layout_util.sh
@@ -43,14 +43,22 @@
local adjust_part="$3"
get_disk_layout_path
- local temp_script_file=$(mktemp)
+ local part_vars
+ part_vars="$(dirname "${partition_script_path}")/partition_vars.json"
+
+ local temp_script_file
+ local temp_vars_file
+ temp_script_file=$(mktemp)
+ temp_vars_file=$(mktemp)
sudo mkdir -p "$(dirname "${partition_script_path}")"
cgpt_py ${adjust_part:+--adjust_part "${adjust_part}"} \
write "${image_type}" "${DISK_LAYOUT_PATH}" \
- "${temp_script_file}"
+ "${temp_script_file}" "${temp_vars_file}"
sudo mv "${temp_script_file}" "${partition_script_path}"
+ sudo mv "${temp_vars_file}" "${part_vars}"
sudo chmod a+r "${partition_script_path}"
+ sudo chmod a+r "${part_vars}"
}
run_partition_script() {
diff --git a/build_library/legacy_disk_layout.json b/build_library/legacy_disk_layout.json
index 49fc68e..71fb304 100644
--- a/build_library/legacy_disk_layout.json
+++ b/build_library/legacy_disk_layout.json
@@ -176,6 +176,16 @@
# Used for recovery images.
"recovery": [
{
+ # Kernel for Slot A, no file system.
+ # Make the partition as large as the firmware supports so we can boot
+ # larger recovery kernels. The kernel won't actually be this large.
+ # NB: firmware defaulted to 32 MiB since 2015
+ # (https://crrev.com/c/281806), and was increased to 512 MiB in 2018
+ # (https://crbug.com/873135#c5).
+ "num": 2,
+ "size": "32 MiB"
+ },
+ {
# Slot B rootfs, unused on USB, but pad to 2M.
# installation will expand this to size from base.
"num": 5,
diff --git a/build_packages.sh b/build_packages.sh
deleted file mode 100644
index 6cd8c77..0000000
--- a/build_packages.sh
+++ /dev/null
@@ -1,556 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# This script is being deprecated and moved to chromite/ by 2022-06-30.
-# Do not add any references to this script.
-# Please reach out if you have any questions.
-
-. "$(dirname "$0")/common.sh" || exit 1
-
-if [[ "$1" != "--script-is-run-only-by-chromite-and-not-users" ]]; then
- die_notrace 'This script must not be run by users.' \
- 'Please run `build_packages` from $PATH (in chromite/bin/) instead.'
-fi
-# Discard the magic marker flag.
-shift
-
-# Script must run inside the chroot
-restart_in_chroot_if_needed "$@"
-
-assert_not_root_user
-
-# Developer-visible flags.
-DEFINE_string board "${DEFAULT_BOARD}" \
- "The board to build packages for."
-DEFINE_boolean chrome "${FLAGS_FALSE}" \
- "Ensure chrome instead of chromium. Alias for --internal --nouse_any_chrome."
-DEFINE_boolean usepkg "${FLAGS_TRUE}" \
- "Use binary packages to bootstrap when possible."
-DEFINE_boolean usepkgonly "${FLAGS_FALSE}" \
- "Only use binary packages to bootstrap; abort if any are missing."
-DEFINE_boolean workon "${FLAGS_TRUE}" \
- "Force-build workon packages."
-DEFINE_boolean showoutput "${FLAGS_FALSE}" \
- "Show all output from parallel_emerge."
-DEFINE_boolean withautotest "${FLAGS_TRUE}" \
- "Build autotest client code."
-DEFINE_boolean withdebugsymbols "${FLAGS_FALSE}" \
- "Install the debug symbols for all packages"
-DEFINE_boolean withrevdeps "${FLAGS_TRUE}" \
- "Calculate reverse dependencies on changed ebuilds."
-DEFINE_boolean autosetgov "${FLAGS_FALSE}" \
- "Automatically set cpu governor to 'performance'."
-DEFINE_boolean autosetgov_sticky "${FLAGS_FALSE}" \
- "Remember --autosetgov setting for future runs."
-DEFINE_boolean use_any_chrome "${FLAGS_TRUE}" \
- "Use any Chrome prebuilt available, even if the prebuilt doesn't match exactly."
-DEFINE_boolean cleanbuild "${FLAGS_FALSE}" \
- "Perform a clean build; delete sysroot if it exists before building."
-DEFINE_boolean internal "${FLAGS_FALSE}" \
- "Build the internal version of chrome (set the chrome_internal USE flag)."
-DEFINE_boolean pretend "${FLAGS_FALSE}" \
- "Don't build packages, just display which packages would have been installed."
-
-# The --board_root flag specifies the environment variables ROOT and PKGDIR.
-# This allows fetching and emerging of all packages to specified board_root.
-# Note that --board_root will setup the board normally in /build/$BOARD, if it's
-# not setup yet. It also expects the toolchain to already be installed in the
-# board_root. --usepkgonly and --norebuild are required, because building is not
-# supported when board_root is set.
-# enforce this)."
-DEFINE_string board_root "" \
- "Emerge packages to board_root."
-
-FLAGS_HELP="usage: $(basename $0) [flags] [packages]
-
-build_packages updates the set of binary packages needed by Chrome OS. It will
-cross compile all packages that have been updated into the given target's root
-and build binary packages as a side-effect. The output packages will be picked
-up by the build_image script to put together a bootable Chrome OS image.
-
-If [packages] are specified, only build those specific packages (and any
-dependencies they might need).
-
-For the fastest builds, use --nowithautotest --noworkon.
-"
-
-# The following options are advanced options, only available to those willing
-# to read the source code. They are not shown in help output, since they are
-# not needed for the typical developer workflow.
-DEFINE_string accept_licenses "" \
- "Licenses to append to the accept list."
-DEFINE_boolean eclean "${FLAGS_TRUE}" \
- "Run eclean to delete old binpkgs."
-DEFINE_integer jobs -1 \
- "How many packages to build in parallel at maximum."
-DEFINE_boolean norebuild "${FLAGS_FALSE}" \
- "Don't automatically rebuild dependencies."
-DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
- "Don't run the chroot upgrade automatically; use with care."
-DEFINE_boolean skip_setup_board "${FLAGS_FALSE}" \
- "Don't run setup_board. Implies skip_chroot_upgrade and" \
- "skip_toolchain_update."
-DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
- "Don't update toolchain automatically."
-DEFINE_boolean withdev "${FLAGS_TRUE}" \
- "Build useful developer friendly utilities."
-DEFINE_boolean withdebug "${FLAGS_TRUE}" \
- "Build debug versions of Chromium-OS-specific packages."
-DEFINE_boolean withfactory "${FLAGS_TRUE}" \
- "Build factory installer."
-DEFINE_boolean withtest "${FLAGS_TRUE}" \
- "Build packages required for testing."
-DEFINE_boolean expandedbinhosts "${FLAGS_TRUE}" \
- "Allow expanded binhost inheritance."
-
-# The --reuse_pkgs_from_local_boards flag tells Portage to share binary
-# packages between boards that are built locally, so that the total time
-# required to build several boards is reduced. This flag is only useful
-# when you are not able to use remote binary packages, since remote binary
-# packages are usually more up to date than anything you have locally.
-DEFINE_boolean reuse_pkgs_from_local_boards "${FLAGS_FALSE}" \
- "Bootstrap from local packages instead of remote packages."
-
-# --run_goma option is designed to be used on bots.
-# If you're trying to build pacakges with goma in your local dev env, this is
-# *not* the option you're looking for. Please see comments below.
-# This option; 1) starts goma, 2) builds packages (expecting that goma is
-# used), then 3) stops goma explicitly.
-# 3) is a request from the goma team, so that stats/logs can be taken.
-# Note: GOMA_DIR and GOMA_SERVICE_ACCOUNT_JSON_FILE are expected to be passed
-# via env var.
-#
-# In local dev env cases, compiler_proxy is expected to keep running.
-# In such a case;
-# $ python ${GOMA_DIR}/goma_ctl.py ensure_start
-# $ ./build_packages (... and options without --run_goma ...)
-# is an expected commandline sequence. If you set --run_goma flag while
-# compiler_proxy is already running, the existing compiler_proxy will be
-# stopped.
-DEFINE_boolean run_goma "${FLAGS_FALSE}" \
- "If set to true, (re)starts goma, builds packages, and then stops goma."
-
-# This option is for building chrome remotely.
-#1) starts reproxy 2) builds chrome with reproxy and 3) stops reproxy so
-# logs/stats can be collected.
-# Note: RECLIENT_DIR and REPROXY_CFG are expected to be passed via env var.
-DEFINE_boolean run_remoteexec "${FLAGS_FALSE}" \
- "If set to true, starts RBE reproxy, builds packages, and then stops reproxy."
-
-# Parse command line
-FLAGS "$@" || exit 1
-eval set -- "${FLAGS_ARGV}"
-
-# Die on any errors.
-switch_to_strict_mode
-
-# Chrome packages that need to be treated the same. These are the chrome and
-# chrome follow-on packages that share the same version as chrome and are
-# updated in lock step.
-CHROME_PACKAGES=(
- "chromeos-base/chromeos-chrome"
- "chromeos-base/chrome-icu"
-)
-
-# Alias/implied flag translations.
-if [[ "${FLAGS_chrome}" -eq "${FLAGS_TRUE}" ]]; then
- FLAGS_internal="${FLAGS_TRUE}"
- FLAGS_use_any_chrome="${FLAGS_FALSE}"
-fi
-
-if [[ "${FLAGS_internal}" -eq "${FLAGS_TRUE}" ]]; then
- export USE="${USE} chrome_internal"
-fi
-
-# Right now build_packages has to be run from scripts/
-. ${SRC_ROOT}/third_party/chromiumos-overlay/chromeos/config/chromeos_version.sh
-
-# On some systems, powersave can take a long time to ramp up. Inform the user
-# so they can get faster builds. https://crbug.com/1008932
-CHROMITE_CONFIG_DIR=$(
- python -c 'from chromite.lib.chromite_config import DIR; print(DIR)')
-CONFIG_AUTOSETGOV="${CHROMITE_CONFIG_DIR}/autosetgov"
-if [[ "${FLAGS_autosetgov_sticky}" -eq "${FLAGS_TRUE}" ]]; then
- mkdir -p "${CHROMITE_CONFIG_DIR}"
- if [[ "${FLAGS_autosetgov}" -eq "${FLAGS_TRUE}" ]]; then
- info "Future runs of build_packages will *always* use --autosetgov"
- echo "# Delete this file to turn off automatic build_packages" \
- "--autosetgov." >"${CONFIG_AUTOSETGOV}"
- else
- info "Future runs of build_packages will respect --autosetgov"
- rm -f "${CONFIG_AUTOSETGOV}"
- fi
-fi
-if [[ -e "${CONFIG_AUTOSETGOV}" ]]; then
- FLAGS_autosetgov="${FLAGS_TRUE}"
-fi
-# Make sure we can actually support "performance".
-if grep -qs performance \
- /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_governors; then
- curr_gov=$(sort -u /sys/devices/system/cpu/cpufreq/policy*/scaling_governor)
- if [[ "${curr_gov}" != "performance" ]]; then
- if [[ "${FLAGS_autosetgov}" -eq "${FLAGS_TRUE}" ]]; then
- if [[ $(echo "${curr_gov}" | wc -l) -gt 1 ]]; then
- warn "Too many active CPU governors; refusing to use 'performance'."
- else
- info "Temporarily setting cpu governor to 'performance'"
- trap "sudo cpupower -c all frequency-set -g powersave >&/dev/null" EXIT
- sudo cpupower -c all frequency-set -g performance >&/dev/null
- fi
- elif [[ "${curr_gov}" == "powersave" ]]; then
- warn "Current CPU governor set to 'powersave' which can slow down builds."
- warn "Use --autosetgov to automatically (and temporarily) switch to" \
- "'performance'."
- fi
- fi
-fi
-
-if [[ -z "${FLAGS_board}" ]]; then
- echo "Error: --board is required."
- exit 1
-fi
-
-BOARD_ROOT="${FLAGS_board_root:-/build/${FLAGS_board}}"
-
-# Skip revdeps when we don't already have a built sysroot.
-if [[ "${FLAGS_cleanbuild}" -eq "${FLAGS_TRUE}" || ! -d "${BOARD_ROOT}" ]]; then
- SKIP_REVDEPS="${FLAGS_TRUE}"
-else
- SKIP_REVDEPS="${FLAGS_FALSE}"
-fi
-
-if [[ "${FLAGS_skip_setup_board}" -eq "${FLAGS_FALSE}" ]]; then
- # Before we can run any tools, we need to update chroot or setup_board.
- UPDATE_ARGS=()
- if [[ -n ${FLAGS_accept_licenses} ]]; then
- UPDATE_ARGS+=( --accept-licenses "${FLAGS_accept_licenses}" )
- fi
- if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
- UPDATE_ARGS+=( --usepkg )
- else
- UPDATE_ARGS+=( --nousepkg )
- fi
- if [[ "${FLAGS_jobs}" -ne -1 ]]; then
- UPDATE_ARGS+=( --jobs=${FLAGS_jobs} )
- fi
- if [ "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ]; then
- UPDATE_ARGS+=( --reuse-pkgs-from-local-boards )
- fi
- if [ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_TRUE}" ]; then
- UPDATE_ARGS+=( --skip-toolchain-update )
- fi
- if [ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_TRUE}" ]; then
- UPDATE_ARGS+=( --skip-chroot-upgrade )
- fi
- if [[ -n ${FLAGS_board_root} ]]; then
- UPDATE_ARGS+=( --board-root "${FLAGS_board_root}" )
- fi
- if [ "${FLAGS_cleanbuild}" -eq "${FLAGS_TRUE}" ]; then
- UPDATE_ARGS+=( --force )
- fi
- if [[ "${FLAGS_expandedbinhosts}" -eq "${FLAGS_FALSE}" ]]; then
- UPDATE_ARGS+=( --fewer-binhosts )
- fi
-
- setup_board --quiet --board=${FLAGS_board} "${UPDATE_ARGS[@]}"
-fi
-
-sudo_clear_shadow_locks "${BOARD_ROOT}"
-PORTAGE_BINHOST=$(portageq-${FLAGS_board} envvar 'PORTAGE_BINHOST')
-info "PORTAGE_BINHOST: ${PORTAGE_BINHOST}"
-
-
-# Setup all the emerge command/flags.
-EMERGE_FLAGS=( -uDNv --backtrack=30 --newrepo --with-test-deps y )
-
-EMERGE_CMD=(
- "${CHROMITE_BIN}/parallel_emerge"
- --board=${FLAGS_board}
-)
-
-if [[ "${FLAGS_use_any_chrome}" -eq "${FLAGS_TRUE}" ]]; then
- for pkg in "${CHROME_PACKAGES[@]}"; do
- EMERGE_CMD+=( "--force-remote-binary=${pkg}" )
- done
-fi
-
-EMERGE_CMD+=( ${EXTRA_BOARD_FLAGS} )
-
-if [[ "${FLAGS_pretend}" -eq "${FLAGS_TRUE}" ]]; then
- EMERGE_FLAGS+=( "--pretend" )
-fi
-
-if [[ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ||
- "${FLAGS_reuse_pkgs_from_local_boards}" -eq "${FLAGS_TRUE}" ||
- "${FLAGS_usepkgonly}" -eq "${FLAGS_TRUE}" ]]; then
- # Use binary packages. Include all build-time dependencies,
- # so as to avoid unnecessary differences between source
- # and binary builds.
- EMERGE_FLAGS+=( --getbinpkg --with-bdeps y )
- if [[ ${FLAGS_usepkgonly} -eq ${FLAGS_TRUE} ]]; then
- EMERGE_FLAGS+=( --usepkgonly )
- else
- EMERGE_FLAGS+=( --usepkg )
- fi
-fi
-
-if [[ "${FLAGS_jobs}" -ne -1 ]]; then
- EMERGE_FLAGS+=( --jobs=${FLAGS_jobs} )
-fi
-
-if [[ "${FLAGS_norebuild}" -eq "${FLAGS_FALSE}" ]]; then
- EMERGE_FLAGS+=( --rebuild-if-new-rev )
-fi
-if [[ "${FLAGS_showoutput}" -eq "${FLAGS_TRUE}" ]]; then
- EMERGE_FLAGS+=( --show-output )
-fi
-
-if [[ "${FLAGS_withdebug}" -eq "${FLAGS_FALSE}" ]]; then
- export USE="${USE} -cros-debug"
-fi
-
-# Figure out which packages we should be building.
-PACKAGES=( "$@" )
-FORCE_LOCAL_BUILD_PKGS=()
-if [[ $# -eq 0 ]]; then
- PACKAGES=( virtual/target-os )
- if [[ "${FLAGS_withdev}" -eq "${FLAGS_TRUE}" ]]; then
- PACKAGES+=( virtual/target-os-dev )
- fi
- if [[ "${FLAGS_withfactory}" -eq "${FLAGS_TRUE}" ]]; then
- PACKAGES+=( virtual/target-os-factory )
- PACKAGES+=( virtual/target-os-factory-shim )
- fi
- if [[ "${FLAGS_withtest}" -eq "${FLAGS_TRUE}" ]]; then
- PACKAGES+=( virtual/target-os-test )
- # chromeos-ssh-testkeys may generate ssh keys if the right USE flag is set.
- # We force rebuilding this package from source every time, so that
- # consecutive builds don't share ssh keys.
- FORCE_LOCAL_BUILD_PKGS+=( chromeos-base/chromeos-ssh-testkeys )
- fi
- if [[ "${FLAGS_withautotest}" -eq "${FLAGS_TRUE}" ]]; then
- PACKAGES+=( chromeos-base/autotest-all )
- fi
-fi
-
-info "Rebuilding Portage cache"
-# Before running any emerge operations, regenerate the Portage dependency cache
-# in parallel.
-info_run "${EMERGE_CMD[@]}" --regen --quiet
-
-# Clean out any stale binpkgs we've accumulated. This is done immediately after
-# regenerating the cache in case ebuilds have been removed (e.g. from a revert).
-if [[ "${FLAGS_eclean}" -eq "${FLAGS_TRUE}" ]]; then
- info "Cleaning stale binpkgs"
- get_eclean_exclusions | "eclean-${FLAGS_board}" -e /dev/stdin packages
-fi
-
-# Verify that all packages can be emerged from scratch, without any
-# backtracking. Only print the output if this step fails.
-info "Checking package dependencies are correct: ${PACKAGES[*]}"
-if ! OUTPUT=$(emerge-${FLAGS_board} -pe --backtrack=0 \
- "${PACKAGES[@]}" 2>&1); then
- printf "%s\n" "${OUTPUT}"
- die_notrace "emerge detected broken ebuilds. See error message above."
-fi
-
-# Build cros_workon packages when they are changed.
-CROS_WORKON_PKGS=()
-if [ "${FLAGS_workon}" -eq "${FLAGS_TRUE}" ]; then
- LIST_MODIFIED_PACKAGES="${CHROMITE_BIN}/cros_list_modified_packages"
- MODIFIED_PACKAGES=( $("${LIST_MODIFIED_PACKAGES}" --board=${FLAGS_board}) )
- info "cros_workon modified packages '${MODIFIED_PACKAGES[*]}' detected"
- CROS_WORKON_PKGS+=( "${MODIFIED_PACKAGES[@]}" )
-
- # TODO(anush): Make chrome a fake cros-workon package.
- if [[ -n "${CHROME_ORIGIN}" ]]; then
- CROS_WORKON_PKGS+=( "${CHROME_PACKAGES[@]}" )
- fi
-fi
-
-# cros_workon packages always have to be rebuilt.
-FORCE_LOCAL_BUILD_PKGS+=( "${CROS_WORKON_PKGS[@]}" )
-
-# Any package that directly depends on an active cros_workon package also needs
-# to be rebuilt in order to be correctly built against the current set of
-# changes a user may have made to the cros_workon package.
-if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then
- # Collect all installed packages that depend on active cros_workon packages.
- WORKON_PKG_CONSUMERS=()
- mapfile -t WORKON_PKG_CONSUMERS < <( \
- equery-${FLAGS_board} -q depends "${CROS_WORKON_PKGS[@]}" | \
- sort -u | \
- grep -Ev "^\s*$" )
-
- # Transform this list of packages with versions in to a list of just
- # $CATEGORY/$NAME entries, since we don't want to pass packages with explicit
- # version numbers as arguments to `emerge`.
- if [[ ${#WORKON_PKG_CONSUMERS[@]} -gt 0 ]]; then
- WORKON_REBUILD_PKGS=()
- mapfile -t WORKON_REBUILD_PKGS < <( \
- equery-${FLAGS_board} list -p -o --format='$category/$name' \
- "${WORKON_PKG_CONSUMERS[@]}" | sort -u )
-
- info "The following packages depend directly on an active" \
- "cros_workon package and will be rebuilt: ${WORKON_REBUILD_PKGS[*]}"
-
- FORCE_LOCAL_BUILD_PKGS+=( "${WORKON_REBUILD_PKGS[@]}" )
- fi
-fi
-
-if [[ -n "${FLAGS_board_root}" ]]; then
- export ROOT="${FLAGS_board_root}"
- export PORTAGE_CONFIGROOT="${ROOT}"
- export SYSROOT="${ROOT}"
- export PKGDIR="${ROOT}"/packages
-fi
-
-# Temporarily modify the emerge flags so we can calculate the revdeps
-# on the modified packages.
-if [[ "${FLAGS_withrevdeps}" -eq "${FLAGS_TRUE}" &&
- "${SKIP_REVDEPS}" -eq "${FLAGS_FALSE}" ]]; then
- info "starting reverse dependency calculations ..."
- SIM_EMERGE_FLAGS=( "${EMERGE_FLAGS[@]}" --pretend --columns )
-
- if [[ ${#PACKAGES[@]} -gt 0 ]]; then
- SIM_EMERGE_FLAGS+=(
- --reinstall-atoms="${PACKAGES[*]}"
- --usepkg-exclude="${PACKAGES[*]}"
- )
- fi
-
- # cros-workon packages are always going to be force reinstalled, so we add
- # the forced reinstall behavior to the modified package calculation. This is
- # necessary to include when a user has already installed a 9999 ebuild and is
- # now reinstalling that package with additional local changes, because
- # otherwise the modified package calculation would not see that a 'new'
- # package is being installed.
- if [[ ${#CROS_WORKON_PKGS[@]} -gt 0 ]]; then
- SIM_EMERGE_FLAGS+=(
- --reinstall-atoms="${CROS_WORKON_PKGS[*]}"
- --usepkg-exclude="${CROS_WORKON_PKGS[*]}"
- )
- fi
-
- # Calculate only the ebuild changes from the emerge simulation ignoring
- # the virtual packages and the forced rebuild of autotest-all package.
- # The lines of the following block do the following operations:
- # 1. Do a pretend `emerge` command to get a list of what would be built.
- # 2. Filter to only packages that will be installed to the board sysroot.
- # 3. Filter to only packages that would be built from source and rewrite the
- # line from Portage's full output to only $CATEGORY/$PACKAGE
- # 4. Filter the list of packages to a heuristic set of packages known to have
- # incorrectly specified dependencies.
- # 5. Sort the output and remove any duplicate entries.
- BASE_INSTALL_PKGS=( $( \
- sudo -E "${EMERGE_CMD[@]}" "${SIM_EMERGE_FLAGS[@]}" "${PACKAGES[@]}" | \
- grep -e 'to /build/' | \
- sed -n -E '/^\[ebuild /{s:^[^]]+\] +::;s: .*::;p}' | \
- grep -E '/(coreboot-private-files.*|tast-build-deps)$' | \
- sort -u ) )
-
- MOD_PKGS=()
- if [[ "${#BASE_INSTALL_PKGS[@]}" -gt 0 ]]; then
- info "Forced rebuild packages detected: ${BASE_INSTALL_PKGS[*]}."
- # Convert specific versions into base package names
- MOD_PKGS+=( $(\
- equery-${FLAGS_board} list -p -o --format='$category/$name' \
- "${BASE_INSTALL_PKGS[@]}" | sort -u ) )
- # Remove Chrome as rebuilding it is expensive and almost never makes sense.
- # Ignore grep exit status in case chromeos-chrome is the only package.
- grep_cmd=( grep -v )
- for pkg in "${CHROME_PACKAGES[@]}"; do
- grep_cmd+=( -e "${pkg}" )
- done
- MOD_PKGS=( $(printf '%s\n' "${MOD_PKGS[@]}" | "${grep_cmd[@]}" || :) )
- fi
-
- FORCE_LOCAL_BUILD_PKGS+=( "${MOD_PKGS[@]}" )
-
- if [[ "${#MOD_PKGS[@]}" -gt 0 ]]; then
- info "calculating reverse dependencies on packages: ${MOD_PKGS[*]}"
- REV_DEPS=( $(\
- equery-${FLAGS_board} -q depends --indirect "${MOD_PKGS[@]}" |\
- awk '{print $1}' | grep -v ^virtual/ | sort -u) )
- if [[ "${#REV_DEPS[@]}" -gt 0 ]]; then
- # Convert specific versions into base package names
- RMOD_PKGS=( $(\
- equery-${FLAGS_board} -q list -p -o --format='$category/$name' \
- "${REV_DEPS[@]}" | sort -u ) )
- # Remove Chrome as rebuilding it is expensive and almost never makes
- # sense. Ignore grep exit status in case chromeos-chrome is the only
- # package.
- grep_cmd=( grep -v )
- for pkg in "${CHROME_PACKAGES[@]}"; do
- grep_cmd+=( -e "${pkg}" )
- done
- RMOD_PKGS=( $(printf '%s\n' "${RMOD_PKGS[@]}" | "${grep_cmd[@]}" || :) )
- info "final reverse dependencies that will be rebuilt: ${RMOD_PKGS[*]}"
- FORCE_LOCAL_BUILD_PKGS+=( "${RMOD_PKGS[@]}" )
- fi
- fi
-fi # end FLAGS_withrevdeps
-
-if [[ ${#FORCE_LOCAL_BUILD_PKGS[@]} -gt 0 ]]; then
- EMERGE_FLAGS+=(
- --reinstall-atoms="${FORCE_LOCAL_BUILD_PKGS[*]}"
- --usepkg-exclude="${FORCE_LOCAL_BUILD_PKGS[*]}"
- )
-fi
-
-# A list of critical system packages that should never be incidentally
-# reinstalled as a side effect of build_packages. All packages in this list
-# are special cased to prefer matching installed versions, overriding the
-# typical logic of upgrading to the newest available version.
-#
-# This list can't include any package that gets installed to a board!
-# Packages such as LLVM or binutils must not be in this list as the normal
-# rebuild logic must still apply to them for board targets.
-#
-# TODO(crbug/1050752): Remove this list and the corresponding arguments
-# to `emerge` below once we figure out how to exclude toolchain packages from
-# being upgraded transitively via BDEPEND relations.
-CRITICAL_SDK_PACKAGES=(
- "dev-lang/rust"
- "dev-lang/go"
- "sys-libs/glibc"
- "sys-devel/gcc"
-)
-
-info "Merging board packages now"
-(
-
- # Start reproxy for remote execution of building chrome.
- if [[ "${FLAGS_run_remoteexec}" -eq "${FLAGS_TRUE}" ]]; then
- info "Starting RBE reproxy."
- bootstrap="${RECLIENT_DIR}/bootstrap --cfg=${REPROXY_CFG} \
- --re_proxy=${RECLIENT_DIR}/reproxy"
- ${bootstrap}
- trap "${bootstrap} --shutdown" EXIT
- # Support goma on bots. This has to run in subshell, otherwise EXIT trap
- # handler is overwritten.
- elif [[ "${FLAGS_run_goma}" -eq "${FLAGS_TRUE}" ]]; then
- info "Starting goma compiler_proxy."
- goma_ctl="${GOMA_DIR:-${HOME}/goma}/goma_ctl.py"
- "${goma_ctl}" restart
- trap "'${goma_ctl}' stop" EXIT
- fi
-
- info_run sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "${PACKAGES[@]}" \
- --useoldpkg-atoms="${CRITICAL_SDK_PACKAGES[*]}" \
- --rebuild-exclude="${CRITICAL_SDK_PACKAGES[*]}"
-)
-
-echo "Builds complete"
-
-if [[ ${FLAGS_withdebugsymbols} -eq ${FLAGS_TRUE} ]]; then
- info "fetching the debug symbols"
- info_run sudo -E "${CHROMITE_BIN}/cros_install_debug_syms" \
- "--board=${FLAGS_board}" "--all"
-fi
-
-command_completed
-echo "Done"
diff --git a/build_sdk_board b/build_sdk_board
index 1074921..eec815e 100755
--- a/build_sdk_board
+++ b/build_sdk_board
@@ -38,8 +38,7 @@
# Locations we will need
BOARD_ROOT="/build/${BOARD}"
-CROSSDEV_OVERLAY="/usr/local/portage/crossdev"
-CHROMIUMOS_OVERLAY="/usr/local/portage/chromiumos"
+CHROMIUMOS_OVERLAY="${CHROOT_TRUNK_DIR}/src/third_party/chromiumos-overlay"
CHROMIUMOS_CONFIG="${CHROMIUMOS_OVERLAY}/chromeos/config"
CHROMIUMOS_PROFILES="${CHROMIUMOS_OVERLAY}/profiles"
BOARD_ETC="${BOARD_ROOT}/etc"
@@ -122,7 +121,7 @@
if [[ ${#TOOLCHAIN_PACKAGES[@]} -eq 0 ]]; then
die_notrace "cros_setup_toolchains failed"
fi
-PACKAGES=( system virtual/target-sdk world )
+PACKAGES=( system virtual/target-sdk )
run_emerge() {
info_run sudo -E ${EMERGE_CMD} "$@"
@@ -136,8 +135,8 @@
run_emerge "${TOOLCHAIN_PACKAGES[@]}"
# Then build everything else.
-run_emerge --emptytree --with-bdeps=y \
- --exclude --verbose "${TOOLCHAIN_PACKAGES[*]}" \
+run_emerge --verbose --emptytree --with-bdeps=y \
+ --exclude "${TOOLCHAIN_PACKAGES[*]}" \
"${PACKAGES[@]}" virtual/target-sdk-nobdeps
info_run sudo eclean -d packages
diff --git a/chroot_version_hooks.d/161_cleanup_sdk_binpkgs b/chroot_version_hooks.d/161_cleanup_sdk_binpkgs
deleted file mode 100644
index c4775ee..0000000
--- a/chroot_version_hooks.d/161_cleanup_sdk_binpkgs
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Prune orphaned binpkgs from the SDK that are never used by builders or devs.
-# See https://crbug.com/927122 for more details.
-
-exec sudo rm -rf /packages/
diff --git a/chroot_version_hooks.d/162_post_protobuf_upgrade b/chroot_version_hooks.d/162_post_protobuf_upgrade
deleted file mode 100644
index 16f93ee..0000000
--- a/chroot_version_hooks.d/162_post_protobuf_upgrade
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Clean local portage caches. This is necessary if there are
-# cros_workon packages.
-sudo rm -rf /var/cache/portage/* /build/*/var/cache/portage/*
-
-# Clean board packages.
-for board_root in /build/* ; do
- board=${board_root##*/}
- [[ "${board}" == "bin" || "${board}" == "dev" || \
- "${board}" == "proc" ]] && continue
-
- # Find packages using either protobuf or grpc.
- PKGS=( $(qdepends-${board} -qCN -Q dev-libs/protobuf && \
- qdepends-${board} -qCN -Q net-libs/grpc | sort -u))
- if [[ ${#PKGS[@]} -eq 0 ]]; then
- continue
- fi
-
- echo "Cleaning packages using protobuf or grpc from ${board}"
- PKGDIR=$(portageq-${board} envvar PKGDIR)
- cd "${PKGDIR}"
- # Remove existing binary packages.
- for pkg in "${PKGS[@]}"; do
- sudo rm -f "${pkg}"-[0-9]*.tbz2
- done
- # Unmerge all packages using either protobuf or grpc.
- qmerge-${board} -Uqy "${PKGS[@]}"
-done
diff --git a/chroot_version_hooks.d/163_use_postsubmit_binpkgs b/chroot_version_hooks.d/163_use_postsubmit_binpkgs
deleted file mode 100644
index a8ccdcd..0000000
--- a/chroot_version_hooks.d/163_use_postsubmit_binpkgs
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Force regeneration of every board configuration in order to include the
-# postsubmit builders as binpkg sources.
-# crbug.com/898996
-
-for board_root in /build/*; do
- board_name=${board_root##*/}
- if [[ -e "/build/${board_name}/etc/make.conf.board" ]]; then
- /mnt/host/source/chromite/bin/setup_board \
- --board=${board_name} \
- --skip-board-pkg-init \
- --skip-chroot-upgrade \
- --regen-configs &
- fi
-done
-wait
diff --git a/chroot_version_hooks.d/164_post_protobuf_upgrade b/chroot_version_hooks.d/164_post_protobuf_upgrade
deleted file mode 120000
index 127de13..0000000
--- a/chroot_version_hooks.d/164_post_protobuf_upgrade
+++ /dev/null
@@ -1 +0,0 @@
-162_post_protobuf_upgrade
\ No newline at end of file
diff --git a/chroot_version_hooks.d/165_clean_libbrillo_so b/chroot_version_hooks.d/165_clean_libbrillo_so
deleted file mode 100644
index 5e9b1bf..0000000
--- a/chroot_version_hooks.d/165_clean_libbrillo_so
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Some library and package config files of libbrillo were created at
-# src_install() phase into the cache. It prevents overwriting when we switch to
-# create those files at src_compile with the user privilege.
-
-sudo rm -rf /var/cache/portage/* /build/*/var/cache/portage/*
diff --git a/chroot_version_hooks.d/166_gtest_upgrade b/chroot_version_hooks.d/166_gtest_upgrade
deleted file mode 100644
index 7db485a..0000000
--- a/chroot_version_hooks.d/166_gtest_upgrade
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2014 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# gtest/gmock 1.7 is not binary compatible with the 1.6 version since
-# some internal symbols exposed in the .so file changed. The portage
-# cache in /build/$BOARD/var/cache/portage/chromeos-base will keep files
-# even if the ebuild is recompiled with the new gtest, so we need to remove
-# those files as well.
-#
-
-for sysroot in /build/* /; do
- (
- info "Cleaning /var/cache/portage/chromeos-base from ${sysroot}"
- sudo rm -rf "${sysroot}"/var/cache/portage/chromeos-base
- ) &
-done
-wait
diff --git a/chroot_version_hooks.d/167_remove_svn b/chroot_version_hooks.d/167_remove_svn
deleted file mode 100644
index cc31348..0000000
--- a/chroot_version_hooks.d/167_remove_svn
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Purge subversion (and its unique deps) from the SDK as no one uses it anymore.
-# https://crbug.com/982351
-
-PKGS=(
- dev-vcs/subversion
- net-libs/neon
- net-libs/serf
-
- # Some boards might need these for other packages (e.g. apache), but our build
- # system will just pull them back in on demand, so purging should be safe.
- dev-libs/apr
- dev-libs/apr-util
-)
-
-# Clear the SDK first.
-echo "Clearing subversion (and deps) from the SDK"
-sudo qmerge -Uqy "${PKGS[@]}" &
-
-# Clean board packages.
-for board_root in /build/*; do
- board=${board_root##*/}
- if [[ -d "${board_root}/var/db/pkg" ]]; then
- echo "Clearing subversion (and deps) from ${board}"
- sudo qmerge-${board} -Uqy "${PKGS[@]}" &
- fi
-done
-
-wait
diff --git a/chroot_version_hooks.d/168_python34_cleanup b/chroot_version_hooks.d/168_python34_cleanup
deleted file mode 100644
index 71673be..0000000
--- a/chroot_version_hooks.d/168_python34_cleanup
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Drop Python 3.4 from the SDK as we've moved to Python 3.6 everywhere.
-# Make sure 3.6 is installed in case this is an old chroot.
-
-echo "Cleaning up Python 3.4 to migrate to 3.6"
-
-export CLEAN_DELAY=0
-
-# Install python 3.6 if the sdk is out of date.
-if ! python3.6 --version >&/dev/null; then
- sudo emerge -ug dev-lang/python:3.6
-fi
-
-# Point python3 to the new version.
-sudo eselect python update --python3
-
-# Remove python 3.4 from the system.
-sudo emerge -Cq dev-lang/python:3.4
-
-# Remove java-config if needed to avoid circular deps.
-reinstall_java="false"
-if ! python3.6 -c 'import java_config_2' >&/dev/null; then
- reinstall_java="true"
- sudo emerge -Cq dev-java/java-config
-fi
-
-# Upgrade portage itself finally.
-sudo emerge -Ugqv sys-apps/portage
-
-# Reinstall java-config in case we purged it.
-if [[ "${reinstall_java}" == "true" ]]; then
- sudo emerge -ugqv dev-java/java-config
-fi
diff --git a/chroot_version_hooks.d/169_deleted_binhosts_cleanup b/chroot_version_hooks.d/169_deleted_binhosts_cleanup
deleted file mode 100644
index a8097cd..0000000
--- a/chroot_version_hooks.d/169_deleted_binhosts_cleanup
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Force regeneration of every board configuration in order to clean out
-# old binhosts. crbug.com/965244
-
-for board_root in /build/*; do
- board_name=${board_root##*/}
- if [[ -e "/build/${board_name}/etc/make.conf.board" ]]; then
- /mnt/host/source/chromite/bin/setup_board \
- --board=${board_name} \
- --skip-board-pkg-init \
- --skip-chroot-upgrade \
- --regen-configs &
- fi
-done
-wait
diff --git a/chroot_version_hooks.d/170_deleted_binhosts_cleanup b/chroot_version_hooks.d/170_deleted_binhosts_cleanup
deleted file mode 120000
index 7bac7f7..0000000
--- a/chroot_version_hooks.d/170_deleted_binhosts_cleanup
+++ /dev/null
@@ -1 +0,0 @@
-169_deleted_binhosts_cleanup
\ No newline at end of file
diff --git a/chroot_version_hooks.d/171_world_set_cleanup b/chroot_version_hooks.d/171_world_set_cleanup
deleted file mode 100644
index 3ddf7b8..0000000
--- a/chroot_version_hooks.d/171_world_set_cleanup
+++ /dev/null
@@ -1,7 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Make sure the @world set is empty and doesn't contain packages we don't want
-# it to like rust. https://crbug.com/1013967
-sudo rm -f /var/lib/portage/world
diff --git a/chroot_version_hooks.d/172_deleted_binhosts_cleanup b/chroot_version_hooks.d/172_deleted_binhosts_cleanup
deleted file mode 120000
index 7bac7f7..0000000
--- a/chroot_version_hooks.d/172_deleted_binhosts_cleanup
+++ /dev/null
@@ -1 +0,0 @@
-169_deleted_binhosts_cleanup
\ No newline at end of file
diff --git a/chroot_version_hooks.d/173_rust_slot_purge b/chroot_version_hooks.d/173_rust_slot_purge
deleted file mode 100644
index 5157ae9..0000000
--- a/chroot_version_hooks.d/173_rust_slot_purge
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# A hack to try and unwedge some rust rebuilds. https://crbug.com/1018589
-
-export CLEAN_DELAY=0
-
-for board_root in /build/*; do
- board_name=${board_root##*/}
- if [[ -d "${board_root}/var/db/pkg/dev-rust" ]]; then
- if rust_pkgs=$(qlist-"${board_name}" -IC dev-rust); then
- emerge-"${board_name}" -Cq ${rust_pkgs} &
- fi
- fi
-done
-wait
diff --git a/chroot_version_hooks.d/174_python3_sdk_default b/chroot_version_hooks.d/174_python3_sdk_default
deleted file mode 100644
index 73a382c..0000000
--- a/chroot_version_hooks.d/174_python3_sdk_default
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Set Python 3 as the default implementation.
-sudo eselect python update
-
-# Install the new symlinks. This is normally handled only when the SDK is
-# first created.
-sudo ln -sfT /mnt/host/source/chromite \
- /usr/lib64/python3.6/site-packages/chromite
-
-# Sanity check.
-python -c 'import sys; assert sys.version_info.major >= 3'
diff --git a/chroot_version_hooks.d/175_clang-python_cleanup b/chroot_version_hooks.d/175_clang-python_cleanup
deleted file mode 100644
index 955aeeb..0000000
--- a/chroot_version_hooks.d/175_clang-python_cleanup
+++ /dev/null
@@ -1,7 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Remove dev-python/clang-python package which will clean-up out-dated
-# dependencies in the old chroot.
-sudo qmerge -Uqy dev-python/clang-python
diff --git a/chroot_version_hooks.d/176_binutils_fixup b/chroot_version_hooks.d/176_binutils_fixup
deleted file mode 100644
index b33c38e..0000000
--- a/chroot_version_hooks.d/176_binutils_fixup
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Fix binutils headers and library conflicts when installing binutils-libs.
-# Remove symlinks to headers that are now provided by binutils-libs.
-# Note re-building binutils does not remove these symlinks so
-# a manual removal is needed to avoid merge conflicts.
-files_to_remove=(
- /usr/include/symcat.h
- /usr/include/plugin-api.h
- /usr/include/bfdlink.h
- /usr/include/bfd.h
- /usr/include/dis-asm.h
- /usr/include/ansidecl.h
- /usr/include/libiberty/sort.h
- /usr/include/libiberty/fibheap.h
- /usr/include/libiberty/safe-ctype.h
- /usr/include/libiberty/demangle.h
- /usr/include/libiberty/objalloc.h
- /usr/include/libiberty/hashtab.h
- /usr/include/libiberty/timeval-utils.h
- /usr/include/libiberty/splay-tree.h
- /usr/include/libiberty/partition.h
- /usr/include/libiberty/dyn-string.h
- /usr/include/libiberty/libiberty.h
- /usr/include/libiberty/floatformat.h
- /usr/include/libiberty/ansidecl.h
- /usr/include/libiberty.h
-)
-echo "Fixing up binutils installation"
-sudo rm -f "${files_to_remove[@]}"
-sudo emerge -ugq sys-devel/binutils-config
-sudo emerge -ugq sys-libs/binutils-libs
diff --git a/chroot_version_hooks.d/177_purge_crosutils b/chroot_version_hooks.d/177_purge_crosutils
deleted file mode 100644
index 30e258e..0000000
--- a/chroot_version_hooks.d/177_purge_crosutils
+++ /dev/null
@@ -1,6 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# This package has been removed, so make sure it's cleaned up.
-sudo qmerge -Uqy dev-util/crosutils
diff --git a/chroot_version_hooks.d/178_merge_distdir b/chroot_version_hooks.d/178_merge_distdir
deleted file mode 100644
index ff79034..0000000
--- a/chroot_version_hooks.d/178_merge_distdir
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# We changed the distfile download dirs to a combined one, so move the old
-# content up to the common location.
-
-# Delete the old symlinks.
-sudo rm -f /var/lib/portage/distfiles /var/lib/portage/distfiles-target
-
-# Merge the content.
-echo "Merging old distfiles paths"
-cd /var/cache/distfiles
-for srcdir in target host; do
- if [[ ! -d "${srcdir}" ]]; then
- continue
- fi
-
- echo " ${srcdir} ..."
- # Merge the two git subdirs.
- for gitdir in egit-src git3-src; do
- mkdir -p "${gitdir}"
- if [[ -d "${srcdir}/${gitdir}" ]]; then
- for d in $(find "${srcdir}/${gitdir}/" -mindepth 1 -maxdepth 1 -type d \
- -printf '%f\n'); do
- if [[ ! -d "${gitdir}/${d}" ]]; then
- mv "${srcdir}/${gitdir}/${d}" "${gitdir}/" || exit 1
- fi
- done
- fi
- done
-
- # Clear all the subdirs now that they're migrated.
- sudo rm -rf "${srcdir}"/*/
-
- # Merge the files.
- rm -f "${srcdir}"/*._checksum_failure_.*
- for f in "${srcdir}"/*; do
- if [[ -e "${f}" && ! -e "${f##*/}" ]]; then
- mv "${f}" ./ || exit 1
- fi
- done
-
- # Delete the dir.
- sudo rm -rf "${srcdir}" || exit 1
-done
diff --git a/chroot_version_hooks.d/179_purge_ruby b/chroot_version_hooks.d/179_purge_ruby
deleted file mode 100644
index 61be2bb..0000000
--- a/chroot_version_hooks.d/179_purge_ruby
+++ /dev/null
@@ -1,10 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# These packages have been removed, so make sure they're cleaned up.
-sudo qmerge -Uqy \
- app-admin/puppet \
- app-eselect/eselect-ruby \
- dev-lang/ruby \
- $(qlist -IC virtual/ruby dev-ruby/)
diff --git a/chroot_version_hooks.d/180_select_jvm b/chroot_version_hooks.d/180_select_jvm
deleted file mode 100644
index e76d71c..0000000
--- a/chroot_version_hooks.d/180_select_jvm
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Specify JDK as openjdk-bin-11 upon entering chroot.
-# See crrev.com/c/2335921 for more details.
-
-# Make sure dependencies are installed for older SDKs.
-sudo emerge -qug '>=virtual/jdk-9'
-
-sudo eselect java-vm set system openjdk-bin-11
-
-# Log current state for debugging issues.
-eselect java-vm list
-whereis java
diff --git a/chroot_version_hooks.d/181_rust_cleanup b/chroot_version_hooks.d/181_rust_cleanup
deleted file mode 100644
index 6bf0fc6..0000000
--- a/chroot_version_hooks.d/181_rust_cleanup
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright 2020 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-# Clean ebuilds that have been upgraded but aren't automatically removed.
-
-export CLEAN_DELAY=0
-
-to_remove=(
- "~dev-rust/bitflags-1.1.0"
- "~dev-rust/byteorder-1.3.1"
- "~dev-rust/proc-macro2-1.0.8"
- "~serde-1.0.81"
- "~serde_derive-1.0.81"
- "~syn-1.0.14"
-)
-
-sudo emerge -Cq "${to_remove[@]}" &
-
-for board_root in /build/*; do
- board_name=${board_root##*/}
- if [[ -d "${board_root}/var/db/pkg/dev-rust" ]]; then
- emerge-"${board_name}" -Cq "${to_remove[@]}" &
- fi
-done
-wait
diff --git a/chroot_version_hooks.d/182_binutils235_upgrade b/chroot_version_hooks.d/182_binutils235_upgrade
index 51139d8..9a8b21c 100644
--- a/chroot_version_hooks.d/182_binutils235_upgrade
+++ b/chroot_version_hooks.d/182_binutils235_upgrade
@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+# See comments in the hook.
+. /mnt/host/source/src/scripts/chroot_version_hooks.d/204_zstd
+
# Upgrade binutils to avoid binutils being pulled into
# build graph when building target packages, https://crbug.com/1171084
sudo emerge -ugq sys-devel/binutils
diff --git a/chroot_version_hooks.d/183_sandbox_update b/chroot_version_hooks.d/183_sandbox_update
index 66bcdbb..bdaed62 100644
--- a/chroot_version_hooks.d/183_sandbox_update
+++ b/chroot_version_hooks.d/183_sandbox_update
@@ -2,5 +2,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+# See comments in the hook.
+. /mnt/host/source/src/scripts/chroot_version_hooks.d/204_zstd
+
# Upgrade sandbox to avoid renameat failures, https://crbug.com/1176957
sudo emerge -ugq sys-apps/sandbox
diff --git a/chroot_version_hooks.d/186_bash_upgrade b/chroot_version_hooks.d/186_bash_upgrade
index 231d956..a2c1d77 100644
--- a/chroot_version_hooks.d/186_bash_upgrade
+++ b/chroot_version_hooks.d/186_bash_upgrade
@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+# See comments in the hook.
+. /mnt/host/source/src/scripts/chroot_version_hooks.d/204_zstd
+
# Upgrade bash to the latest version we support.
# Bash updates change behavior, and build scripts are made to support the
# latest bash version, and may not work with older versions.
diff --git a/chroot_version_hooks.d/187_cros_camera_cleanup b/chroot_version_hooks.d/187_cros_camera_cleanup
index 5532427..de82cf9 100644
--- a/chroot_version_hooks.d/187_cros_camera_cleanup
+++ b/chroot_version_hooks.d/187_cros_camera_cleanup
@@ -4,4 +4,6 @@
# Clean local portage caches. We have stale mojom headers in chroot after camera
# package consolidation (b/177958529).
+# Clean local portage caches. We have stale intermediates after the protobuf
+# # upgrade (crbug.com/1346059).
sudo rm -rf /var/cache/portage/* /build/*/var/cache/portage/*
diff --git a/chroot_version_hooks.d/199_emerge_libunwind b/chroot_version_hooks.d/199_emerge_libunwind
index 6e069e0..a53bc8f 100644
--- a/chroot_version_hooks.d/199_emerge_libunwind
+++ b/chroot_version_hooks.d/199_emerge_libunwind
@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+# See comments in the hook.
+. /mnt/host/source/src/scripts/chroot_version_hooks.d/204_zstd
+
# Force all boards to re-emerge llvm-libunwind
# b/210927982
diff --git a/chroot_version_hooks.d/201_remove_i686-pc-linux-gnu b/chroot_version_hooks.d/201_remove_i686-pc-linux-gnu
new file mode 100644
index 0000000..33e75a0
--- /dev/null
+++ b/chroot_version_hooks.d/201_remove_i686-pc-linux-gnu
@@ -0,0 +1,9 @@
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# The i686-pc-linux-gnu toolchain was renamed to i686-cros-linux-gnu, so delete
+# the old toolchain.
+# https://issuetracker.google.com/issues/187786439
+# lakitu: skip this hook for bootstrapping
+# sudo crossdev --force -C i686-pc-linux-gnu
diff --git a/chroot_version_hooks.d/202_clean_python2 b/chroot_version_hooks.d/202_clean_python2
new file mode 100644
index 0000000..7cdd10b
--- /dev/null
+++ b/chroot_version_hooks.d/202_clean_python2
@@ -0,0 +1,23 @@
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# These packages have been removed, so make sure they're cleaned up.
+pkgs=(
+ dev-python/enum34
+ dev-python/mox
+ dev-python/backports-functools-lru-cache
+ dev-python/enum34
+ dev-python/funcsigs
+ dev-python/functools32
+ dev-python/futures
+ dev-python/ipaddress
+ dev-python/mox
+ dev-python/pathlib
+ dev-python/subprocess32
+ virtual/python-enum34
+ virtual/python-funcsigs
+ virtual/python-futures
+)
+
+sudo qmerge -Uqy "${pkgs[@]}"
diff --git a/chroot_version_hooks.d/203_emerge_libcxx b/chroot_version_hooks.d/203_emerge_libcxx
new file mode 100644
index 0000000..9dde085
--- /dev/null
+++ b/chroot_version_hooks.d/203_emerge_libcxx
@@ -0,0 +1,17 @@
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# See comments in the hook.
+. /mnt/host/source/src/scripts/chroot_version_hooks.d/204_zstd
+
+# Force all boards to re-emerge sys-libs/libcxxabi sys-libs/libcxx.
+# This is required for switch to runtimes builds (b/204093890).
+
+sudo emerge -gqu --nodeps sys-libs/libcxxabi sys-libs/libcxx
+for board_root in /build/*; do
+ board_name=${board_root##*/}
+ if [[ -d "${board_root}/var/db/pkg" ]]; then
+ emerge-${board_name} -gqu --nodeps sys-libs/libcxxabi sys-libs/libcxx
+ fi
+done
diff --git a/chroot_version_hooks.d/204_zstd b/chroot_version_hooks.d/204_zstd
new file mode 100644
index 0000000..1cb0298
--- /dev/null
+++ b/chroot_version_hooks.d/204_zstd
@@ -0,0 +1,10 @@
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# If an older SDK doesn't have zstd yet, the SDK produces the zstd binpkg using
+# zstd, so we can't install it. Force it from source.
+
+if ! type -P zstd >/dev/null; then
+ sudo emerge -1O --buildpkg=n zstd
+fi
diff --git a/chroot_version_hooks.d/205_delete_unused_symlinks b/chroot_version_hooks.d/205_delete_unused_symlinks
new file mode 100644
index 0000000..aec5ca7
--- /dev/null
+++ b/chroot_version_hooks.d/205_delete_unused_symlinks
@@ -0,0 +1,10 @@
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# These shouldn't be used anymore.
+
+sudo ln -sfT \
+ "$(realpath /etc/portage/make.profile)" \
+ /etc/portage/make.profile
+sudo rm -f /usr/local/portage/{chromiumos,eclass-overlay,stable}
diff --git a/chroot_version_hooks.d/206_emerge_glibc_without_libcrypt b/chroot_version_hooks.d/206_emerge_glibc_without_libcrypt
new file mode 100644
index 0000000..6cedd58
--- /dev/null
+++ b/chroot_version_hooks.d/206_emerge_glibc_without_libcrypt
@@ -0,0 +1,8 @@
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Remove old virtual/libcrypt-1 and rebuild chroot glibc without libcrypt
+# which is replaced by libxcrypt + virtual/libcrypt-2.
+sudo qmerge -Uqy virtual/libcrypt
+sudo emerge -ugq sys-libs/glibc virtual/libcrypt sys-libs/libxcrypt
diff --git a/chroot_version_hooks.d/207_chromite_cipd_dir b/chroot_version_hooks.d/207_chromite_cipd_dir
new file mode 100644
index 0000000..8befb57
--- /dev/null
+++ b/chroot_version_hooks.d/207_chromite_cipd_dir
@@ -0,0 +1,5 @@
+# Copyright 2022 The ChromiumOS Authors.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+rm -rf ~/chromiumos/chromite/.cipd_bin/
diff --git a/chroot_version_hooks.d/208_proto_cleanup b/chroot_version_hooks.d/208_proto_cleanup
new file mode 120000
index 0000000..c1542c6
--- /dev/null
+++ b/chroot_version_hooks.d/208_proto_cleanup
@@ -0,0 +1 @@
+./187_cros_camera_cleanup
\ No newline at end of file
diff --git a/chroot_version_hooks.d/209_update_portage b/chroot_version_hooks.d/209_update_portage
new file mode 100644
index 0000000..5208b3b
--- /dev/null
+++ b/chroot_version_hooks.d/209_update_portage
@@ -0,0 +1,6 @@
+# Copyright 2022 The ChromiumOS Authors.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Update portage to pick up recent fixes before updating rest of chroot.
+sudo emerge -gu -j8 sys-apps/portage
diff --git a/common.sh b/common.sh
index 1d3d623..6de1573 100644
--- a/common.sh
+++ b/common.sh
@@ -87,7 +87,11 @@
# Declare these asap so that code below can safely assume they exist.
_message() {
local prefix
- prefix="$(date +%H:%M:%S) $1"
+ # Turn nanoseconds into milliseconds.
+ prefix="$(date +%H:%M:%S.%N)"
+ prefix="${prefix:0:-6}"
+
+ prefix+=" $1"
shift
if [[ $# -eq 0 ]]; then
echo -e "${prefix}${CROS_LOG_PREFIX:-""}:${V_VIDOFF}" >&2
@@ -186,66 +190,9 @@
# e.g. has "foo" "foo bar baz" is true, but has "f" "foo bar baz" is not.
has() { [[ " ${*:2} " == *" $1 "* ]]; }
-# Directory locations inside the dev chroot; try the new default,
-# falling back to user specific paths if the upgrade has yet to
-# happen.
-_user="${USER}"
-[[ ${USER} == "root" ]] && _user="${SUDO_USER}"
-_CHROOT_TRUNK_DIRS=( "/home/${_user}/chromiumos" /mnt/host/source )
-_DEPOT_TOOLS_DIRS=( "/home/${_user}/depot_tools" /mnt/host/depot_tools )
-unset _user
-
-_process_mount_pt() {
- # Given 4 arguments; the root path, the variable to set,
- # the old location, and the new; finally, forcing the upgrade is doable
- # via if a 5th arg is provided.
- # This will then try to migrate the old to new if we can do so right now
- # (else leaving symlinks in place w/in the new), and will set $1 to the
- # new location.
- local base=${1:-/} var=$2 old=$3 new=$4 force=${5:-false}
- local _sudo=$([[ ${USER} != "root" ]] && echo sudo)
- local val=${new}
- if ${force} || [[ -L ${base}/${new} ]] || [[ ! -e ${base}/${new} ]]; then
- # Ok, it's either a symlink or this is the first run. Upgrade if we can-
- # specifically, if we're outside the chroot and we can rmdir the old.
- # If we cannot rmdir the old, that's due to a mount being bound to that
- # point (even if we can't see it, it's there)- thus fallback to adding
- # compat links.
- if ${force} || ( [[ ${INSIDE_CHROOT} -eq 0 ]] && \
- ${_sudo} rmdir "${base}/${old}" 2>/dev/null ); then
- ${_sudo} rm -f "${base}/${new}" || :
- ${_sudo} mkdir -p "${base}/${new}" "$(dirname "${base}/${old}" )"
- ${_sudo} ln -s "${new}" "${base}/${old}"
- else
- if [[ ! -L ${base}/${new} ]]; then
- # We can't do the upgrade right now; install compatibility links.
- ${_sudo} mkdir -p "$(dirname "${base}/${new}")" "${base}/${old}"
- ${_sudo} ln -s "${old}" "${base}/${new}"
- fi
- val=${old}
- fi
- fi
- eval "${var}=\"${val}\""
-}
-
-set_chroot_trunk_dir() {
- # This takes two optional arguments; the first being the path to the chroot
- # base; this is only used by enter_chroot. The second argument is whether
- # or not to force the new pathways; this is only used by make_chroot. Passing
- # a non-null value for $2 forces the new paths.
- if [[ ${INSIDE_CHROOT} -eq 0 ]] && [[ -z ${1-} ]]; then
- # Can't do the upgrade, thus skip trying to do so.
- CHROOT_TRUNK_DIR="${_CHROOT_TRUNK_DIRS[1]}"
- DEPOT_TOOLS_DIR="${_DEPOT_TOOLS_DIRS[1]}"
- return
- fi
- _process_mount_pt "${1:-}" CHROOT_TRUNK_DIR "${_CHROOT_TRUNK_DIRS[@]}" \
- ${2:+true}
- _process_mount_pt "${1:-}" DEPOT_TOOLS_DIR "${_DEPOT_TOOLS_DIRS[@]}" \
- ${2:+true}
-}
-
-set_chroot_trunk_dir
+# Directory locations inside the dev chroot.
+CHROOT_TRUNK_DIR="/mnt/host/source"
+DEPOT_TOOLS_DIR="/mnt/host/depot_tools"
# Construct a list of possible locations for the source tree. This list is
# based on various environment variables and globals that may have been set
@@ -362,9 +309,6 @@
fi
fi
-# Directory to store built images. Should be set by sourcing script when used.
-BUILD_DIR=
-
# Path to the verified boot directory where we get signing related keys/scripts.
VBOOT_DIR="${CHROOT_TRUNK_DIR}/src/platform/vboot_reference"
VBOOT_TESTKEYS_DIR="${VBOOT_DIR}/tests/testkeys"
@@ -398,6 +342,7 @@
*.hxx
*.proto
*/.keep*
+ /build/bin
/build/initramfs
/build/libexec/tast
/build/manatee
@@ -423,6 +368,7 @@
/usr/local/autotest-chrome
/usr/man
/usr/share/aclocal
+ /usr/share/applications
/usr/share/cups/drv
/usr/share/doc
/usr/share/gettext
diff --git a/hooks/filesystem-layout.py b/hooks/filesystem-layout.py
index d824b00..00170c7 100755
--- a/hooks/filesystem-layout.py
+++ b/hooks/filesystem-layout.py
@@ -25,7 +25,7 @@
# Those are: /dev
VALID_ROOT = {
'bin', 'etc', 'home', 'lib', 'lib32', 'lib64', 'media',
- 'mnt', 'opt', 'proc', 'root', 'run', 'sbin', 'sys', 'tmp', 'usr', 'var',
+ 'mnt', 'opt', 'proc', 'root', 'run', 'sbin', 'sys', 'usr', 'var',
}
# Paths that are allowed in the / dir for boards.
@@ -40,7 +40,7 @@
# Paths under / that should not have any subdirs.
NOSUBDIRS_ROOT = {
- 'bin', 'dev', 'proc', 'sbin', 'sys', 'tmp',
+ 'bin', 'dev', 'proc', 'sbin', 'sys',
}
# Paths that are allowed in the /usr dir.
@@ -68,7 +68,6 @@
KNOWN_TARGETS = {
# These are historical names that we want to change to *-cros-* someday.
'arm-none-eabi',
- 'i686-pc-linux-gnu',
# This is the host SDK name.
'x86_64-pc-linux-gnu',
'*-cros-eabi',
@@ -76,13 +75,6 @@
'*-cros-linux-gnu*',
}
-# These packages need fixing.
-# NB: Do *not* add more packages here.
-BAD_ROOT_PACKAGES = {
- # TODO(crbug.com/1003107): Delete this.
- 'dev-go/go-tools',
-}
-
# These SDK packages need cleanup.
# NB: Do *not* add more packages here.
BAD_HOST_USR_LOCAL_PACKAGES = {
@@ -94,7 +86,6 @@
BAD_VAR_PACKAGES = {
'app-accessibility/brltty',
'app-admin/eselect',
- 'app-admin/puppet',
'app-admin/rsyslog',
'app-admin/sudo',
'app-admin/sysstat',
@@ -103,11 +94,7 @@
'app-crypt/trousers',
'app-emulation/containerd',
'app-emulation/lxc',
- # https://crbug.com/1059308
- 'chromeos-base/chromeos-bsp-initramfs-rialto',
'chromeos-base/chromeos-initramfs',
- # https://crbug.com/1054646
- 'chromeos-base/devserver',
'dev-python/django',
'media-gfx/sane-backends',
'media-sound/alsa-utils',
@@ -123,7 +110,6 @@
'sys-apps/dbus',
'sys-apps/fwupd',
'sys-apps/iproute2',
- 'sys-apps/journald',
'sys-apps/portage',
'sys-apps/sandbox',
'sys-apps/systemd',
@@ -142,20 +128,6 @@
'net-fs/samba',
}
-# Ignore some packages installing into /tmp for now.
-# NB: Do *not* add more packages here.
-BAD_TMP_PACKAGES = {
- # https://crbug.com/1057059
- 'chromeos-base/chromeos-bsp-caroline-private',
- 'chromeos-base/chromeos-bsp-elm-private',
- 'chromeos-base/chromeos-config-bsp',
- 'chromeos-base/chromeos-config-bsp-coral',
- 'chromeos-base/chromeos-config-bsp-coral-private',
- 'chromeos-base/chromeos-config-bsp-nami',
- 'chromeos-base/chromeos-config-bsp-scarlet-private',
- 'chromeos-base/cros-config-test',
-}
-
def has_subdirs(path):
"""See if |path| has any subdirs."""
@@ -227,9 +199,7 @@
else:
unknown -= VALID_BOARD_ROOT
- if atom in BAD_ROOT_PACKAGES:
- logging.warning('Ignoring known bad / install for now')
- elif unknown:
+ if unknown:
logging.error('Paths are not allowed in the root dir:\n %s\n |-- %s',
root, '\n |-- '.join(sorted(unknown)))
ret = False
@@ -237,13 +207,10 @@
# Some of these may have subdirs at runtime, but not from package installs.
for path in NOSUBDIRS_ROOT:
if has_subdirs(os.path.join(root, path)):
- if path == 'tmp' and atom in BAD_TMP_PACKAGES:
- logging.warning('Ignoring known bad /tmp install for now')
- else:
- ret = False
+ ret = False
# Special case /var due to so many misuses currently.
- if has_subdirs(os.path.join(root, 'var')):
+ if os.path.exists(os.path.join(root, 'var')):
if atom in BAD_VAR_PACKAGES:
logging.warning('Ignoring known bad /var install for now')
elif os.environ.get('PORTAGE_REPO_NAME') == 'portage-stable':
@@ -256,7 +223,7 @@
logging.warning('Package has improved; please update BAD_VAR_PACKAGES')
# Special case /run due to so many misuses currently.
- if has_subdirs(os.path.join(root, 'run')):
+ if os.path.exists(os.path.join(root, 'run')):
if atom in BAD_RUN_PACKAGES:
logging.warning('Ignoring known bad /run install for now')
elif os.environ.get('PORTAGE_REPO_NAME') == 'portage-stable':
@@ -317,8 +284,9 @@
if not check_root(opts.root, opts.host):
logging.critical(
- "This package does not conform to CrOS's filesystem conventions. "
- 'Please review the paths flagged above and adjust its layout.')
+ "Package '%s' does not conform to CrOS's filesystem conventions. "
+ 'Please review the paths flagged above and adjust its layout.',
+ get_current_package())
return 1
else:
return 0
diff --git a/hooks/install/check-upstart-scripts.sh b/hooks/install/check-upstart-scripts.sh
index f69cd3a..7ff914f 100755
--- a/hooks/install/check-upstart-scripts.sh
+++ b/hooks/install/check-upstart-scripts.sh
@@ -5,7 +5,8 @@
# Various upstart init script checks.
-DOC_RESOURCE_URL="https://dev.chromium.org/chromium-os/chromiumos-design-docs/boot-design#TOC-Runtime-Resource-Limits"
+DOC_RESOURCE_URL="https://www.chromium.org/chromium-os/chromiumos-design-docs/boot-design/#runtime-resource-limits"
+CROS_COMMAND="/mnt/host/source/chromite/bin/cros"
# Default portage vars to make shellcheck happy.
: "${CATEGORY:=}"
@@ -92,7 +93,6 @@
chromeos-base/chromeos-firewall-init|\
chromeos-base/chromeos-firewall-init-mobbase|\
chromeos-base/chromeos-imageburner|\
- chromeos-base/chromeos-init|\
chromeos-base/chromeos-installer|\
chromeos-base/chromeos-login|\
chromeos-base/chromeos-machine-id-regen|\
@@ -156,7 +156,6 @@
chromeos-base/rialto-cellular-autoconnect|\
chromeos-base/rialto-modem-watchdog|\
chromeos-base/runtime_probe|\
- chromeos-base/shill|\
chromeos-base/sirenia|\
chromeos-base/smbprovider|\
chromeos-base/swap-init|\
@@ -256,7 +255,12 @@
# Main entry point for this hook.
check() {
- local arg ret_oom=0
+ local arg ret_oom=0 ret_lint=0
+
+ # If nothing to scan, don't keep running below.
+ if [[ $# -eq 0 ]]; then
+ return
+ fi
for arg in "$@"; do
if [[ -L "${arg}" ]]; then
@@ -267,6 +271,9 @@
: $(( ret_oom += $? ))
done
+ "${CROS_COMMAND}" lint --relaxed "$@"
+ : $(( ret_lint += $? ))
+
if [[ ${ret_oom} -eq 0 ]]; then
if known_bad_oom; then
eqawarn "Please remove ${CATEGORY}/${PN} from known_bad_oom in $0."
@@ -276,6 +283,10 @@
die "Init scripts have errors."
fi
fi
+
+ if [[ ${ret_lint} -ne 0 ]]; then
+ die "Init scripts have lint errors."
+ fi
}
usage() {
diff --git a/hooks/install/gen-package-licenses.sh b/hooks/install/gen-package-licenses.sh
index 0e70f96..fc09899 100755
--- a/hooks/install/gen-package-licenses.sh
+++ b/hooks/install/gen-package-licenses.sh
@@ -5,13 +5,15 @@
generate_licensing()
{
- local PKG="${CATEGORY}/${PF}"
+ local PKG="${CATEGORY:-}/${PF:-}"
# This expands to something like
# /build/x86-alex/tmp/portage/dev-util/libc-bench-0.0.1-r6
# Run FEATURES='noclean' emerge-x86-alex libc-bench to prevent having the
# directory cleaned up if you are debugging.
- einfo "Generating license for ${PKG} in ${PORTAGE_BUILDDIR}"
+ einfo "Generating license for ${PKG} in ${PORTAGE_BUILDDIR:-}"
+ # TODO(b/187794810): Move this to python3.8 once portage supports it.
+ python3.6 \
/mnt/host/source/chromite/licensing/ebuild_license_hook \
--builddir "${PORTAGE_BUILDDIR}" || die "
Failed Generating Licensing for ${PKG}
diff --git a/hooks/install/qa-elf.sh b/hooks/install/qa-elf.sh
index 3e1656c..60fca47 100755
--- a/hooks/install/qa-elf.sh
+++ b/hooks/install/qa-elf.sh
@@ -42,6 +42,10 @@
check_binaries()
{
+ # We don't care as much about security checks in the SDK itself.
+ if [[ "${ROOT:-/}" == "/" ]] ; then
+ return
+ fi
if [[ " ${RESTRICT} " == *" binchecks "* ]] ; then
return
fi
diff --git a/mod_for_test_scripts/100setupTestingInterface b/mod_for_test_scripts/100setupTestingInterface
index 662aaec..24530ec 100755
--- a/mod_for_test_scripts/100setupTestingInterface
+++ b/mod_for_test_scripts/100setupTestingInterface
@@ -16,7 +16,6 @@
ORIG_CONF=${ROOT_FS_DIR}/etc/init/shill.conf
TEMP_CONF=${ORIG_CONF}.tmp
sed -e "s@SHILL_TEST_DEVICES=\"\"@SHILL_TEST_DEVICES=\"${test_devices}\"@" \
- -e "s@SHILL_ACCEPT_HOSTNAME_FROM=@SHILL_ACCEPT_HOSTNAME_FROM=\"pseudoethernet0\"@" \
${ORIG_CONF} > ${TEMP_CONF}
mv -f ${TEMP_CONF} ${ORIG_CONF}
diff --git a/mod_image_for_recovery.sh b/mod_image_for_recovery.sh
index ca076c3..2988b5a 100755
--- a/mod_image_for_recovery.sh
+++ b/mod_image_for_recovery.sh
@@ -503,6 +503,7 @@
fi
FACTORY_ROOT="${BOARD_ROOT}/factory-root"
+: "${USE:=}"
if [ -z "${FLAGS_kernel_image}" ]; then
# Build the recovery kernel.
diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh
index a1af4b2..90306e0 100755
--- a/sdk_lib/enter_chroot.sh
+++ b/sdk_lib/enter_chroot.sh
@@ -83,6 +83,10 @@
FILES_TO_COPY_TO_CHROOT=(
# Creds used to authenticate with LUCI services.
.config/chrome_infra/auth/creds.json
+
+ # Creds used to authenticate with GCP services.
+ .config/gcloud/application_default_credentials.json
+
.gdata_cred.txt # User/password for Google Docs on chromium.org
.gdata_token # Auth token for Google Docs on chromium.org
.goma_client_oauth2_config # Auth token for Goma
@@ -103,10 +107,6 @@
LOCKFILE="${FLAGS_chroot}/.enter_chroot.lock"
MOUNTED_PATH=$(readlink -f "$FLAGS_chroot")
-# Reset the depot tools/internal trunk pathways to what they'll
-# be w/in the chroot.
-set_chroot_trunk_dir "${FLAGS_chroot}"
-
# Writes stdin to the given file name as the sudo user in overwrite mode.
#
# $@ - The output file names.
@@ -161,10 +161,14 @@
# Already mounted!
;;
*)
- # If it doesn't exist, assume they want a dir. But don't blindly run
- # it all the time in case they're trying to bind mount a file.
- if [[ ! -e ${mounted_path} ]]; then
- mkdir -p "${mounted_path}"
+ # Don't blindly mkdir in case they're trying to bind mount a file.
+ if [[ ! -e "${mounted_path}" ]]; then
+ if [[ -d "${source}" ]]; then
+ mkdir -p "${mounted_path}"
+ elif [[ -f "${source}" ]]; then
+ mkdir -p "$(dirname "${mounted_path}")"
+ touch "${mounted_path}"
+ fi
fi
# The args are left unquoted on purpose.
if [[ -n ${source} ]]; then
@@ -482,6 +486,8 @@
user_mkdir "${FLAGS_cache_dir}"/distfiles
user_mkdir "${FLAGS_chroot}/${chroot_cache}"
setup_mount "${FLAGS_cache_dir}" "--bind" "${chroot_cache}"
+ # Create /var/log/asan directory (b/222311476).
+ user_mkdir "${FLAGS_chroot}/var/log/asan"
# TODO(build): remove this as of 12/01/12.
# Because of how distfiles -> cache_dir was deployed, if this isn't
# a symlink, we *know* the ondisk pathways aren't compatible- thus
diff --git a/sdk_lib/make_chroot.sh b/sdk_lib/make_chroot.sh
index c88eac7..4c1aa23 100755
--- a/sdk_lib/make_chroot.sh
+++ b/sdk_lib/make_chroot.sh
@@ -35,6 +35,8 @@
DEFINE_string chroot "$DEFAULT_CHROOT_DIR" \
"Destination dir for the chroot environment."
+DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
+ "Skip automatic SDK/toolchain upgrade. Will eventually break as ToT moves on."
DEFINE_boolean usepkg $FLAGS_TRUE "Use binary packages to bootstrap."
DEFINE_integer jobs -1 "How many packages to build in parallel at maximum."
DEFINE_string cache_dir "" "Directory to store caches within."
@@ -117,7 +119,6 @@
init_setup () {
info "Running init_setup()..."
mkdir -p -m 755 "${FLAGS_chroot}/usr" \
- "${FLAGS_chroot}${OVERLAYS_ROOT}" \
"${FLAGS_chroot}"/"${CROSSDEV_OVERLAY}/metadata"
# Newer portage complains about bare overlays. Create the file that crossdev
# will also create later on.
@@ -129,12 +130,6 @@
use-manifests = true
thin-manifests = true
EOF
- ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/eclass-overlay" \
- "${FLAGS_chroot}"/"${ECLASS_OVERLAY}"
- ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/chromiumos-overlay" \
- "${FLAGS_chroot}"/"${CHROOT_OVERLAY}"
- ln -sf "${CHROOT_TRUNK_DIR}/src/third_party/portage-stable" \
- "${FLAGS_chroot}"/"${PORTAGE_STABLE_OVERLAY}"
# Use the standardized upgrade script to setup proxied vars.
load_environment_whitelist
@@ -158,7 +153,9 @@
mkdir -p "${FLAGS_chroot}/etc/portage"
ln -sf "${CHROOT_CONFIG}/make.conf.amd64-host" \
"${FLAGS_chroot}/etc/make.conf"
- ln -sf "${CHROOT_OVERLAY}/profiles/default/linux/amd64/10.0/sdk" \
+ local cros_overlay="${CHROOT_TRUNK_DIR}/src/third_party/chromiumos-overlay"
+ ln -sf \
+ "${cros_overlay}/profiles/default/linux/amd64/10.0/sdk" \
"${FLAGS_chroot}/etc/portage/make.profile"
# Create make.conf.user .
@@ -208,13 +205,6 @@
info "Running post-inst configuration hacks"
early_enter_chroot env-update
- # This is basically a sanity check of our chroot. If any of these
- # don't exist, then either bind mounts have failed, an invocation
- # from above is broke, or some assumption about the stage3 is no longer
- # true.
- early_enter_chroot ls -l /etc/make.conf /etc/portage/make.profile \
- /usr/local/portage/chromiumos/profiles/default/linux/amd64/10.0
-
target="${FLAGS_chroot}/etc/profile.d"
mkdir -p "${target}"
ln -sfT \
@@ -242,10 +232,7 @@
CONFIG_DIR="${OVERLAY}/chromeos/config"
CHROOT_CONFIG="${CHROOT_TRUNK_DIR}/src/third_party/chromiumos-overlay/chromeos/config"
OVERLAYS_ROOT="/usr/local/portage"
-ECLASS_OVERLAY="${OVERLAYS_ROOT}/eclass-overlay"
-PORTAGE_STABLE_OVERLAY="${OVERLAYS_ROOT}/stable"
CROSSDEV_OVERLAY="${OVERLAYS_ROOT}/crossdev"
-CHROOT_OVERLAY="${OVERLAYS_ROOT}/chromiumos"
# Pass proxy variables into the environment.
for type in http ftp all; do
@@ -273,9 +260,10 @@
eclean -e <(get_eclean_exclusions) packages'
fi
-info "Updating portage"
-early_enter_chroot emerge -uNv --quiet --ignore-world portage
-
+if [[ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_FALSE}" ]]; then
+ info "Updating portage"
+ early_enter_chroot emerge -uNv --quiet --ignore-world portage
+fi
# Add chromite into python path.
for python_path in "${FLAGS_chroot}/usr/lib/"python*.*; do
python_path+="/site-packages"
@@ -283,50 +271,53 @@
sudo ln -s -fT "${CHROOT_TRUNK_DIR}"/chromite "${python_path}"/chromite
done
-# Now that many of the fundamental packages should be in a good state, update
-# the host toolchain. We have to do this step by step ourselves to avoid races
-# when building tools that are actively used (e.g. updating the assembler while
-# also compiling other packages that use the assembler).
-# https://crbug.com/715788
-info "Updating host toolchain"
-TOOLCHAIN_ARGS=( --deleteold )
-if [[ "${FLAGS_usepkg}" == "${FLAGS_FALSE}" ]]; then
- TOOLCHAIN_ARGS+=( --nousepkg )
-fi
-# First the low level compiler tools. These should be fairly independent of
-# the C library, so we can do it first.
-early_enter_chroot ${EMERGE_CMD} -uNv ${USEPKG} ${USEPKGONLY} ${EMERGE_JOBS} \
- sys-devel/binutils
-# Next the C library. The compilers often use newer features, but the C library
-# is often designed to work with older compilers.
-early_enter_chroot ${EMERGE_CMD} -uNv ${USEPKG} ${USEPKGONLY} ${EMERGE_JOBS} \
- sys-kernel/linux-headers sys-libs/glibc
-# Now we can let the rest of the compiler packages build in parallel as they
-# don't generally rely on each other.
-# Note: early_enter_chroot executes as root.
-early_enter_chroot "${CHROOT_TRUNK_DIR}/chromite/bin/cros_setup_toolchains" \
- --hostonly "${TOOLCHAIN_ARGS[@]}"
+if [[ "${FLAGS_skip_chroot_upgrade}" -eq "${FLAGS_FALSE}" ]]; then
+ # Now that many of the fundamental packages should be in a good state, update
+ # the host toolchain. We have to do this step by step ourselves to avoid
+ # races when building tools that are actively used (e.g. updating the
+ # assembler while also compiling other packages that use the assembler).
+ # https://crbug.com/715788
+ info "Updating host toolchain"
+ TOOLCHAIN_ARGS=( --deleteold )
+ if [[ "${FLAGS_usepkg}" == "${FLAGS_FALSE}" ]]; then
+ TOOLCHAIN_ARGS+=( --nousepkg )
+ fi
+ # First the low level compiler tools. These should be fairly independent of
+ # the C library, so we can do it first.
+ early_enter_chroot ${EMERGE_CMD} -uNv ${USEPKG} ${USEPKGONLY} ${EMERGE_JOBS} \
+ sys-devel/binutils
+ # Next the C library. The compilers often use newer features,
+ # but the C library is often designed to work with older compilers.
+ early_enter_chroot ${EMERGE_CMD} -uNv ${USEPKG} ${USEPKGONLY} ${EMERGE_JOBS} \
+ sys-kernel/linux-headers sys-libs/glibc
-info "Updating Perl modules"
-early_enter_chroot \
- "${CHROOT_TRUNK_DIR}/src/scripts/build_library/perl_rebuild.sh"
+ # Next libcxx. This is required due to the migration to LLVM runtime builds.
+ early_enter_chroot ${EMERGE_CMD} -uN --nodeps ${USEPKG} \
+ sys-libs/libcxxabi sys-libs/libcxx
-# If we're creating a new chroot, we also want to set it to the latest version.
-enter_chroot run_chroot_version_hooks --init-latest
+ # Now we can let the rest of the compiler packages build in parallel as they
+ # don't generally rely on each other.
+ # Note: early_enter_chroot executes as root.
+ early_enter_chroot "${CHROOT_TRUNK_DIR}/chromite/bin/cros_setup_toolchains" \
+ --hostonly "${TOOLCHAIN_ARGS[@]}"
-# Update chroot.
-# Skip toolchain update because it already happened above, and the chroot is
-# not ready to emerge all cross toolchains.
-UPDATE_ARGS=( --skip_toolchain_update --noeclean )
-if [[ "${FLAGS_usepkg}" == "${FLAGS_TRUE}" ]]; then
- UPDATE_ARGS+=( --usepkg )
+ # Update chroot.
+ # Skip toolchain update because it already happened above, and the chroot is
+ # not ready to emerge all cross toolchains.
+ UPDATE_ARGS=( --skip_toolchain_update --noeclean )
+ if [[ "${FLAGS_usepkg}" == "${FLAGS_TRUE}" ]]; then
+ UPDATE_ARGS+=( --usepkg )
+ else
+ UPDATE_ARGS+=( --nousepkg )
+ fi
+ if [[ "${FLAGS_jobs}" -ne -1 ]]; then
+ UPDATE_ARGS+=( --jobs="${FLAGS_jobs}" )
+ fi
+ enter_chroot "${CHROOT_TRUNK_DIR}/src/scripts/update_chroot" \
+ "${UPDATE_ARGS[@]}"
else
- UPDATE_ARGS+=( --nousepkg )
+ warn "SDK and toolchain update were skipped. It will eventually stop working."
fi
-if [[ "${FLAGS_jobs}" -ne -1 ]]; then
- UPDATE_ARGS+=( --jobs="${FLAGS_jobs}" )
-fi
-enter_chroot "${CHROOT_TRUNK_DIR}/src/scripts/update_chroot" "${UPDATE_ARGS[@]}"
# The java-config package atm does not support $ROOT. Select a default
# VM ourselves until that gets fixed upstream.
diff --git a/sdk_lib/make_conf_util.sh b/sdk_lib/make_conf_util.sh
index 6da7961..9865032 100644
--- a/sdk_lib/make_conf_util.sh
+++ b/sdk_lib/make_conf_util.sh
@@ -22,7 +22,7 @@
output_opt="-O"
else
cmd=curl
- options="--ipv4 -f -y 30 --retry 9 -L"
+ options="-f -y 30 --retry 9 -L"
resume_opt="-C -"
output_opt="--output"
fi
diff --git a/update_chroot b/update_chroot
index 15e1158..782365f 100755
--- a/update_chroot
+++ b/update_chroot
@@ -14,7 +14,7 @@
assert_not_root_user
# Developer-visible flags.
-DEFINE_boolean usepkg $FLAGS_TRUE \
+DEFINE_boolean usepkg "${FLAGS_TRUE}" \
"Use binary packages to bootstrap."
FLAGS_HELP="usage: $(basename $0) [flags]
@@ -27,11 +27,12 @@
# not needed for the typical developer workflow.
DEFINE_integer jobs -1 \
"How many packages to build in parallel at maximum."
-DEFINE_boolean skip_toolchain_update $FLAGS_FALSE \
+DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
"Don't update the toolchains."
DEFINE_string toolchain_boards "" \
"Extra toolchains to setup for the specified boards."
DEFINE_boolean eclean "${FLAGS_TRUE}" "Run eclean to delete old binpkgs."
+DEFINE_integer backtrack 10 "See emerge --backtrack."
# Parse command line flags
FLAGS "$@" || exit 1
@@ -41,7 +42,7 @@
# so will die prematurely if 'switch_to_strict_mode' is specified before now.
switch_to_strict_mode
-. ${SCRIPTS_DIR}/sdk_lib/make_conf_util.sh
+. "${SCRIPTS_DIR}"/sdk_lib/make_conf_util.sh
# Run version hooks as pre-update
run_chroot_version_hooks
@@ -70,7 +71,6 @@
if [[ -n ${FLAGS_toolchain_boards} ]]; then
TOOLCHAIN_FLAGS+=(
- "--targets=boards"
"--include-boards=${FLAGS_toolchain_boards}"
)
fi
@@ -104,7 +104,7 @@
info "Updating the SDK"
-EMERGE_FLAGS=( -uNv --with-bdeps=y )
+EMERGE_FLAGS=( -uNv --with-bdeps=y --backtrack="${FLAGS_backtrack}" )
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
EMERGE_FLAGS+=( --getbinpkg )
@@ -133,9 +133,14 @@
EMERGE_FLAGS+=( --reinstall-atoms="${pkg}" --usepkg-exclude="${pkg}" )
done
+# Regenerate docbook catalog to avoid manpage compilation errors.
+info_run sudo -E "${EMERGE_CMD}" "${EMERGE_FLAGS[@]}" \
+ app-text/docbook-xml-dtd app-text/build-docbook-catalog
+info_run sudo build-docbook-catalog
+
# Second pass, update everything else.
EMERGE_FLAGS+=( --deep )
-info_run sudo -E ${EMERGE_CMD} "${EMERGE_FLAGS[@]}" virtual/target-sdk world
+info_run sudo -E "${EMERGE_CMD}" "${EMERGE_FLAGS[@]}" virtual/target-sdk world
# Install post cross packages if binary pkgs are available.
if [ "${FLAGS_usepkg}" -eq "${FLAGS_TRUE}" ]; then
@@ -145,7 +150,7 @@
EMERGE_FLAGS+=(
$("${CHROMITE_BIN}/cros_setup_toolchains" --show-packages host-post-cross)
)
- sudo -E ${EMERGE_CMD} "${EMERGE_FLAGS[@]}"
+ sudo -E "${EMERGE_CMD}" "${EMERGE_FLAGS[@]}"
# Install nobdeps packages only when binary pkgs are available, since we don't
# want to accidentally pull in build deps for a rebuild.
diff --git a/update_kernel.sh b/update_kernel.sh
index a082e1a..acc29db 100755
--- a/update_kernel.sh
+++ b/update_kernel.sh
@@ -13,6 +13,14 @@
# Script must be run inside the chroot.
restart_in_chroot_if_needed "$@"
+FLAGS_HELP="usage: $(basename "$0") [flags]
+
+Updates the kernel on a target CrOS system. Supports both real hardware and
+QEMU VMs (e.g., betty/amd64-generic with 'cros_vm'). Note that there are some
+assumptions about the CrOS verified boot chain, so command-line updates may not
+work well on non-vboot systems (such as BIOS or EFI boot on QEMU).
+"
+
DEFINE_string board "" "Override board reported by target"
DEFINE_string device "" "Override boot device reported by target"
DEFINE_string partition "" "Override kernel partition reported by target"
@@ -70,15 +78,6 @@
info "Target reports root device is ${FLAGS_device}"
}
-# Delete the fixed numbers after R65 when we don't care about <R57 upgrades.
-load_default_partition_numbers() {
- PARTITION_NUM_KERN_A=2
- PARTITION_NUM_ROOT_A=3
- PARTITION_NUM_KERN_B=4
- PARTITION_NUM_ROOT_B=5
- PARTITION_NUM_EFI_SYSTEM=12
-}
-
# Ask the target what the kernel partition is
learn_partition_and_ro() {
remote_sh rootdev || die_notrace
@@ -269,6 +268,14 @@
update_syslinux_kernel() {
local basedir="$1" # rootfs directory (could be in /tmp) or empty string
+
+ # All systems _should_ have the EFI variables defined, but we already make
+ # syslinux updates optional, so make this optional too.
+ if [[ -z "${PARTITION_NUM_EFI_SYSTEM}" ]]; then
+ warn "Target is missing EFI partition info; skipping syslinux"
+ return
+ fi
+
# ARM does not have the syslinux directory, so skip it when the
# partition is missing, the file system fails to mount, or the syslinux
# vmlinuz target is missing.
@@ -326,9 +333,11 @@
learn_device
learn_partition_layout
- if [[ -z "${PARTITION_NUM_KERN_A}" ]]; then
- info "Target has no partition number info, use default instead"
- load_default_partition_numbers
+ if [[ -z "${PARTITION_NUM_KERN_A}" ||
+ -z "${PARTITION_NUM_KERN_B}" ||
+ -z "${PARTITION_NUM_ROOT_A}" ||
+ -z "${PARTITION_NUM_ROOT_B}" ]]; then
+ die_notrace "Target is missing partition number info"
fi
learn_partition_and_ro
@@ -360,7 +369,8 @@
if [[ ${FLAGS_syslinux} -eq ${FLAGS_TRUE} ]]; then
if [[ ${FLAGS_clean} -eq ${FLAGS_TRUE} ]]; then
info "Cleaning /boot"
- remote_sh rm -rf "${remote_basedir}"/boot/{System,config,vmlinuz}'*'
+ remote_sh rm -rf \
+ "${remote_basedir}"/boot/{Image,System,config,vmlinuz}'*'
fi
info "Copying syslinux and /boot"
remote_send_to /build/"${FLAGS_board}"/boot/ "${remote_basedir}"/boot/
