blob: b7e74d75aeeba8e0d260d73cd11c6fd531be233a [file] [log] [blame]
# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Common constants for build scripts
# This must evaluate properly for both /bin/bash and /bin/sh
# All scripts should die on error unless commands are specifically excepted
# by prefixing with '!' or surrounded by 'set +e' / 'set -e'.
# TODO: Re-enable this once shflags is less prone to dying.
#set -e
# The number of jobs to pass to tools that can run in parallel (such as make
# and dpkg-buildpackage
NUM_JOBS=$(grep -c "^processor" /proc/cpuinfo)
# True if we have the 'pv' utility - also set up COMMON_PV_CAT for convenience
pv -V >/dev/null 2>&1 || COMMON_PV_OK=0
if [ $COMMON_PV_OK -eq 0 ]; then
# Make sure we have the location and name of the calling script, using
# the current value if it is already set.
SCRIPT_LOCATION=${SCRIPT_LOCATION:-$(dirname "$(readlink -f "$0")")}
SCRIPT_NAME=${SCRIPT_NAME:-$(basename "$0")}
# Detect whether we're inside a chroot or not
if [ -e /etc/debian_chroot ]
# Construct a list of possible locations for the source tree. This list is
# based on various environment variables and globals that may have been set
# by the calling script.
function get_gclient_root_list() {
if [ $INSIDE_CHROOT -eq 1 ]; then
echo "/home/${USER}/trunk"
if [ -n "${SUDO_USER}" ]; then echo "/home/${SUDO_USER}/trunk"; fi
if [ -n "${COMMON_SH}" ]; then echo "$(dirname "$COMMON_SH")/../.."; fi
if [ -n "${BASH_SOURCE}" ]; then echo "$(dirname "$BASH_SOURCE")/../.."; fi
# Based on the list of possible source locations we set GCLIENT_ROOT if it is
# not already defined by looking for a src directory in each seach path
# location. If we do not find a valid looking root we error out.
function get_gclient_root() {
if [ -n "${GCLIENT_ROOT}" ]; then
for path in $(get_gclient_root_list); do
if [ -d "${path}/src" ]; then
if [ -z "${GCLIENT_ROOT}" ]; then
# Using dash or sh, we don't know where we are. $0 refers to the calling
# script, not ourselves, so that doesn't help us.
echo "Unable to determine location for If you are sourcing"
echo " from a script run via dash or sh, you must do it in the"
echo "following way:"
echo ' COMMON_SH="$(dirname "$0")/../../scripts/"'
echo ' . "$COMMON_SH"'
echo "where the first line is the relative path from your script to"
echo ""
exit 1
# Find root of source tree
# Canonicalize the directories for the root dir and the calling script.
# readlink is part of coreutils and should be present even in a bare chroot.
# This is better than just using
# FOO = "$(cd $FOO ; pwd)"
# since that leaves symbolic links intact.
# Note that 'realpath' is equivalent to 'readlink -f'.
# Other directories should always be pathed down from GCLIENT_ROOT.
# Load developer's custom settings. Default location is in scripts dir,
# since that's available both inside and outside the chroot. By convention,
# settings from this file are variables starting with 'CHROMEOS_'
if [ -f "$CHROMEOS_DEV_SETTINGS" ]; then
# Turn on exit-on-error during custom settings processing
SAVE_OPTS=$(set +o)
set -e
# Read settings
# Restore previous state of exit-on-error
eval "$SAVE_OPTS"
# Load shflags
# NOTE: This code snippet is in particular used by the au-generator (which
# stores shflags in ./lib/shflags/) and should not be touched.
if [ -f "${SCRIPTS_DIR}/lib/shflags/shflags" ]; then
. "${SCRIPTS_DIR}/lib/shflags/shflags"
. ./lib/shflags/shflags || die "Couldn't find shflags"
# Our local mirror
# Upstream mirrors and build suites come in 2 flavors
# DEV - development chroot, used to build the chromeos image
# IMG - bootable image, to run on actual hardware
# Default location for chroot
# All output files from build should go under $DEFAULT_BUILD_ROOT, so that
# they don't pollute the source directory.
# Set up a global ALL_BOARDS value
if [ -d "$SRC_ROOT/overlays" ]; then
ALL_BOARDS=$(cd "$SRC_ROOT/overlays"; \
ls -1d overlay-* 2>&- | sed 's,overlay-,,g')
# Strip CR
# Set a default BOARD
#DEFAULT_BOARD=x86-generic # or...
DEFAULT_BOARD=$(echo $ALL_BOARDS | awk '{print $NF}')
# Enable --fast by default.
# Directory to store built images. Should be set by sourcing script when used.
# Standard filenames
# Directory locations inside the dev chroot
# Install make for portage ebuilds. Used by build_image and gmergefs.
# TODO: Is /usr/local/autotest-chrome still used by anyone?
# Determine and set up variables needed for fancy color output (if supported).
if tput colors >/dev/null 2>&1; then
# order matters: we want VIDOFF last so that when we trace with `set -x`,
# our terminal doesn't bleed colors as bash dumps the values of vars.
V_BOLD_RED="$(tput bold; tput setaf 1)"
V_BOLD_GREEN="$(tput bold; tput setaf 2)"
V_BOLD_YELLOW="$(tput bold; tput setaf 3)"
V_REVERSE="$(tput rev)"
V_VIDOFF="$(tput sgr0)"
# -----------------------------------------------------------------------------
# Functions
function setup_board_warning {
echo "$V_REVERSE================= WARNING ======================$V_VIDOFF"
echo "*** No default board detected in " \
echo "*** Either run setup_board with default flag set"
echo "*** or echo |board_name| > $GCLIENT_ROOT/src/scripts/.default_board"
# Sets the default board variable for calling script
function get_default_board {
if [ -f "$GCLIENT_ROOT/src/scripts/.default_board" ] ; then
DEFAULT_BOARD=$(cat "$GCLIENT_ROOT/src/scripts/.default_board")
# Enter a chroot and restart the current script if needed
function restart_in_chroot_if_needed {
# NB: Pass in ARGV: restart_in_chroot_if_needed "$@"
if [ $INSIDE_CHROOT -ne 1 ]; then
# Get inside_chroot path for script.
local chroot_path="$(reinterpret_path_for_chroot "$0")"
exec $GCLIENT_ROOT/chromite/bin/cros_sdk -- "$chroot_path" "$@"
# Fail unless we're inside the chroot. This guards against messing up your
# workstation.
function assert_inside_chroot {
if [ $INSIDE_CHROOT -ne 1 ]; then
echo "This script must be run inside the chroot. Run this first:"
echo " cros_sdk"
exit 1
# Fail if we're inside the chroot. This guards against creating or entering
# nested chroots, among other potential problems.
function assert_outside_chroot {
if [ $INSIDE_CHROOT -ne 0 ]; then
echo "This script must be run outside the chroot."
exit 1
function assert_not_root_user {
if [ $(id -u) = 0 ]; then
echo "This script must be run as a non-root user."
exit 1
# Check that all arguments are flags; that is, there are no remaining arguments
# after parsing from shflags. Allow (with a warning) a single empty-string
# argument.
# TODO: fix buildbot so that it doesn't pass the empty-string parameter,
# then change this function.
# Usage: check_flags_only_and_allow_null_arg "$@" && set --
function check_flags_only_and_allow_null_arg {
if [[ $# == 1 && -z "$@" ]]; then
echo "$0: warning: ignoring null argument" >&2
if [[ $# -gt 0 ]]; then
echo "error: invalid arguments: \"$@\"" >&2
exit 1
return $do_shift
function info {
echo -e >&2 "${V_BOLD_GREEN}INFO ${CROS_LOG_PREFIX:-""}: $@${V_VIDOFF}"
function warn {
function error {
echo -e >&2 "${V_BOLD_RED}ERROR ${CROS_LOG_PREFIX:-""}: $@${V_VIDOFF}"
function die {
error "$@"
exit 1
# Retry an emerge command according to $FLAGS_retries
# The $EMERGE_JOBS flags will only be added the first time the command is run
function eretry () {
local i
for i in $(seq $FLAGS_retries); do
echo "Retrying $@"
"$@" $EMERGE_JOBS && return 0
"$@" && return 0
return 1
# Removes single quotes around parameter
# Arguments:
# $1 - string which optionally has surrounding quotes
# Returns:
# None, but prints the string without quotes.
function remove_quotes() {
echo "$1" | sed -e "s/^'//; s/'$//"
# Writes stdin to the given file name as root using sudo in overwrite mode.
# $1 - The output file name.
function sudo_clobber() {
sudo tee "$1" > /dev/null
# Writes stdin to the given file name as root using sudo in append mode.
# $1 - The output file name.
function sudo_append() {
sudo tee -a "$1" > /dev/null
# Locate all mounts below a specified directory.
# $1 - The root tree.
function sub_mounts() {
# Assume that `mount` outputs a list of mount points in the order
# that things were mounted (since it always has and hopefully always
# will). As such, we have to unmount in reverse order to cleanly
# unmount submounts (think /dev/pts and /dev).
mount | \
awk -v path="$1" -v len="${#1}" \
'(substr($3, 1, len) == path) { print $3 }' | \
# Unmounts a directory, if the unmount fails, warn, and then lazily unmount.
# $1 - The path to unmount.
function safe_umount_tree {
local mounts=$(sub_mounts "$1")
# Hmm, this shouldn't normally happen, but anything is possible.
if [ -z "${mounts}" ] ; then
return 0
# First try to unmount in one shot to speed things up.
if sudo umount -d ${mounts}; then
return 0
# Well that didn't work, so lazy unmount remaining ones.
mounts=$(sub_mounts "$1")
warn "Failed to unmount ${mounts}"
warn "Doing a lazy unmount"
if ! sudo umount -d -l ${mounts}; then
mounts=$(sub_mounts "$1")
die "Failed to lazily unmount ${mounts}"
# Fixes symlinks that are incorrectly prefixed with the build root ${1}
# rather than the real running root '/'.
# TODO(sosa) - Merge setup - cleanup below with this method.
fix_broken_symlinks() {
local build_root="${1}"
local symlinks=$(find "${build_root}/usr/local" -lname "${build_root}/*")
local symlink
for symlink in ${symlinks}; do
echo "Fixing ${symlink}"
local target=$(ls -l "${symlink}" | cut -f 2 -d '>')
# Trim spaces from target (bashism).
target=${target/ /}
# Make new target (removes rootfs prefix).
new_target=$(echo ${target} | sed "s#${build_root}##")
echo "Fixing symlink ${symlink}"
sudo unlink "${symlink}"
sudo ln -sf "${new_target}" "${symlink}"
# Sets up symlinks for the developer root. It is necessary to symlink
# usr and local since the developer root is mounted at /usr/local and
# applications expect to be installed under /usr/local/bin, etc.
# This avoids packages installing into /usr/local/usr/local/bin.
# ${1} specifies the symlink target for the developer root.
# ${2} specifies the symlink target for the var directory.
# ${3} specifies the location of the stateful partition.
setup_symlinks_on_root() {
# Give args better names.
local dev_image_target=${1}
local var_target=${2}
local dev_image_root="${3}/dev_image"
# If our var target is actually the standard var, we are cleaning up the
# symlinks (could also check for /usr/local for the dev_image_target).
if [ ${var_target} = "/var" ]; then
echo "Cleaning up /usr/local symlinks for ${dev_image_root}"
echo "Setting up symlinks for /usr/local for ${dev_image_root}"
# Set up symlinks that should point to ${dev_image_target}.
local path
for path in usr local; do
if [ -h "${dev_image_root}/${path}" ]; then
sudo unlink "${dev_image_root}/${path}"
elif [ -e "${dev_image_root}/${path}" ]; then
die "${dev_image_root}/${path} should be a symlink if exists"
sudo ln -s "${dev_image_target}" "${dev_image_root}/${path}"
# Setup var symlink.
if [ -h "${dev_image_root}/var" ]; then
sudo unlink "${dev_image_root}/var"
elif [ -e "${dev_image_root}/var" ]; then
die "${dev_image_root}/var should be a symlink if it exists"
sudo ln -s "${var_target}" "${dev_image_root}/var"
# These two helpers clobber the ro compat value in our root filesystem.
# When the system is built with --enable_rootfs_verification, bit-precise
# integrity checking is performed. That precision poses a usability issue on
# systems that automount partitions with recognizable filesystems, such as
# ext2/3/4. When the filesystem is mounted 'rw', ext2 metadata will be
# automatically updated even if no other writes are performed to the
# filesystem. In addition, ext2+ does not support a "read-only" flag for a
# given filesystem. That said, forward and backward compatibility of
# filesystem features are supported by tracking if a new feature breaks r/w or
# just write compatibility. We abuse the read-only compatibility flag[1] in
# the filesystem header by setting the high order byte (le) to FF. This tells
# the kernel that features R24-R31 are all enabled. Since those features are
# undefined on all ext-based filesystem, all standard kernels will refuse to
# mount the filesystem as read-write -- only read-only[2].
# [1] 32-bit flag we are modifying:
# [2] Mount behavior is enforced here:
# N.B., if the high order feature bits are used in the future, we will need to
# revisit this technique.
disable_rw_mount() {
local rootfs="$1"
local offset="${2-0}" # in bytes
local ro_compat_offset=$((0x464 + 3)) # Set 'highest' byte
printf '\377' |
sudo dd of="$rootfs" seek=$((offset + ro_compat_offset)) \
conv=notrunc count=1 bs=1
enable_rw_mount() {
local rootfs="$1"
local offset="${2-0}"
local ro_compat_offset=$((0x464 + 3)) # Set 'highest' byte
printf '\000' |
sudo dd of="$rootfs" seek=$((offset + ro_compat_offset)) \
conv=notrunc count=1 bs=1
# Get current timestamp. Assumes runs at startup.
start_time=$(date +%s)
# Print time elsapsed since start_time.
print_time_elapsed() {
local end_time=$(date +%s)
local elapsed_seconds=$(($end_time - $start_time))
local minutes=$(($elapsed_seconds / 60))
local seconds=$(($elapsed_seconds % 60))
echo "Elapsed time: ${minutes}m${seconds}s"
# The board and variant command line options can be used in a number of ways
# to specify the board and variant. The board can encode both pieces of
# information separated by underscores. Or the variant can be passed using
# the separate variant option. This function extracts the canonical board and
# variant information and provides it in the BOARD, VARIANT and BOARD_VARIANT
# variables.
get_board_and_variant() {
local flags_board="${1}"
local flags_variant="${2}"
BOARD=$(echo "$flags_board" | cut -d '_' -f 1)
VARIANT=${flags_variant:-$(echo "$flags_board" | cut -s -d '_' -f 2)}
if [ -n "$VARIANT" ]; then
# This function converts a chromiumos image into a test image, either
# in place or by copying to a new test image filename first. It honors
# the following flags (see
# --factory
# --factory_install
# --force_copy
# On entry, pass the directory containing the image, and the image filename
# On exit, it puts the pathname of the resulting test image into
# (yes this is ugly, but perhaps less ugly than the alternatives)
# Usage:
# SRC_IMAGE=$(prepare_test_image "directory" "imagefile")
prepare_test_image() {
# If we're asked to modify the image for test, then let's make a copy and
# modify that instead.
# Check for manufacturing image.
local args
if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ]; then
# Check for install shim.
if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ]; then
# Check for forcing copy of image
if [ ${FLAGS_force_copy} -eq ${FLAGS_TRUE} ]; then
args="${args} --force_copy"
# Modify the image for test, creating a new test image
"${SCRIPTS_DIR}/" --board=${FLAGS_board} \
--image="$1/$2" --noinplace ${args}
# From now on we use the just-created test image
if [ ${FLAGS_factory} -eq ${FLAGS_TRUE} ]; then
# Check that the specified file exists. If the file path is empty or the file
# doesn't exist on the filesystem generate useful error messages. Otherwise
# show the user the name and path of the file that will be used. The padding
# parameter can be used to tabulate multiple name:path pairs. For example:
# check_for_file "really long name" "...:" ""
# check_for_file "short name" ".........:" ""
# Results in the following output:
# Using really long name...:
# Using short name.........:
# If tabulation is not required then passing "" for padding generates the
# output "Using <name> <path>"
check_for_file() {
local name=$1
local padding=$2
local path=$3
if [ -z "${path}" ]; then
die "No ${name} file specified."
if [ ! -e "${path}" ]; then
die "No ${name} file found at: ${path}"
info "Using ${name}${padding} ${path}"
# Check that the specified tool exists. If it does not exist in the PATH
# generate a useful error message indicating how to install the ebuild
# that contains the required tool.
check_for_tool() {
local tool=$1
local ebuild=$2
if ! which "${tool}" >/dev/null ; then
error "The ${tool} utility was not found in your path. Run the following"
error "command in your chroot to install it: sudo -E emerge ${ebuild}"
exit 1
# Reinterprets path from outside the chroot for use inside.
# Returns "" if "" given.
# $1 - The path to reinterpret.
function reinterpret_path_for_chroot() {
if [ $INSIDE_CHROOT -ne 1 ]; then
if [ -z "${1}" ]; then
echo ""
local path_abs_path=$(readlink -f "${1}")
local gclient_root_abs_path=$(readlink -f "${GCLIENT_ROOT}")
# Strip the repository root from the path.
local relative_path=$(echo ${path_abs_path} \
| sed "s:${gclient_root_abs_path}/::")
if [ "${relative_path}" = "${path_abs_path}" ]; then
die "Error reinterpreting path. Path ${1} is not within source tree."
# Prepend the chroot repository path.
echo "/home/${USER}/trunk/${relative_path}"
# Path is already inside the chroot :).
echo "${1}"
function emerge_custom_kernel() {
local install_root="$1"
local root=${FLAGS_build_root}/${FLAGS_board}
local tmp_pkgdir=${root}/custom-packages
# Clean up any leftover state in custom directories.
sudo rm -rf "${tmp_pkgdir}"
# Update chromeos-initramfs to contain the latest binaries from the build
# tree. This is basically just packaging up already-built binaries from
# $root. We are careful not to muck with the existing prebuilts so that
# prebuilts can be uploaded in parallel.
# TODO(davidjames): Implement ABI deps so that chromeos-initramfs will be
# rebuilt automatically when its dependencies change.
sudo -E PKGDIR="${tmp_pkgdir}" $EMERGE_BOARD_CMD -1 \
chromeos-base/chromeos-initramfs || die "Cannot emerge chromeos-initramfs"
# Verify all dependencies of the kernel are installed. This should be a
# no-op, but it's good to check in case a developer didn't run
# build_packages.
local kernel=$(portageq-${FLAGS_board} expand_virtual ${root} virtual/kernel)
sudo -E PKGDIR="${tmp_pkgdir}" $EMERGE_BOARD_CMD --onlydeps \
${kernel} || die "Cannot emerge kernel dependencies"
# Build the kernel. This uses the standard root so that we can pick up the
# initramfs from there. But we don't actually install the kernel to the
# standard root, because that'll muck up the kernel debug symbols there,
# which we want to upload in parallel.
sudo -E PKGDIR="${tmp_pkgdir}" $EMERGE_BOARD_CMD --buildpkgonly \
${kernel} || die "Cannot emerge kernel"
# Install the custom kernel to the provided install root.
sudo -E PKGDIR="${tmp_pkgdir}" $EMERGE_BOARD_CMD --usepkgonly \
--root=${install_root} ${kernel} || die "Cannot emerge kernel to root"