chromeos/scripts/setup_board

#!/bin/bash

# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script sets up a the sysroot for a particular target board.

# Load common CrOS utilities.  Inside the chroot this file is installed in
# /usr/lib/crosutils.  Outside the chroot we find it relative to the script's
# location.
find_common_sh() {
  local common_paths=(/usr/lib/crosutils $(dirname "$0"))
  local path

  SCRIPT_ROOT=
  for path in "${common_paths[@]}"; do
    local common="${path}/common.sh"
    if ([ -r "${common}" ] && . "${common}" && [ -d "${SCRIPTS_DIR}" ]); then
      SCRIPT_ROOT=${path}
      break
    fi
  done
}

find_common_sh
. "${SCRIPT_ROOT}/common.sh" || ! echo "Unable to load common.sh" || exit 1

# Script must run inside the chroot
restart_in_chroot_if_needed "$@"

get_default_board

# Flags
DEFINE_string board "$DEFAULT_BOARD" \
  "The name of the board to set up."
DEFINE_string build_root "/build" \
  "The root location for board sysroots."
DEFINE_string board_overlay "" \
  "Location of the board overlay."
DEFINE_string variant "" \
  "Board variant."
DEFINE_string toolchain "" \
  "Toolchain.  For example: i686-pc-linux-gnu, armv7a-softfloat-linux-gnueabi"
DEFINE_boolean usepkg $FLAGS_TRUE \
  "Use binary packages to bootstrap."
DEFINE_boolean force $FLAGS_FALSE \
  "Force re-creating board root."
DEFINE_string binutils_version "2.20.1-r7" \
  "Version of binutils to use."
DEFINE_string gcc_version "4.4.3-r7" \
  "Version of gcc to use."
DEFINE_string libc_version "2.10.1-r2" \
  "Version of libc to use."
DEFINE_string kernel_version "2.6.30-r1" \
  "Version of kernel headers to use."
DEFINE_boolean default $FLAGS_FALSE \
  "Set board to the default board in your chroot"
DEFINE_boolean fast ${DEFAULT_FAST} "Call many emerges in parallel"
DEFINE_string profile "" \
  "The portage configuration profile to use. Profile must be located in overlay-board/profiles"


# builds wrappers like equery-arm-generic.
# args:
#   $1:  command to wrap
#   rest:  extra arguments to pass to the command
generate_wrapper() {
  local command="${1}"
  shift
  local extra_args="$@"
  local temp=$(mktemp)


  local target="/usr/local/bin/${command}-${BOARD_VARIANT}"
  sudo_clobber "${target}" <<EOF
#!/bin/bash

export CHROMEOS_ROOT="$GCLIENT_ROOT"
export CHOST="$FLAGS_toolchain"
export PORTAGE_CONFIGROOT="$BOARD_ROOT"
export SYSROOT="$BOARD_ROOT"
if [ -z "\$PORTAGE_USERNAME" ]; then
  export PORTAGE_USERNAME=\$(basename \$HOME)
fi
export ROOT="$BOARD_ROOT"
exec sudo -E ${command} ${extra_args} "\$@"
EOF
  sudo chmod +rx ${target}
  sudo chown root:root ${target}
  eval ${command^^}_WRAPPER="${target}" # ${foo^^} returns toupper($foo)
  echo created wrapper ${target}
}

tc_pkg_needs_update() {
  CURRENT_VERSION=$(equery l cross-${FLAGS_toolchain}/* | grep $1 | \
    cut -d '/' -f 2 | cut -d '-' -f 2-)
  eval NEW_VERSION="\${FLAGS_${1}_version}"
  if [ "${CURRENT_VERSION}" == "${NEW_VERSION}" ]; then
    echo ${FLAGS_FALSE}
  else
    echo ${FLAGS_TRUE}
  fi
}

uninstall_toolchain() {
  echo "Uninstalling the toolchain."
  # Even if the uninstall fails, keep going. It's likely
  # that it didn't exist in the first place.
  yes | sudo crossdev -v -C $FLAGS_toolchain || true
}

# Build the toolchain with crossdev.
build_toolchain() {
  echo "Building the toolchain."
  # TODO(davidjames): Fix uploading of cross prebuilts so that we don't have
  #                   to use the old location to get the cross prebuilts.
  #BINHOST=$(portageq envvar PORTAGE_BINHOST)
  BINHOST="http://commondatastorage.googleapis.com/chromeos-prebuilt/host/amd64/13.04.11.101441/packages"
  CROSS_USEPKG=""
  if [ $FLAGS_usepkg -eq $FLAGS_TRUE ]; then
    # Grab the latest packages from the prebuilt server.
    #  -G: Use packages from the prebuilt server. Don't build from source.
    #  --binpkg-respect-use=n: Don't discard binary packages due to use flags.
    #  --usepkg=n: Don't use local binary packages.
    #  --without-headers: Don't build headers-only versions of packages for
    #                     bootstrapping. Because we use binary packages, this
    #                     isn't necessary.
    CROSS_USEPKG="--portage -G --portage --binpkg-respect-use=n"
    CROSS_USEPKG="${CROSS_USEPKG} --portage --usepkg=n --without-headers"
  fi

  CROSS_BINUTILS="--binutils $FLAGS_binutils_version"
  CROSS_GCC="--gcc $FLAGS_gcc_version"
  CROSS_KERNEL="--kernel $FLAGS_kernel_version"
  CROSS_LIBC="--libc $FLAGS_libc_version"
  sudo -E PORTAGE_BINHOST="${BINHOST}" crossdev -v -P --oneshot  \
      --target "$FLAGS_toolchain" \
      $CROSS_BINUTILS \
      $CROSS_GCC      \
      $CROSS_KERNEL   \
      $CROSS_LIBC     \
      $CROSS_USEPKG

  # Set up toolchain wrappers to force $SYSROOT and perform sanity checks.
  GCC_PROFILE="${FLAGS_toolchain}-${GCC_VERSION}"
  GCC_BIN_PATH=$(gcc-config --get-bin-path "$GCC_PROFILE")
  CCACHE_BIN=$(which ccache || true)
  for i in c++ cpp g++ gcc gfortran ; do
    GCC_BINARY="${GCC_BIN_PATH}/$FLAGS_toolchain-${i}"
    if [ -n "$(file -hb ${GCC_BINARY} 2>&1 | grep ^ELF)" ] ; then
      sudo mv "${GCC_BINARY}" "${GCC_BINARY}.real"
      sudo ln -s "${SCRIPTS_DIR}/sysroot_wrapper" "${GCC_BINARY}"
    fi
    if [ -f "${CCACHE_BIN}" ]; then
      sudo ln -sf "${CCACHE_BIN}" "/usr/lib/ccache/bin/${FLAGS_toolchain}-${i}"
    fi
  done
}

# Install all of the stuff that depends on the toolchain versions
# into the board root.
install_toolchain_in_board() {
  echo "Installing the toolchain into the board root."
  #
  # Untar glibc to get most of the headers required to build.
  #
  # TODO(raymes): Remove this check after some time which ensures
  # backward compatibility with the crossdev location change
  PKGDIR=$(portageq envvar PKGDIR)
  LIBC_TAR="glibc-${FLAGS_libc_version}.tbz2"
  NEW_LIBC_PATH="${PKGDIR}/cross-${FLAGS_toolchain}/${LIBC_TAR}"
  OLD_LIBC_PATH="${PKGDIR}/cross/${FLAGS_toolchain}/cross-${FLAGS_toolchain}/\
${LIBC_TAR}"
  if [ -e "${NEW_LIBC_PATH}" ] ; then
    LIBC_PATH="${NEW_LIBC_PATH}"
  else
    LIBC_PATH="${OLD_LIBC_PATH}"
  fi

  sudo tar jxpf "${LIBC_PATH}" -C "${BOARD_ROOT}" --strip-components=3

  # Symlink for libstdc++.la issues. It appears that when packages get merged
  # the .la files will be updated, and for libstdc++ it will use the wrong
  # location. This works around that issue.
  sudo ln -sf /usr/lib64/gcc "${BOARD_ROOT}/usr/lib/gcc"

  # We need to install libstdc++ manually from the cross toolchain.
  # TODO: Figure out a better way of doing this?
  sudo cp -a /usr/lib/gcc/"${FLAGS_toolchain}"/"${GCC_VERSION}"/libgcc_s.so* "${BOARD_ROOT}/lib"
  sudo cp -a /usr/lib/gcc/"${FLAGS_toolchain}"/"${GCC_VERSION}"/libstdc++.so* \
      "${BOARD_ROOT}/usr/lib"

  # Some header files are needed also for rpcbind (NFS support)
  # TODO: Figure out a better way of doing this too?
  sudo cp -a /usr/include/rpcsvc/mount.h "${BOARD_ROOT}/usr/include/rpcsvc"
  sudo cp -a /usr/include/rpcsvc/rquota.h "${BOARD_ROOT}/usr/include/rpcsvc"
  sudo cp -a /usr/include/rpcsvc/nfs_prot.h "${BOARD_ROOT}/usr/include/rpcsvc"
  sudo cp -a /usr/include/rpcsvc/yppasswd.h "${BOARD_ROOT}/usr/include/rpcsvc"

  # Tell portage that toolchain packages are already installed in the sysroot.
  PROVIDED="${BOARD_PROFILE}/package.provided"
  sudo sh -c \
      "echo \"sys-devel/binutils-${FLAGS_binutils_version}\" >> \"${PROVIDED}\""
  sudo sh -c "echo \"sys-devel/gcc-${FLAGS_gcc_version}\" >> \"${PROVIDED}\""
  sudo sh -c "echo \"sys-libs/glibc-${FLAGS_libc_version}\" >> \"${PROVIDED}\""
}

# Parse command line flags
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Only now can we die on error.  shflags functions leak non-zero error codes,
# so will die prematurely if 'set -e' is specified before now.
set -e

if [ -z "$FLAGS_board" ] ; then
  error "--board required."
  exit 1
fi

#
# Before we can run any tools, we need to emerge hard-host-depends.
#
EMERGE_CMD="emerge"
if [ "$FLAGS_fast" -eq "${FLAGS_TRUE}" ]; then
  EMERGE_CMD="${GCLIENT_ROOT}/chromite/bin/parallel_emerge"
fi
sudo -E ${EMERGE_CMD} -uDNvg chromeos-base/hard-host-depends world

get_board_and_variant $FLAGS_board $FLAGS_variant

#
# Check if there are any board overlays.  There should be at least a top
# level board specific overlay.
#
PRIMARY_BOARD_OVERLAY=$(cros_overlay_list --board "$BOARD" --primary_only)

#
# Fetch the toolchain from the board overlay.
#
BOARD_TOOL_CHAIN="${PRIMARY_BOARD_OVERLAY}/toolchain.conf"
FLAGS_toolchain=${FLAGS_toolchain:-$(cat ${BOARD_TOOL_CHAIN})}

# Figure out ARCH from the given toolchain
# TODO: Move to common.sh as a function after scripts are switched over.
if [ -z "${FLAGS_toolchain}" ]; then
  error "No toolchain specified in board overlay or on command line."
  exit 1
fi

TC_ARCH=$(echo "$FLAGS_toolchain" | awk -F'-' '{ print $1 }')
case "$TC_ARCH" in
  arm*)
    ARCH="arm"
    ;;
  *86)
    ARCH="x86"
    ;;
  *)
    error "Unable to determine ARCH from toolchain: $FLAGS_toolcahin"
    exit 1
esac

# Locations we will need
BOARD_ROOT="${FLAGS_build_root}/${BOARD_VARIANT}"
CHROMIUMOS_OVERLAY="/usr/local/portage/chromiumos"
CHROMIUMOS_CONFIG="${CHROMIUMOS_OVERLAY}/chromeos/config"
CHROMIUMOS_PROFILES="${CHROMIUMOS_OVERLAY}/profiles"
BOARD_ETC="${BOARD_ROOT}/etc"
BOARD_PROFILE="${BOARD_ETC}/portage/profile"

#
# Construct board overlay list.
#
BOARD_OVERLAY_LIST=$(cros_overlay_list \
  --board "$BOARD" \
  --board_overlay "$FLAGS_board_overlay" \
  --variant "$VARIANT")

#
# Populate the BOARD_MAKE_CONF_LIST with any make.conf files found in the
# overlays.
#
BOARD_MAKE_CONF_LIST=""

for overlay in ${BOARD_OVERLAY_LIST} ; do
  if [ -f "${overlay}/make.conf" ]; then
    BOARD_MAKE_CONF_LIST="${BOARD_MAKE_CONF_LIST} ${overlay}/make.conf"
  fi
done

BINHOST_CONF="${CHROMIUMOS_OVERLAY}/chromeos/binhost/target/${ARCH}.conf"
if [ -f "${BINHOST_CONF}" ]; then
  BOARD_MAKE_CONF_LIST="${BOARD_MAKE_CONF_LIST} ${BINHOST_CONF}"
fi

# Version of GCC we will be installing, without the ebuild revision number.
GCC_VERSION=`echo ${FLAGS_gcc_version} | sed -e 's/\-r[0-9]*$//g'`

# Create the crossdev overlay in case it hasn't been created.
# TODO(raymes): Remove this when everyone has run make_chroot.
CROSSDEV_OVERLAY="/usr/local/portage/crossdev"
sudo mkdir -p ${CROSSDEV_OVERLAY}
sudo chmod 755 ${CROSSDEV_OVERLAY}

if [ "$(tc_pkg_needs_update "binutils")" = "$FLAGS_TRUE" ] ||
  [ "$(tc_pkg_needs_update "gcc")" = "$FLAGS_TRUE" ] ||
  [ "$(tc_pkg_needs_update "libc")" = "$FLAGS_TRUE" ]; then
  warn "Toolchain needs to be updated! Updating toolchain..."
  uninstall_toolchain
  build_toolchain
  # If the board root already exists, re-install the toolchain there.
  if [ -d "$BOARD_ROOT" ] && [ "$FLAGS_force" = "$FLAGS_FALSE" ]; then
    install_toolchain_in_board
  fi
fi

if [ "${ARCH}" == "x86" ] ; then
  echo "Switching on gold as the default linker."
  BINUTILS_VERSION=$(echo ${FLAGS_binutils_version} | sed 's/-r[0-9]\+//g')
  sudo binutils-config "${FLAGS_toolchain}-${BINUTILS_VERSION}-gold"
fi

if [ -d "$BOARD_ROOT" ] ; then
  if [[ $FLAGS_force -eq $FLAGS_TRUE ]]; then
    echo "--force set.  Re-creating $BOARD_ROOT..."
    # Removal takes long. Make it asynchronous.
    TEMP_DIR=`mktemp -d`
    sudo mv $BOARD_ROOT $TEMP_DIR
    sudo rm -rf $TEMP_DIR &
  else
    warn "Board output directory '$BOARD_ROOT' already exists."
    warn "Not setting up board root. "
    warn "Use --force to clobber the board root and start again."
    exit 1
  fi
fi

sudo mkdir -p "$BOARD_ROOT"

# TODO(cmasone): Do this more cleanly, if we figure out what "cleanly" means.
# Set up wrapper for pkg-config.  Point a board-specific wrapper at the
# generic wrapper script created by crossdev-wrapper
sudo ln -sf "/usr/bin/cross-pkg-config" "/usr/bin/${FLAGS_toolchain}-pkg-config"

# Setup make.conf and make.profile as symlinks to ones in revision control
sudo mkdir -p "${BOARD_ETC}" "${BOARD_PROFILE}"

# Start with the crossdev etc
# TODO(tedbo): The sed lines are hacks to fix an issue where the crossdev
# wrapper wants to assume that SYSROOT is /usr/${CHOST}. We should fix this
# in the crossdev code and try to upstream.
sudo cp -a /usr/share/crossdev/etc/* "${BOARD_ETC}"
sudo rm -f "${BOARD_ETC}/make.profile"
sudo sed -i 's/${CHOST}/${SYSROOT}/g' "${BOARD_ETC}/portage/bashrc"
sudo sed -i 's/CROSS_ROOT=""/CROSS_ROOT="$1"/g' /usr/bin/cross-fix-root

# Setup the make.confs. We use the following:
#    make.conf             <- Overall target make.conf [arm, x86, etc. version]
#    make.conf.board_setup <- Declares CHOST, ROOT, etc.
#    make.conf.common      <- Common settings across all targets
#    make.conf.board       <- Optional board-supplied make.conf
sudo ln -sf "${CHROMIUMOS_CONFIG}/make.conf.${ARCH}-target" \
  "${BOARD_ETC}/make.conf"
sudo ln -sf "${CHROMIUMOS_CONFIG}/make.conf.common-target" \
  "${BOARD_ETC}/make.conf.common"
cat <<EOF | sudo dd of="${BOARD_ETC}/make.conf.board_setup" > /dev/null 2>&1
# Created by setup_board
CHOST="${FLAGS_toolchain}"
ROOT="${BOARD_ROOT}/"
BOARD_OVERLAY="${BOARD_OVERLAY_LIST}"
MAKEOPTS="-j${NUM_JOBS}"
LIBC_VERSION="$FLAGS_libc_version"
EOF

# We install the toolchain related bits after the BOARD_ROOT, BOARD_PROFILE
# and BOARD_ETC directories have been created.
install_toolchain_in_board

sudo touch "${BOARD_ETC}/make.conf.board"

for make_conf in ${BOARD_MAKE_CONF_LIST} ; do
  sudo bash -c "echo source ${make_conf} >> ${BOARD_ETC}/make.conf.board"
done

# Setup make.globals and the profile.
sudo touch /etc/make.conf.user
sudo ln -sf /etc/make.globals "${BOARD_ROOT}/etc/make.globals"
sudo ln -sf /etc/make.conf.user "${BOARD_ROOT}/etc/make.conf.user"

# Select the profile to build based on the board and profile passed to
# setup_board.  The developer can later change profiles by running
# cros_choose_profile manually.
cros_choose_profile \
  --board "${FLAGS_board}" \
  --build_root "${FLAGS_build_root}" \
  --board_overlay "${FLAGS_board_overlay}" \
  --variant "${FLAGS_variant}" \
  --profile "${FLAGS_profile}"

for wrapper in 'emerge --root-deps' ebuild eclean equery portageq; do
  generate_wrapper $wrapper
done

CROS_WORKON_WRAPPER="/usr/local/bin/cros_workon-${BOARD_VARIANT}"
cat <<EOF | sudo dd of="$CROS_WORKON_WRAPPER" > /dev/null 2>&1
#!/bin/bash
exec cros_workon --board ${BOARD_VARIANT} "\$@"
EOF
sudo chmod +x "$CROS_WORKON_WRAPPER"

#
# Emerge the kernel headers into the board build root.
#
EMERGE_FLAGS=""
if [[ $FLAGS_usepkg -eq $FLAGS_TRUE ]]; then
  EMERGE_FLAGS="${EMERGE_FLAGS} --getbinpkg --usepkg"
fi
sudo -E "${EMERGE_WRAPPER}" ${EMERGE_FLAGS} chromeos-base/kernel-headers

# Temporary hack needed so that we can move headers install back into
# the kernel-headers ebuild.
# TODO(msb): Remove this hack.
sudo chown -Rf ${USER} ~/trunk/.config || true
mkdir -p ~/trunk/.config/headers-fix
touch ~/trunk/.config/headers-fix/"${FLAGS_board}"

if [ $FLAGS_default -eq $FLAGS_TRUE ] ; then
  echo $BOARD_VARIANT > "$GCLIENT_ROOT/src/scripts/.default_board"
fi

echo "Done!"
echo "The SYSROOT is: ${BOARD_ROOT}"