| #!/bin/bash |
| |
| # |
| # This script fetches $FILES_TO_FETCH (kernel headers, source, toolchain, |
| # ...) of a specific COS release and installs them for compiling, |
| # debugging, etc. See usage() for details. |
| # |
| # This script is meant to run in COS toolbox or inside a cos-toolbox |
| # container. |
| # |
| |
| set -eu |
| set -o pipefail |
| |
| # Program name and version. Bump the version number if you change |
| # this script. |
| readonly PROG_NAME="$(basename "${0}")" |
| readonly PROG_VERSION="1.3" |
| |
| # ANSI escape sequences for pretty printing. |
| readonly RED_S="\033[00;31m" |
| readonly BLUE_S="\033[00;34m" |
| readonly PURPLE_S="\033[00;35m" |
| readonly ANSI_E="\033[0m" |
| |
| # Build ID number is passed as an arg or read from $COS_OS_RELEASE. |
| BUILD_ID="" |
| readonly COS_OS_RELEASE="/media/root/etc/os-release" |
| readonly COS_IMAGE_PROJECT="cos-cloud" |
| |
| |
| # For each file to fetch, we have the following properties: |
| # |
| # Property Type |
| # 1. Name in GCS bucket Fixed |
| # 2. Install dir name relative to $INSTALL_DIR Fixed |
| # 3. Full pathname of install dir Dynamic based on $INSTALL_DIR and $BUILD_ID |
| # 4. Installation commnds Dynamic based on $INSTALL_DIR and $BUILS_ID |
| # 5. Installation size in MB Fixed |
| # |
| # 1. Name in GCS bucket. |
| readonly KERNEL_HEADERS="kernel-headers.tgz" |
| readonly KERNEL_SRC="kernel-src.tar.gz" |
| readonly TRUSTED_KEY="trusted_key.pem" |
| readonly TOOLCHAIN="toolchain.tar.xz" |
| readonly TOOLCHAIN_ENV="toolchain_env" |
| # 2. Install dir name relative to $INSTALL_DIR. |
| readonly KERNEL_HEADERS_DIRNAME="cos-kernel-headers" |
| readonly KERNEL_SRC_DIRNAME="cos-kernel-src" |
| readonly TRUSTED_KEY_DIRNAME="${KERNEL_SRC_DIRNAME}" |
| readonly TOOLCHAIN_DIRNAME="cos-toolchain" |
| readonly TOOLCHAIN_ENV_DIRNAME="cos-toolchain-env" |
| # 3. Full pathname of installation directories (see initialize()). |
| KERNEL_HEADERS_DIR="" |
| KERNEL_SRC_DIR="" |
| TRUSTED_KEY_DIR="" |
| TOOLCHAIN_DIR="" |
| TOOLCHAIN_ENV_DIR="" |
| # 4. Installation commnds (see initialize()). |
| declare -A INSTALL_CMD |
| INSTALL_CMD[${KERNEL_HEADERS}]="" |
| INSTALL_CMD[${KERNEL_SRC}]="" |
| INSTALL_CMD[${TRUSTED_KEY}]="" |
| INSTALL_CMD[${TOOLCHAIN}]="" |
| INSTALL_CMD[${TOOLCHAIN_ENV}]="" |
| # 5. Installation size in MB. |
| declare -A INSTALL_SIZE |
| INSTALL_SIZE[${KERNEL_HEADERS}]="120" |
| INSTALL_SIZE[${KERNEL_SRC}]="1000" |
| INSTALL_SIZE[${TRUSTED_KEY}]="1" |
| INSTALL_SIZE[${TOOLCHAIN}]="2200" |
| INSTALL_SIZE[${TOOLCHAIN_ENV}]="1" |
| |
| readonly FILES_TO_FETCH=("${KERNEL_HEADERS}" "${KERNEL_SRC}" "${TRUSTED_KEY}" "${TOOLCHAIN}" "${TOOLCHAIN_ENV}") |
| readonly FETCHED_FILES_DIRNAME="fetched-files" |
| FETCHED_FILES_DIR="" |
| |
| # Temporary files created for the list subcommand. |
| readonly TMP_IMAGE_LIST="/tmp/image_list" |
| readonly TMP_BUILD_ID_LIST="/tmp/build_id_list" |
| readonly TMP_BUILD_ID_FILES="/tmp/build_id_files" |
| |
| # Compilation environment variables. |
| CC="" |
| CXX="" |
| |
| SUBCOMMAND="" |
| NO_DISK_SPACE=false |
| |
| # Set the defaults that can be changed by command line flags. |
| HELP="" # -h |
| INSTALL_DIR="${HOME}" # -i |
| ECHO=":" # -v |
| GLOBAL_OPTIONS="$(cat <<EOF |
| -h, --help print help message |
| -i, --instdir install directory (default \$HOME: $HOME) |
| -v, --verbose enable verbose mode |
| EOF |
| )" |
| |
| ALL="" # -a |
| LIST_OPTIONS="$(cat <<EOF |
| -h, --help print help message |
| -a, --all include deprecated builds |
| EOF |
| )" |
| |
| EXTRACT=true # -x |
| REMOVE=true # -r |
| FETCH_OPTIONS="$(cat <<EOF |
| -h, --help print help message |
| -r, --no-remove do not remove fetched files after installation |
| -x, --no-xtract do not extract files from their tarballs |
| EOF |
| )" |
| |
| KERNEL_CONFIG="" # -c |
| PRINT_CMD="" # -p |
| MAKE_VERBOSE="" # -V |
| BUILD_OPTIONS="$(cat <<EOF |
| -h, --help print help message |
| -c, --kconf specify path to kernel configuration file |
| -p, --print print commands to build the kernel, but do not execute |
| -V enable make's verbose mode |
| EOF |
| )" |
| |
| REMOVE_OPTIONS="$(cat <<EOF |
| -h, --help print help message |
| -a, --all remove all fetched and installed files |
| EOF |
| )" |
| |
| |
| usage() { |
| local -r exit_code="$1" |
| |
| cat <<EOF |
| ${PROG_NAME} v${PROG_VERSION} |
| |
| Usage: |
| ${PROG_NAME} [<global-options>] <subcommand> [<subcommand-options>] [<build-id>] |
| |
| Subcommmands: |
| list list available builds |
| fetch fetch kernel headers, source, and toolchain tarballs |
| build build kernel (implies fetch) |
| remove remove fetched and extracted files |
| help print help message |
| |
| Global options: |
| ${GLOBAL_OPTIONS} |
| |
| list options: |
| ${LIST_OPTIONS} |
| |
| fetch options: |
| ${FETCH_OPTIONS} |
| |
| build options: |
| ${BUILD_OPTIONS} |
| |
| remove options: |
| ${REMOVE_OPTIONS} |
| |
| Environment: |
| HOME default installation directory |
| EOF |
| |
| help_disk_space |
| exit "${exit_code}" |
| } |
| |
| |
| main() { |
| get_cos_tools_bucket |
| check_arch |
| |
| local options |
| |
| parse_args "${@}" |
| |
| # Global help message. |
| if [[ -z "${SUBCOMMAND}" || "${SUBCOMMAND}" == "help" ]]; then |
| usage 0 |
| fi |
| |
| # Subcommand-specific help message. |
| if [[ -n "${HELP}" ]]; then |
| echo "${SUBCOMMAND}" specific options: |
| options="${SUBCOMMAND^^}_OPTIONS" |
| echo "${!options}" |
| exit 0 |
| fi |
| |
| # No need to initialize if we're listing available releases. |
| if [[ "${SUBCOMMAND}" != "list" ]]; then |
| initialize |
| fi |
| |
| case "${SUBCOMMAND}" in |
| "list") subcmd_list;; |
| "fetch") subcmd_fetch; if "${EXTRACT}"; then extract_files; fi;; |
| "build") subcmd_build;; |
| "remove") subcmd_remove;; |
| *) fatal internal error processing "${SUBCOMMAND}" |
| esac |
| } |
| |
| |
| ####################################### |
| # Choose the public GCS bucket of COS to fetch files from |
| # "cos-tools", "cos-tools-eu" and "cos-tools-asia" |
| # based on where the VM is running. |
| # Arguments: |
| # None |
| # Globals: |
| # COS_GCS_BUCKET |
| ####################################### |
| get_cos_tools_bucket() { |
| # Get the zone the VM is running in. |
| # Example output: projects/438692578867/zones/us-west2-a |
| # If not running on GCE, use "gs://cos-tools" by default. |
| metadata_zone="$(curl -H Metadata-Flavor:Google http://metadata/computeMetadata/v1/instance/zone)" || { |
| readonly COS_GCS_BUCKET="gs://cos-tools" |
| return |
| } |
| zone="$( echo $metadata_zone | rev | cut -d '/' -f 1 | rev )" |
| prefix="$( echo $zone | cut -d '-' -f 1 )" |
| case $prefix in |
| "us" | "northamerica" | "southamerica") |
| readonly COS_GCS_BUCKET="gs://cos-tools" |
| ;; |
| "europe") |
| readonly COS_GCS_BUCKET="gs://cos-tools-eu" |
| ;; |
| "asia" | "australia") |
| readonly COS_GCS_BUCKET="gs://cos-tools-asia" |
| ;; |
| *) |
| readonly COS_GCS_BUCKET="gs://cos-tools" |
| ;; |
| esac |
| } |
| |
| check_arch() { |
| arch=$(uname -m) |
| if [ $arch == "arm64" ] || [ $arch == "aarch64" ]; then |
| echo "cos-kernel not supported on ARM" |
| exit 1 |
| fi |
| } |
| |
| parse_args() { |
| local args |
| |
| if ! args=$(getopt \ |
| --options "ac:hi:prvVx" \ |
| --longoptions "all config: help instdir: print no-remove verbose no-xtract" \ |
| -- "$@"); then |
| # getopt has printed an appropriate error message. |
| exit 1 |
| fi |
| eval set -- "${args}" |
| |
| while [[ "${#}" -gt 0 ]]; do |
| case "$1" in |
| -a|--all) |
| ALL="yes";; |
| -c|--kconf) |
| shift |
| KERNEL_CONFIG="$1";; |
| -h|--help) |
| HELP="yes";; |
| -i|--instdir) |
| shift |
| INSTALL_DIR="$1";; |
| -p|--print) |
| PRINT_CMD="echo";; |
| -r|--no-remove) |
| REMOVE=false;; |
| -v|--verbose) |
| ECHO="info";; |
| -V) |
| MAKE_VERBOSE="V=1";; |
| -x|--no-xtract) |
| EXTRACT=false;; |
| --) |
| ;; |
| *) |
| if [[ $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then |
| BUILD_ID="$1" |
| shift |
| continue |
| fi |
| if [[ -n "${SUBCOMMAND}" ]]; then |
| fatal specify only one subcommand |
| fi |
| case "$1" in |
| "list") SUBCOMMAND="$1";; |
| "fetch") SUBCOMMAND="$1";; |
| "build") SUBCOMMAND="$1";; |
| "remove") SUBCOMMAND="$1";; |
| "help") SUBCOMMAND="$1";; |
| "--") ;; |
| *) fatal "$1}": invalid build id |
| esac |
| esac |
| shift |
| done |
| |
| if [[ -z "${INSTALL_DIR}" ]]; then |
| fatal install directory not specified |
| fi |
| } |
| |
| |
| initialize() { |
| if [[ "${SUBCOMMAND}" == "remove" && -n "${ALL}" ]]; then |
| return |
| fi |
| |
| # If build ID is not provided as an argument, we assume we're |
| # running on COS and the user wants the current build ID. |
| if [[ -z "${BUILD_ID}" ]]; then |
| if [[ ! -f "${COS_OS_RELEASE}" ]]; then |
| fatal "${COS_OS_RELEASE}" does not exist and build ID not specified |
| fi |
| # shellcheck disable=SC1090 |
| source "${COS_OS_RELEASE}" |
| fi |
| |
| if [[ ! ${BUILD_ID} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then |
| error "${BUILD_ID}": invalid build id |
| return 1 |
| fi |
| |
| FETCHED_FILES_DIR="${INSTALL_DIR}/${FETCHED_FILES_DIRNAME}/${BUILD_ID}" |
| |
| KERNEL_HEADERS_DIR="${INSTALL_DIR}/${KERNEL_HEADERS_DIRNAME}/${BUILD_ID}" |
| KERNEL_SRC_DIR="${INSTALL_DIR}/${KERNEL_SRC_DIRNAME}/${BUILD_ID}" |
| TRUSTED_KEY_DIR="${INSTALL_DIR}/${TRUSTED_KEY_DIRNAME}/${BUILD_ID}/certs" |
| TOOLCHAIN_DIR="${INSTALL_DIR}/${TOOLCHAIN_DIRNAME}/${BUILD_ID}" |
| TOOLCHAIN_ENV_DIR="${INSTALL_DIR}/${TOOLCHAIN_ENV_DIRNAME}/${BUILD_ID}" |
| |
| INSTALL_CMD[${KERNEL_HEADERS}]="tar -C \"${KERNEL_HEADERS_DIR}\" -xf \"${FETCHED_FILES_DIR}/${KERNEL_HEADERS}\"" |
| INSTALL_CMD[${KERNEL_SRC}]="tar -C \"${KERNEL_SRC_DIR}\" -xf \"${FETCHED_FILES_DIR}/${KERNEL_SRC}\" && \ |
| (cd \"$(dirname "${KERNEL_SRC_DIR}")\" && rm -f kernel && ln -s \"$(basename "${KERNEL_SRC_DIR}")\" kernel)" |
| INSTALL_CMD[${TRUSTED_KEY}]="cp -a \"${FETCHED_FILES_DIR}/${TRUSTED_KEY}\" \"${TRUSTED_KEY_DIR}/${TRUSTED_KEY}\"" |
| INSTALL_CMD[${TOOLCHAIN}]="tar -C \"${TOOLCHAIN_DIR}\" -xf \"${FETCHED_FILES_DIR}/${TOOLCHAIN}\"" |
| INSTALL_CMD[${TOOLCHAIN_ENV}]="cp \"${FETCHED_FILES_DIR}/${TOOLCHAIN_ENV}\" \"${TOOLCHAIN_ENV_DIR}\"" |
| |
| info INSTALL_DIR="${INSTALL_DIR}" |
| info BUILD_ID="${BUILD_ID}" |
| echo |
| "${ECHO}" FETCHED_FILES_DIR="${FETCHED_FILES_DIR}" |
| "${ECHO}" KERNEL_HEADERS_DIR="${KERNEL_HEADERS_DIR}" |
| "${ECHO}" KERNEL_SRC_DIR="${KERNEL_SRC_DIR}" |
| "${ECHO}" TRUSTED_KEY_DIR="${TRUSTED_KEY_DIR}" |
| "${ECHO}" TOOLCHAIN_DIR="${TOOLCHAIN_DIR}" |
| "${ECHO}" TOOLCHAIN_ENV_DIR="${TOOLCHAIN_ENV_DIR}" |
| "${ECHO}" |
| } |
| |
| |
| subcmd_list() { |
| local header |
| local n |
| local build_id |
| local all_lines |
| local line |
| |
| # If we generated the list of images within the past hour, use it. |
| if [[ ! -s "${TMP_IMAGE_LIST}" || -z "$(find "${TMP_IMAGE_LIST}" -cmin -60)" ]]; then |
| info getting the list of images from "${COS_IMAGE_PROJECT}" |
| list_cos_images > "${TMP_IMAGE_LIST}" |
| fi |
| |
| # If we generated the list of build IDs within the past hour, use it. |
| if [[ ! -s "${TMP_BUILD_ID_FILES}" || -z "$(find "${TMP_BUILD_ID_FILES}" -cmin -60)" ]]; then |
| info getting the list of builds from "${COS_GCS_BUCKET}" |
| gsutil ls -r "${COS_GCS_BUCKET}" > "${TMP_BUILD_ID_FILES}" |
| fi |
| |
| # Get and sort the list of build IDs in $COS_GCS_BUCKET. |
| if [[ -n "${BUILD_ID}" ]]; then |
| echo "${BUILD_ID}" > "${TMP_BUILD_ID_LIST}" |
| # The $BUILD_ID may be deprecated or obsolete, but |
| # becaue it was specified on the command line, we |
| # still want to print it. |
| ALL="yes" |
| else |
| grep '^gs://.*:$' "${TMP_BUILD_ID_FILES}" | \ |
| grep -E '[0-9]+\.[0-9]+\.[0-9]+' | \ |
| sed -e "s;${COS_GCS_BUCKET}/;;" -e "s;/:;;" | \ |
| sort -V > "${TMP_BUILD_ID_LIST}" |
| fi |
| |
| # Build and print the header. |
| header="BUILD_ID MS FAMILY" |
| if [[ -n "${ALL}" ]]; then |
| header="${header} STAT" |
| fi |
| header="${header} HDR SRC KEY TLC" |
| echo "${header}" |
| |
| n=0 |
| while read -r build_id; do |
| # Although we no longer create releases with the exact |
| # same build ID in different image families, there are |
| # still older releases like cos-65-10323-104-0 and |
| # cos-stable-65-10323-104-0 that do have the same |
| # build ID. So, grep can return multiple lines. |
| all_lines=("$(grep "${build_id//./-}" "${TMP_IMAGE_LIST}")") |
| while read -r line; do |
| if [[ ("${line}" == *"DEPRECATED"* || "${line}" == *"OBSOLETE"*) && -z "${ALL}" ]]; then |
| continue |
| fi |
| mapfile -t milestone_family < <(get_milestone_family "${line}") |
| printf "%-14s %2s %6s" "${build_id}" "${milestone_family[0]}" "${milestone_family[1]}" |
| if [[ -n "${ALL}" ]]; then |
| if [[ "${line}" == *"DEPRECATED"* ]]; then |
| echo -n " dep" |
| elif [[ "${line}" == *"OBSOLETE"* ]]; then |
| echo -n " obs" |
| else |
| echo -n " " |
| fi |
| fi |
| echo -n " " |
| for f in "${FILES_TO_FETCH[@]}"; do |
| if grep -q "/${build_id}/${f}\$" "${TMP_BUILD_ID_FILES}"; then |
| echo -n "+++ " |
| else |
| echo -n "--- " |
| fi |
| done |
| echo |
| n=$((n + 1)) |
| if [[ "${n}" -gt 25 ]]; then |
| echo |
| echo "${header}" |
| n=0 |
| fi |
| done <<< "${all_lines[@]}" |
| done < "${TMP_BUILD_ID_LIST}" |
| } |
| |
| |
| subcmd_fetch() { |
| local f # file to fetch |
| local ff # complete URL of the file to fetch |
| local fetched # were any files fetched? |
| local md5 # md5sum checksum file |
| local bytes # size of file to fetch in bytes |
| |
| mkdir -p "${FETCHED_FILES_DIR}" |
| fetched=false |
| for f in "${FILES_TO_FETCH[@]}"; do |
| # To save disk space, fetched files are deleted by default (see -r) after being verified and installed. |
| if [[ -f "${FETCHED_FILES_DIR}/${f}.verified" && -f "${FETCHED_FILES_DIR}/${f}.installed" ]]; then |
| "${ECHO}" "${f}": already verified and installed |
| continue |
| fi |
| fetched=true |
| |
| if [[ -s "${FETCHED_FILES_DIR}/${f}" ]]; then |
| "${ECHO}" "${f}": already fetched |
| else |
| ff="${COS_GCS_BUCKET}/${BUILD_ID}/${f}" |
| # Does the file to fetch exist in GCS? |
| if ! gsutil -q stat "${ff}" 2> /dev/null; then |
| # A non-existent trusted key, toolchain, or toolchain_env is not fatal |
| # because older releases do not have them. |
| if [[ "${f}" != "${TRUSTED_KEY}" && "${f}" != "${TOOLCHAIN}" && "${f}" != "${TOOLCHAIN_ENV}" ]]; then |
| fatal "${ff}" does not exists |
| fi |
| warn "${ff}" does not exist |
| continue |
| fi |
| |
| # How big is the file to fetch? |
| bytes="$(gsutil stat "${ff}" 2> /dev/null | awk '/Content-Length:/ { print $2 }')" |
| if [[ -z "${bytes}" ]]; then |
| fatal cannot determine the size of "${ff}" |
| fi |
| # Do we have enough disk space for the file to fetch? |
| if ! have_disk_space $((bytes / (1024 * 1024))); then |
| fatal not enough disk space to fetch "${ff}" |
| fi |
| |
| # Fetch the file. |
| "${ECHO}" fetching "${ff}" |
| if ! fetch_file "${ff}" "${FETCHED_FILES_DIR}/${f}"; then |
| fatal could not fetch "${ff}" |
| rm -f "${FETCHED_FILES_DIR}/${f}" |
| fi |
| # Remember that haven't verified or installed the file that we fetched. |
| rm -f "${FETCHED_FILES_DIR}/${f}.verified" "${FETCHED_FILES_DIR}/${f}.installed" |
| fi |
| |
| # See if there's an md5sum file to verify the file we fetched. |
| md5="${f}.md5" |
| if [[ -s "${FETCHED_FILES_DIR}/${md5}" ]]; then |
| "${ECHO}" "${md5}": already fetched |
| else |
| ff="${COS_GCS_BUCKET}/${BUILD_ID}/${md5}" |
| "${ECHO}" fetching "${ff}" |
| # The md5 file is missing for old builds, so we tolerate failure. |
| if ! fetch_file "${ff}" "${FETCHED_FILES_DIR}/${md5}"; then |
| # This error is not fatal because older tarballs do not have |
| # md5sum checksum files. |
| warn could not fetch "${ff}" |
| rm -f "${FETCHED_FILES_DIR}/${md5}" |
| fi |
| fi |
| done |
| |
| if "${fetched}"; then |
| verify_fetched_files |
| fi |
| } |
| |
| |
| subcmd_build() { |
| subcmd_fetch |
| extract_files |
| |
| # We need at least 2.4GB to build the kernel. |
| if ! have_disk_space 2400; then |
| fatal not enough disk space to build the kernel |
| fi |
| |
| set_compilation_env |
| ${PRINT_CMD} cd "${KERNEL_SRC_DIR}" |
| ${PRINT_CMD} make ${MAKE_VERBOSE} -j $(($(nproc) * 2)) CC="${CC}" CXX="${CXX}" |
| } |
| |
| |
| subcmd_remove() { |
| local f |
| |
| if [[ -n "${ALL}" ]]; then |
| for f in "${FETCHED_FILES_DIRNAME}" "${KERNEL_HEADERS_DIRNAME}" "${KERNEL_SRC_DIRNAME}" "${TOOLCHAIN_DIRNAME}" "${TOOLCHAIN_ENV_DIRNAME}"; do |
| info removing "${INSTALL_DIR}/${f}" |
| rm -rf "${INSTALL_DIR:?INSTALL_DIR not set}/${f}" |
| done |
| return |
| fi |
| |
| for f in "${FETCHED_FILES_DIR}" "${KERNEL_HEADERS_DIR}" "${KERNEL_SRC_DIR}" "${TOOLCHAIN_DIR}" "${TOOLCHAIN_ENV_DIR}"; do |
| if [[ -n "${f}" ]]; then |
| info removing "${f}" |
| rm -rf "${f}" |
| fi |
| done |
| |
| for f in "${TMP_IMAGE_LIST}" "${TMP_BUILD_ID_LIST}" "${TMP_BUILD_ID_FILES}"; do |
| if [[ -f "${f}" ]]; then |
| info removing "${f}" |
| rm -f "${f}" |
| fi |
| done |
| } |
| |
| |
| list_cos_images() { |
| gcloud compute images list --project "${COS_IMAGE_PROJECT}" --no-standard-images --show-deprecated |
| } |
| |
| |
| get_milestone_family() { |
| local line="$1" |
| local milestone |
| local family |
| |
| #cos-65-10323-104-0 cos-cloud cos-65-lts DEPRECATED READY |
| #cos-dev-72-11190-0-0 cos-cloud cos-dev DEPRECATED READY |
| if [[ "${line}" =~ ^cos-[0-9][0-9]* ]]; then |
| # shellcheck disable=SC2001 |
| milestone="$(echo "${line}" | sed -e 's/cos-\(.*\)-\(.*\)-\(.*\)-\([0-9][0-9]*\)\( *cos-cloud.*\)/\1/')" |
| family="lts" |
| else |
| # shellcheck disable=SC2001 |
| milestone="$(echo "${line}" | sed -e 's/cos-\(.*\)-\(.*\)-\(.*\)-\(.*\)-\([0-9][0-9]*\)\( *cos-cloud.*\)/\2/')" |
| # shellcheck disable=SC2001 |
| family="$(echo "${line}" | sed -e 's/cos-\(.*\)-\(.*\)-\(.*\)-\(.*\)-\([0-9][0-9]*\)\( *cos-cloud.*\)/\1/')" |
| fi |
| echo -e "${milestone}\n${family}" |
| } |
| |
| |
| fetch_file() { |
| local src="$1" |
| local dst="$2" |
| |
| if ! gsutil cp "${src}" "${dst}" 2>/dev/null; then |
| return 1 |
| fi |
| |
| if ! test -s "${dst}"; then |
| return 1 |
| fi |
| } |
| |
| |
| verify_fetched_files() { |
| local file |
| local f |
| local checksum |
| |
| "${ECHO}" |
| for file in "${FILES_TO_FETCH[@]}"; do |
| f="${FETCHED_FILES_DIR}/${file}" |
| if [[ -f "${f}.verified" ]]; then |
| "${ECHO}" "${file}": already verified |
| continue |
| fi |
| if [[ ! -f "${f}.md5" ]]; then |
| warn "${file}.md5" does not exist, skipping verification |
| continue |
| fi |
| checksum="$(md5sum "${f}" | awk '{ print $1 }')" |
| if [[ "${checksum}" == "$(cat "${f}.md5")" ]]; then |
| "${ECHO}" verified "${file}" |
| touch "${f}.verified" |
| else |
| fatal "${file}" md5sum mismatch: expected "$(cat "${f}.md5")", got "${checksum}" |
| fi |
| done |
| } |
| |
| |
| extract_files() { |
| local f |
| local installed=false |
| |
| "${ECHO}" |
| |
| for f in "${KERNEL_HEADERS}" "${KERNEL_SRC}" "${TOOLCHAIN}" "${TOOLCHAIN_ENV}"; do |
| if install "${f}"; then |
| installed=true |
| fi |
| done |
| |
| if setup_kernel_config; then |
| installed=true |
| fi |
| |
| setup_trusted_key |
| |
| if "${installed}"; then |
| echo |
| fi |
| } |
| |
| |
| install() { |
| local f="$1" # file that was fetched |
| local ff |
| local dir |
| local cmd |
| |
| ff="${FETCHED_FILES_DIR}/${f}" |
| if [[ -f "${ff}.installed" ]]; then |
| "${ECHO}" "${ff}": already installed |
| return 1 |
| fi |
| |
| # Do we have enough disk space to install? |
| if ! have_disk_space "${INSTALL_SIZE["${f}"]}"; then |
| fatal not enough disk space to fetch "${ff}" |
| fi |
| |
| info installing "${ff}" |
| case "${f}" in |
| "${KERNEL_HEADERS}") dir="${KERNEL_HEADERS_DIR}";; |
| "${KERNEL_SRC}") dir="${KERNEL_SRC_DIR}";; |
| "${TOOLCHAIN}") dir="${TOOLCHAIN_DIR}";; |
| "${TOOLCHAIN_ENV}") dir="${TOOLCHAIN_ENV_DIR}";; |
| *) fatal "don't know where to install ${f}";; |
| esac |
| mkdir -p "${dir}" |
| cmd="${INSTALL_CMD[${f}]:-none}" |
| if [[ "${cmd}" == "none" ]]; then |
| fatal "don't know how to install ${ff}" |
| fi |
| eval "${cmd}" |
| touch "${ff}.installed" |
| if "${REMOVE}"; then |
| rm "${ff}" |
| fi |
| return 0 |
| } |
| |
| |
| setup_kernel_config() { |
| local kernel_config="${KERNEL_SRC_DIR}/.config" |
| local f |
| |
| # Was kernel configuration file specified on the command line? |
| if [[ -n "${KERNEL_CONFIG}" ]]; then |
| info creating kernel config file from "${KERNEL_CONFIG}" |
| if [[ "${KERNEL_CONFIG}" == "/proc/config.gz" ]]; then |
| zcat "${KERNEL_CONFIG}" > "${kernel_config}" |
| else |
| cp -a "${KERNEL_CONFIG}" "${KERNEL_SRC_DIR}" |
| fi |
| return 0 |
| fi |
| |
| if [[ -s "${kernel_config}" ]]; then |
| "${ECHO}" "${kernel_config}": already exists |
| return 1 |
| fi |
| |
| info copying kernel config from kernel headers |
| f="$(eval echo "${KERNEL_HEADERS_DIR}"/usr/src/linux-headers-*/.config)" |
| if [[ ! -f "${f}" ]]; then |
| fatal "${f}" does not exist |
| fi |
| cp -a "${f}" "${kernel_config}" |
| return 0 |
| } |
| |
| |
| setup_trusted_key() { |
| local kernel_config="${KERNEL_SRC_DIR}/.config" |
| local output |
| local cmd |
| |
| # Check CONFIG_SYSTEM_TRUSTED_KEYS to see if we have to copy the trusted key |
| # to the kernel source directory. |
| output="$(grep -w "CONFIG_SYSTEM_TRUSTED_KEYS" "${kernel_config}")" || true |
| if [[ -z "${output}" ]]; then |
| warn CONFIG_SYSTEM_TRUSTED_KEYS not in "${kernel_config}" |
| return |
| fi |
| if ! echo "${output}" | grep -qw "certs/${TRUSTED_KEY}"; then |
| return |
| fi |
| |
| # Did we fetch the trusted key? |
| if [[ -f "${FETCHED_FILES_DIR}/${TRUSTED_KEY}" ]]; then |
| if [[ -f "${TRUSTED_KEY_DIR}/${TRUSTED_KEY}" ]]; then |
| "${ECHO}" trusted key "${TRUSTED_KEY_DIR}/${TRUSTED_KEY}": already exists |
| return |
| fi |
| |
| info copying trusted key to "${TRUSTED_KEY_DIR}/${TRUSTED_KEY}" |
| cmd="${INSTALL_CMD[${TRUSTED_KEY}]:-none}" |
| if [[ "${cmd}" == "none" ]]; then |
| fatal "don't know how to install ${TRUSTED_KEY}" |
| fi |
| eval "${cmd}" |
| else |
| warn "modifying CONFIG_SYSTEM_TRUSTED_KEYS's value because we could not fetch the trusted key" |
| sed -i.bak -e 's/CONFIG_SYSTEM_TRUSTED_KEYS=.*/CONFIG_SYSTEM_TRUSTED_KEYS=""/' "${kernel_config}" |
| echo diff "${kernel_config}" "${kernel_config}.bak" |
| diff "${kernel_config}" "${kernel_config}.bak" || true |
| fi |
| } |
| |
| |
| set_compilation_env() { |
| local path |
| |
| path="$(realpath "${TOOLCHAIN_DIR}/bin")" |
| ${PRINT_CMD} export PATH="${path}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/google-cloud-sdk/bin" |
| |
| if [[ -s "${TOOLCHAIN_ENV_DIR}/${TOOLCHAIN_ENV}" ]]; then |
| # shellcheck disable=SC1090 |
| source "${TOOLCHAIN_ENV_DIR}/${TOOLCHAIN_ENV}" |
| else |
| # To support COS build not having toolchain_env file |
| CC="x86_64-cros-linux-gnu-gcc" |
| CXX="x86_64-cros-linux-gnu-g++" |
| fi |
| } |
| |
| |
| have_disk_space() { |
| local need="$1" |
| local avail |
| |
| avail="$(df -BM --output=avail "${INSTALL_DIR}" | sed -n -e 's/M//p')" |
| if [[ "${avail}" -lt "${need}" ]]; then |
| error need at least "${need}"MB, but have only "${avail}"MB in "${INSTALL_DIR}" |
| NO_DISK_SPACE=true |
| return 1 |
| fi |
| return 0 |
| } |
| |
| |
| help_disk_space() { |
| cat <<'END' |
| |
| NOTE: |
| Because by default toolbox uses /var/lib/toolbox as its working directory, |
| you can run out of space if your root partition is not big enough. |
| |
| You can add a second drive to your COS instance and use it as the working |
| directory of toolbox. For example, the following code creates a second |
| disk, attaches it to the instance, uses cloud-init to mount it on each |
| reboot, and assigns it to toolbox: |
| |
| # On your desktop: |
| $ gcloud compute disks create <your-disk> --size=200GB |
| $ gcloud compute instances attach-disk <your-instance> --disk <your-disk> |
| $ cat > user_data <<EOF |
| #cloud-config |
| |
| bootcmd: |
| - if [ -z "$(sudo blkid /dev/sdb)" ]; then mkfs.ext4 /dev/sdb; fi |
| - fsck.ext4 /dev/sdb |
| - mkdir -p /mnt/disks/sdb |
| - mount -t ext4 /dev/sdb /mnt/disks/sdb |
| EOF |
| $ gcloud compute instances add-metadata <your-instance> --metadata-from-file=user-data=user_data |
| |
| # On your COS instance: |
| $ echo TOOLBOX_DIRECTORY="/mnt/disks/sdb" >> $HOME/.toolboxrc |
| $ sudo reboot |
| END |
| } |
| |
| |
| info() { |
| if [[ -n "${*}" ]]; then |
| echo -e "${BLUE_S}INFO: ${*}${ANSI_E}" >&2 |
| else |
| echo |
| fi |
| } |
| |
| |
| warn() { |
| if [[ "${ECHO}" != ":" ]]; then |
| echo -e "${PURPLE_S}WARNING: ${*}${ANSI_E}" >&2 |
| fi |
| } |
| |
| |
| error() { |
| echo -e "${RED_S}ERROR: ${*}${ANSI_E}" >&2 |
| } |
| |
| |
| fatal() { |
| error "${@}" |
| if ${NO_DISK_SPACE}; then |
| help_disk_space |
| fi |
| exit 1 |
| } |
| |
| |
| main "${@}" |