contrib/update_program_fw

#!/bin/bash
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Script to update base firmware in a program's config, and then
# regenerate the configs of projects that are part of the program.
# Also updates the firmware manifest, and uploads all of the changes
# for review.
#
# Usage:
#   ./update_program_fw --board=program --release=NNNNN [ --reviewer=reviewer ]
#                       [ --project=proj... ]
# E.g:
#   ./update_program_fw --board=puff --release=13291 --reviewer=amcrae@google.com
#
# TODO:
#   - Handle minor version
#
CONTRIB_DIR=$(dirname "$(readlink -f "$0")")
. "${CONTRIB_DIR}/common.sh" || exit 1

FLAGS_HELP="
Command to update the firmware version for a board.

Updates the firmware version configuration for a board's
master configuration (program.star) and for selected projects
that include the master configuration.

If no projects are specified, all projects for that board are selected.
An optional skip list can be specified to skip selected boards.

The configurations for the selected projects are regenerated,
and the firmware manifest are updated for the projects.

The necessary CLs for these changes are created and uploaded for review.
An optional reviewer can be specified to send all the CLs to.
"
# Flags
DEFINE_string board "${DEFAULT_BOARD}" "Which board (program) the firmware is for (e.g 'puff')" b
DEFINE_integer release 0  "The firmware release to update to (e.g 13310)" r
DEFINE_string project "${DEFAULT_PROJECT}" "Which projects this release is for (defaults to all), e.g 'duffy'" p
DEFINE_string bug "none" "The bug to reference in the CL e.g b:12345"
DEFINE_string skip "${DEFAULT_SKIP}" "Skip these projects (comma separated list)" s
DEFINE_boolean build "${FLAGS_TRUE}" "Attempt to build coreboot"
DEFINE_boolean program "${FLAGS_TRUE}" "Update the version in the base program.star"
DEFINE_string reviewer "${DEFAULT_REVIEWER}" "The reviewer to send the CLs to (optional)"
DEFINE_string test "none" "The 'TEST=' string added to the commit message"
DEFINE_boolean dryrun "${FLAGS_FALSE}" "Do not perform any actions, just validate and print arguments"

# Set before flag processing
COMMAND=$(basename "$0")
CMDARGS="$*"

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
set -e

# Script must be run inside the chroot.
assert_inside_chroot
#
# Variables
#
PATH="${PATH}:${GCLIENT_ROOT}/src/config/bin"
DIGITS="[1-9][0-9][0-9][0-9][0-9]"
BRANCH=""
PROGRAM_CL=""
MAJORV=" major_version = "
ANY_MV="${MAJORV}${DIGITS}"
NEW_MV="${MAJORV}${FLAGS_release}"
#
# Common functions
#
cleanup() {
  if [[ -d "${TEMPDIR}" ]]; then
    rm -rf "${TEMPDIR}"
  fi
}
#
# Abort the update, and clean up branches and CLs
#
abort() {
  CLS=$(gerrit -i --raw search "owner:me status:open hashtag:${BRANCH}")
  if [[ -n "${CLS}" ]]; then
    echo "Abandoning uploaded CLs"
    for cl in ${CLS}; do
      gerrit -i abandon "${cl}"
    done
  fi
  "cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-firmware-${FLAGS_board}"
  "cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-config-bsp-${FLAGS_board}-private"
  repo abandon "${BRANCH}"
  die "$*"
}
#
# Extract a CL number from the file containing the output of repo upload
#
getcl() {
  CL=$(grep -o "https://chrome-internal-review.googlesource.com/c/chromeos/$1/+/[0-9][0-9]*" "$2")
  if [[ -z "${CL}" ]]; then
    cat "$2"
    abort CL number not found in repo upload output
  fi
  echo "${CL}" | grep -o "[0-9][0-9]*"
}
#
# If not on this branch, start a branch
#
branch() {
  if ! (git branch --show-current | grep -q "${BRANCH}"); then
     repo start "${BRANCH}"
  else
     echo "${BRANCH} already exists, skipping repo start"
  fi
}
#
# Return true if repo has changes.
#
changed() {
  [[ -n $(git status -s) ]]
}
#
# Add a Cq-Depend line to a commit.
#
amend() {
  git log -1 --pretty=%B > "${TEMPDIR}/amend-msg"
  sed -i "/^Change-Id/ i ${1}" "${TEMPDIR}/amend-msg"
  git commit -q --amend -F "${TEMPDIR}/amend-msg"
}
#
# Confirm that $1 is a valid project
#
check_project() {
  PDIR="${GCLIENT_ROOT}/src/project/${FLAGS_board}/${1}"
  if [[ ! -d "${PDIR}" ]]; then
    die "${P} is not a valid project (${PDIR} missing)"
  fi
}
#
# Return true if $1 is in list $2
#
in_list() {
  for S in ${2}; do
     if [[ "$1" == "${S}" ]]; then
       return 0
     fi
  done
  return 1
}
#
# Return 0 if file has version in it.
#
has_version() {
  (grep -q "${ANY_MV}" "${1}")
}
#
# Update the major version in the file passed.
# return 0 if version updated.
# return 1 if version not in file, or unchanged.
#
update_version() {
  # Check for major_version in file.
  if ! (has_version "${1}") ;then
    return 1
  fi
  local nf="${TEMPDIR}/new-${1}"
  sed "/${ANY_MV}/s/${ANY_MV}/${NEW_MV}/" "${1}" > "${nf}"
  #
  # Verify that only 1-5 characters have changed.
  #
  DIFF=$(cmp -l "${1}" "${nf}" | wc -l)
  if [[ "${DIFF}" -gt 5 ]]; then
    diff "${1}" "{nf}}"
    abort "${1} update error"
  fi
  if [[ "${DIFF}" == 0 ]]; then
    return 1
  fi
  cp "${nf}" "${1}"
  return 0
}
#
# Validate arguments
#
if [[ -z "${FLAGS_board}" ]]; then
  die "-b or --board required."
fi
if [[ -z "${FLAGS_release}" ]]; then
  die "-r or --release required."
fi
#
# Program must exist as a directory
#
PROGDIR="${GCLIENT_ROOT}/src/program/${FLAGS_board}"
if [[ ! -d "${PROGDIR}" ]]; then
  die "${FLAGS_board} is not a valid program (${PROGDIR} missing)"
fi
# Release must be a 5 digit number
if [[ ! "${FLAGS_release}" =~ ^${DIGITS}$ ]]; then
  die "release must be a 5 digit number"
fi
# Use a common git branch name.
BRANCH="update_${FLAGS_board}_fw_${FLAGS_release}"
#
# Build the project list.
# If no projects are specified, use all in the programs directory.
#
if [[ -z "${FLAGS_project}" ]]; then
  BDIR="${GCLIENT_ROOT}/src/project/${FLAGS_board}"
  cd "${BDIR}"
  mapfile -t PROJLIST < <(ls)
else
  IFS=',' read -r -a PROJLIST <<< "${FLAGS_project}"
fi
#
# Filter out the projects that are to be skipped.
#
if [[ -n "${FLAGS_skip}" ]]; then
  PROJECTS=()
  IFS=',' read -r -a SKIP_ARRAY <<< "${FLAGS_skip}"
  # Validate skipped projects
  for S in "${SKIP_ARRAY[@]}"; do
    check_project "${S}"
  done
  SKIPPED="${SKIP_ARRAY[*]}"
  for P in "${PROJLIST[@]}"; do
    if ! (in_list "${P}" "${SKIPPED}"); then
      PROJECTS+=("${P}")
    fi
  done
else
  PROJECTS=("${PROJLIST[@]}")
fi
#
# Valid bug number (if any).
# Must be of the form b:nnnnn or chromium:nnnnn
#
if [[ "${FLAGS_bug}" != "none" ]]; then
  BG="b:[0-9]+|chromium:[0-9]+"
  BGRE="^(${BG})(,(${BG}))*$"
  if [[ ! "${FLAGS_bug}" =~ ${BGRE} ]]; then
    echo "Bug must be of the form b:nnn or chromium:nnn"
    die "A comma separated list is allowed"
  fi
fi
#
# Validate project list and file locations.
#
for P in "${PROJECTS[@]}"; do
  check_project "${P}"
done
OVERLAY="${GCLIENT_ROOT}/src/private-overlays/overlay-${FLAGS_board}-private/chromeos-base/chromeos-firmware-${FLAGS_board}"
# Validate project overlay and ebuild file
EB9999="chromeos-firmware-${FLAGS_board}-9999.ebuild"
if [[ ! -f "${OVERLAY}/${EB9999}" ]]; then
  die "${OVERLAY}/${EB9999}: overlay error"
fi
# Make sure dev/contrib is accessible
DEVCONTRIB="${GCLIENT_ROOT}/src/platform/dev/contrib"
if [[ ! -d "${DEVCONTRIB}" ]]; then
  die "${DEVCONTRIB}: invalid directory"
fi
#
# If requesting dry run, dump arguments and exit.
#
if [[ "${FLAGS_dryrun}" -eq  "${FLAGS_TRUE}" ]]; then
  echo "Dry run requested, invoked as:"
  echo "${COMMAND} ${CMDARGS}"
  echo "Program (board) to be updated: ${FLAGS_board}"
  echo -n "Projects to be updated are:   "
  for PROJ in "${PROJECTS[@]}"; do
    echo -n " ${PROJ}"
  done
  if [[ -n "${SKIPPED}" ]]; then
    echo -n " (skipped:"
    for S in "${SKIPPED[@]}"; do
      echo -n " ${S}"
    done
    echo -n ")"
  fi
  echo
  echo "Release number of upgrade:     ${FLAGS_release}"
  echo "BUG string used in commit:     ${FLAGS_bug}"
  echo "TEST string used in commit:    ${FLAGS_test}"
  echo "Reviewer assigned to CLs:      ${FLAGS_reviewer:-None}"
  echo -n "Coreboot build enabled:        "
  if [[ "${FLAGS_build}" -eq  "${FLAGS_FALSE}" ]]; then
    echo "no"
  else
    echo "yes"
  fi
  echo "repo branch to be used is:     ${BRANCH}"
  exit 0
fi
if [[ "${FLAGS_build}" -eq  "${FLAGS_FALSE}" ]]; then
  echo
  echo "******************************************"
  echo "* You have elected not to build coreboot *"
  echo "* This assumes coreboot is already built *"
  echo "******************************************"
  echo
fi
#
# Create a temp directory.
TEMPDIR=$(mktemp -d -t fw-XXXXXXXXXX)

trap "exit 1"           HUP INT PIPE QUIT TERM
trap 'cleanup' EXIT

#
# From now on, all errors should invoke 'abort'
# so that the branches and CLs are cleaned up on exit.
#
# If required, update the firmware version in the program's program.star file
#
if [[ "${FLAGS_program}" -eq "${FLAGS_TRUE}" ]]; then
  cd "${PROGDIR}"
  echo "Updating program.star for board ${FLAGS_board}"
  branch
  if (update_version "program.star") ;then
    #
    # If config has changed, create a CL.
    #
    git add .
    git commit -q -F - <<EOF
${FLAGS_board}: Update program firmware to ${FLAGS_release}

Autogenerated by:
${COMMAND} ${CMDARGS}

BUG=${FLAGS_bug}
TEST=${FLAGS_test}
EOF
    if ! repo upload -y "--ht=${BRANCH}" --cbr . > "${TEMPDIR}/upload.output" 2>&1 ;then
      cat  "${TEMPDIR}/upload.output"
      abort "repo upload failed"
    fi
    PROGRAM_CL=$(getcl "program/${FLAGS_board}" "${TEMPDIR}/upload.output")
  fi
fi
#
# Now walk through the projects and update the version (if present)
# and regenerate the configs.
# Create and upload a CL and capture the CL number and project directory
# if the project has changed.
#
PROJ_CLS=()
PROJ_DIRS=()
for PROJ in "${PROJECTS[@]}"; do
  echo "Updating configs for project ${PROJ}"
  PDIR="${GCLIENT_ROOT}/src/project/${FLAGS_board}/${PROJ}"
  cd "${PDIR}"
  branch
  update_version "config.star" || true
  ./config.star || abort "Generate config failed for ${PROJ}"
  check_config > "${TEMPDIR}/check_config-${PROJ}.output" || abort "check_config failed for ${PROJ}"
  #
  # Check if any files changed.
  #
  if changed; then
    echo "Creating CL for changes to project ${PROJ}"
    git add .
    git commit -q -F - <<EOF
${PROJ}: Update firmware to ${FLAGS_release}

Autogenerated by:
${COMMAND} ${CMDARGS}

BUG=${FLAGS_bug}
TEST=${FLAGS_test}
EOF
    if ! repo upload -y "--ht=${BRANCH}" --cbr . > "${TEMPDIR}/upload.${PROJ}.output" 2>&1 ;then
      cat  "${TEMPDIR}/upload.${PROJ}.output"
      abort "repo upload failed"
    fi
    P_CL=$(getcl "project/${FLAGS_board}/${PROJ}" "${TEMPDIR}/upload.${PROJ}.output")
    PROJ_CLS+=("${P_CL}")
    PROJ_DIRS+=("${PDIR}")
  fi
done
#
# Create a Cq-Depend line with all the project CLs
#
if [[ -n "${PROJ_CLS[*]}" ]];then
  SEP=" "
  PROG_CQD="Cq-Depend:"
  for CL in "${PROJ_CLS[@]}"; do
    PROG_CQD="${PROG_CQD}${SEP}chrome-internal:${CL}"
    SEP=", "
  done
  #
  # If a program CL exists, add the Cq-Depend line to it.
  #
  if [[ -n "${PROGRAM_CL}" ]]; then
    cd "${PROGDIR}"
    amend "${PROG_CQD}"
    if ! repo upload -y "--ht=${BRANCH}" --cbr . > "${TEMPDIR}/upload.amend.output" 2>&1 ;then
      cat  "${TEMPDIR}/upload.amend.output"
      abort "repo upload failed"
    fi
  fi
fi
#
# All the boxster configs have been uploaded.
# Now run the update script and update the firmware manifest.
#
# Build base coreboot files
#
if [[ "${FLAGS_build}" -eq  "${FLAGS_TRUE}" ]]; then
  echo "Running coreboot build. This may take a while..."
  #
  # Attempt to customise the coreboot build depending on the platform.
  #
  case "${FLAGS_board}" in
  "zork")
    PACKAGES=(coreboot-zork chromeos-bootimage)
    ;;
  "puff")
    PACKAGES=(chromeos-ec coreboot depthcharge vboot_reference libpayload chromeos-bootimage coreboot-private-files intel-cmlfsp coreboot-private-files-puff)
    ;;
  *)
    # Use general packages
    echo "Taking a guess at coreboot packages for ${FLAGS_board}"
    echo "If the coreboot build fails, this script may have to be customized for this board"
    PACKAGES=(coreboot depthcharge vboot_reference libpayload chromeos-bootimage)
    ;;
  esac
  if ! ("emerge-${FLAGS_board}" --quiet-build "${PACKAGES[@]}"); then
    abort "coreboot build failed!"
  fi
  echo "coreboot build successful"
else
  echo "Coreboot build not attempted"
fi
EB9999="chromeos-firmware-${FLAGS_board}-9999.ebuild"
#
# Remove any previous attempts to build the firmware.
#
"cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-firmware-${FLAGS_board}"
"cros_workon-${FLAGS_board}" stop "chromeos-base/chromeos-config-bsp-${FLAGS_board}-private"
cd "${OVERLAY}"
branch
cd "${DEVCONTRIB}"
if ! (./cros_update_firmware -q "--board=${FLAGS_board}"); then
  abort "cros_update_firmware failed for ${FLAGS_board}"
fi
cd "${OVERLAY}"
#
# If files have been updated, then create a CL for the changes.
#
OVERLAY_CL=""
if changed; then
  #
  # Bump the version in the ebuild file. Relies on the format
  # of the version so that the last number is at the end of the line.
  #
  CURVERS=$(grep "VERSION=REVBUMP" "${EB9999}" | grep -o "[0-9][0-9]*$")
  NEXTVERS=$((CURVERS + 1))
  sed -i "/VERSION=REVBUMP/s/${CURVERS}$/${NEXTVERS}/" "${EB9999}"
  git add .
  git commit -q -F - <<EOF
${FLAGS_board}: Update firmware to ${FLAGS_release}

Autogenerated by:
${COMMAND} ${CMDARGS}

BUG=${FLAGS_bug}
TEST=${FLAGS_test}

${PROG_CQD}
EOF
  #
  # Upload with no-verify since the file lines are too long.
  #
  if ! repo upload "--ht=${BRANCH}" -y --no-verify --cbr . > "${TEMPDIR}/overlay.output" 2>&1 ;then
    cat  "${TEMPDIR}/overlay.output"
    abort "repo upload failed"
  fi
  OVERLAY_CL=$(getcl "overlays/overlay-${FLAGS_board}-private" "${TEMPDIR}/overlay.output")
  #
  # Go back and amend all the project commit messages with a Cq-Depend on
  # the program and overlay CLs.
  #
  CQD="Cq-Depend: chrome-internal:${OVERLAY_CL}"
  if [[ -n "${PROGRAM_CL}" ]]; then
    CQD="${CQD}, chrome-internal:${PROGRAM_CL}"
  fi
  for DIR in "${PROJ_DIRS[@]}"; do
    cd "${DIR}"
    amend "${CQD}"
    if ! repo upload -y --cbr . > "${TEMPDIR}/cqd.output" 2>&1 ;then
      cat  "${TEMPDIR}/cqd.output"
      abort "repo upload failed"
    fi
  done
fi
#
# Send all of the CLs to the CQ for a dry run.
#
ALL_CLS=$(gerrit -i --raw search "owner:me status:open hashtag:${BRANCH}")
if [[ -z "${ALL_CLS}" ]]; then
  echo "No changes required for program ${FLAGS_board}"
  repo abandon "${BRANCH}"
  exit 0
fi
for cl in ${ALL_CLS}; do
  gerrit -i label-cq "${cl}" 1
  gerrit -i label-v "${cl}" 1
  gerrit -i label-as "${cl}" 1
done
#
# If reviewer is set, then add them to the CLs
#
if [[ -n "${FLAGS_reviewer}" ]]; then
  echo "Sending CLs ${ALL_CLS} to ${FLAGS_reviewer} for review"
  for cl in ${ALL_CLS}; do
    gerrit -i reviewers "${cl}" "${FLAGS_reviewer}"
  done
else
  echo "Send CLs for review by running:"
  echo "   for cl in ${ALL_CLS}; do gerrit -i reviewers \$cl <reviewer>; done"
fi
#
# Final instructions.
#
echo "Run:"
echo "  /build/${FLAGS_board}/usr/sbin/chromeos-firmwareupdate --manifest"
echo "to verify firmware update"
echo "When submitted, cleanup by running:"
echo "repo abandon ${BRANCH}"