afdo_tools/update_kernel_afdo

#!/bin/bash
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Due to crbug.com/1081332, we need to update AFDO metadata
# manually. This script performs a few checks and generates a
# new kernel_afdo.json file, which can then be submitted.
#

USAGE="
Usage: $(basename $0) [main|beta|stable|all] [--help]

Description:
  The script takes one optional argument which is the channel where we want
to update the kernel afdo and creates a commit (or commits with \"all\"
channels) in the corresponding branch.
  No arguments defaults to \"all\".
  Follow the prompt to submit the changes.
  NO CLEAN-UP NEEDED. The script ignores any local changes and keeps
the current branch unchanged.
"

set -eu
set -o pipefail

GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel
KVERS="3.18 4.4 4.14 4.19 5.4"
failed_channels=""

script_dir=$(dirname "$0")
tc_utils_dir="${script_dir}/.."
metadata_dir="${tc_utils_dir}/afdo_metadata"
outfile="$(realpath --relative-to="${tc_utils_dir}" \
  "${metadata_dir}"/kernel_afdo.json)"
# Convert toolchain_utils into the absolute path.
abs_tc_utils_dir="$(realpath ${tc_utils_dir})"

# Check profiles uploaded within the last week.
expected_time=$(date +%s -d "week ago")

declare -A branch branch_number commit
remote_repo=$(git -C "${tc_utils_dir}" remote)
canary_ref="refs/heads/main"
# Read the last two release-Rxx from remote branches
# and assign them to stable_ref and beta_ref.
# sort -V is the version sort which puts R100 after R99.
last_branches=$(git -C "${tc_utils_dir}" ls-remote -h "${remote_repo}" \
  release-R\* | cut -f2 | sort -V | tail -n 2)
# We need `echo` to convert newlines into spaces for read.
read stable_ref beta_ref <<< $(echo ${last_branches})
# Branch names which start from release-R.
branch["beta"]=${beta_ref##*/}
branch["stable"]=${stable_ref##*/}
branch["canary"]=${canary_ref##*/}

# Get current branch numbers (number which goes after R).
branch_number["stable"]=$(echo "${branch["stable"]}" | \
  sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p")
branch_number["beta"]="$((branch_number[stable] + 1))"
branch_number["canary"]="$((branch_number[beta] + 1))"

# Without arguments the script updates all branches.
channels=${1:-"all"}
case "${channels}" in
  stable | canary | beta )
    ;;
  main )
    channels="canary"
    ;;
  all )
    channels="canary beta stable"
    ;;
  --help | help | -h )
    echo "$USAGE"
    exit 0
    ;;
  * )
    echo "Channel \"${channels}\" is not supported.
Must be main (or canary), beta, stable or all." >&2
    echo "$USAGE"
    exit 1
esac

# Fetch latest branches.
git -C "${tc_utils_dir}" fetch "${remote_repo}"

worktree_dir=$(mktemp -d)
echo "-> Working in ${worktree_dir}"
# Create a worktree and make changes there.
# This way we don't need to clean-up and sync toolchain_utils before the
# change. Neither we should care about clean-up after the submit.
git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}"
trap "git -C ${abs_tc_utils_dir} worktree remove ${worktree_dir}" EXIT
cd "${worktree_dir}"

for channel in ${channels}
do
  errs=""
  successes=0
  curr_branch_number=${branch_number[${channel}]}
  curr_branch=${branch[${channel}]}
  echo
  echo "Checking \"${channel}\" channel..."
  echo "branch_number=${curr_branch_number} branch=${curr_branch}"
  json="{"
  sep=""
  for kver in $KVERS
  do
    # Sort the gs output by timestamp (default ordering is by name, so
    # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz)
    latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \
             grep "R${curr_branch_number}" | tail -1 || true)
    if [[ -z "$latest" && "${channel}" != "stable" ]]
    then
      # if no profiles exist for the current branch, try the previous branch
      latest=$(gsutil.py ls -l "$GS_BASE/$kver/" | sort -k2 | \
        grep "R$((curr_branch_number - 1))" | tail -1)
    fi

    # Verify that the file has the expected date.
    file_time=$(echo "$latest" | awk '{print $2}')
    file_time_unix=$(date +%s -d "$file_time")
    if [ $file_time_unix -lt $expected_time ]
    then
      expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @$expected_time)
      echo "Wrong date for $kver: $file_time is before $expected" >&2
      errs="$errs $kver"
      continue
    fi

    # Generate JSON.
    json_kver=$(echo "$kver" | tr . _)
    # b/147370213 (migrating profiles from gcov format) may result in the
    # pattern below no longer doing the right thing.
    name=$(echo "$latest" | sed 's%.*/\(.*\)\.gcov.*%\1%')
    json=$(cat <<EOT
$json$sep
    "chromeos-kernel-$json_kver": {
        "name": "$name"
    }
EOT
    )
    sep=","
    successes=$((successes + 1))
  done

  # If we did not succeed for any kvers, exit now.
  if [[ $successes -eq 0 ]]
  then
    echo "error: AFDO profiles out of date for all kernel versions" >&2
    failed_channels="${failed_channels} ${channel}"
    continue
  fi

  git reset --hard HEAD
  echo git checkout "${remote_repo}/${curr_branch}"
  git checkout "${remote_repo}/${curr_branch}"

  # Write new JSON file.
  # Don't use `echo` since `json` might have esc characters in it.
  printf "%s\n}\n" "$json" > "$outfile"

  # If no changes were made, say so.
  outdir=$(dirname "$outfile")
  shortstat=$(cd "$outdir" && git status --short $(basename "$outfile"))
  [ -z "$shortstat" ] && echo $(basename "$outfile")" is up to date." \
    && continue

  # If we had any errors, warn about them.
  if [[ -n "$errs" ]]
  then
    echo "warning: failed to update $errs in ${channel}" >&2
    failed_channels="${failed_channels} ${channel}"
    continue
  fi

  git add afdo_metadata/kernel_afdo.json
  case "${channel}" in
    canary )
      commit_contents="afdo_metadata: Publish the new kernel profiles

Update chromeos-kernel-3_18
Update chromeos-kernel-4_4
Update chromeos-kernel-4_14
Update chromeos-kernel-4_19
Update chromeos-kernel-5_4

BUG=None
TEST=Verified in kernel-release-afdo-verify-orchestrator"
      ;;
    beta | stable )
      commit_contents="afdo_metadata: Publish the new kernel profiles\
 in ${curr_branch}

Have PM pre-approval because this shouldn't break the release branch.

BUG=None
TEST=Verified in kernel-release-afdo-verify-orchestrator"
      ;;
    * )
      echo "internal error: unhandled channel \"${channel}\"" >&2
      exit 2
  esac

  git commit -v -e -m "${commit_contents}"

  commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD)
done

echo
# Array size check doesn't play well with the unbound variable option.
set +u
if [[ ${#commit[@]} -gt 0 ]]
then
  set -u
  echo "The change is applied in ${!commit[@]}."
  echo "Run these commands to submit the change:"
  echo
  for channel in ${!commit[@]}
  do
    echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \
${commit[${channel}]}:refs/for/${branch[${channel}]}"
  done

  # Report failed channels.
  if [[ -n "${failed_channels}" ]]
  then
    echo
    echo "error: failed to update kernel afdo in ${failed_channels}" >&2
    exit 3
  fi
else
  # No commits. Check if it is due to failures.
  if [[ -z "${failed_channels}" ]]
  then
    echo "No changes are applied. It looks like AFDO versions are up to date."
  else
    echo "error: update in ${failed_channels} failed" >&2
    exit 3
  fi
fi