afdo_tools/update_kernel_afdo

#!/bin/bash
# Copyright 2020 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Due to crbug.com/1081332, we need to update AFDO metadata
# manually. This script performs a few checks and generates a
# new kernel_afdo.json file, which can then be submitted.
#

USAGE="
Usage: $(basename "$0") [main|beta|stable|all] [--help]

Description:
  The script takes one optional argument which is the channel where we want
to update the kernel afdo and creates a commit (or commits with \"all\"
channels) in the corresponding branch.
  No arguments defaults to \"all\".
  Follow the prompt to upload the changes.
  NO CLEAN-UP NEEDED. The script ignores any local changes and keeps
the current branch unchanged.
"

set -eu
set -o pipefail

GS_BASE=gs://chromeos-prebuilt/afdo-job/vetted/kernel
KVERS="4.14 4.19 5.4 5.10"
failed_channels=""
# Add skipped chrome branches in ascending order here.
SKIPPED_BRANCHES="95"

# NOTE: We enable/disable kernel AFDO starting from a particular branch.
#   For example if we want to enable kernel AFDO in 5.15, first, we do it
#   in main. In this case we want to disable it in beta and stable branches.
#   The second scenario is when we want to disable kernel AFDO (when all devices
#   move to kernelnext and there are no new profiles from the field). In this
#   case we disable AFDO in main but still keep it live in beta and stable.
declare -A SKIPPED_KVERS_IN_BRANCHES
# In SKIPPED_KVERS_IN_BRANCHES
# - key is a branch number string;
# - value is the list of kernels separated by space.
# Example: SKIPPED_KVERS_IN_BRANCHES["105"]="4.4 4.14"

# b/223115767. In M-100 there are no new profiles in 5.10. And AFDO is not
# enabled on any 5.10 board in M-100 either.
SKIPPED_KVERS_IN_BRANCHES["100"]="5.10"

script_dir=$(dirname "$0")
tc_utils_dir="${script_dir}/.."
metadata_dir="${tc_utils_dir}/afdo_metadata"
outfile="$(realpath --relative-to="${tc_utils_dir}" \
  "${metadata_dir}"/kernel_afdo.json)"
# Convert toolchain_utils into the absolute path.
abs_tc_utils_dir="$(realpath "${tc_utils_dir}")"

# Check profiles uploaded within the last week.
expected_time=$(date +%s -d "week ago")

declare -A branch branch_number commit
remote_repo=$(git -C "${tc_utils_dir}" remote)
canary_ref="refs/heads/main"
# Read the last two release-Rxx from remote branches
# and assign them to stable_ref and beta_ref.
# sort -V is the version sort which puts R100 after R99.
# We need `echo` to convert newlines into spaces for read.
read -r stable_ref beta_ref <<< "$(git -C "${tc_utils_dir}" ls-remote -h \
  "${remote_repo}" release-R\* | cut -f2 | sort -V | tail -n 2 | paste -s)"
# Branch names which start from release-R.
branch["beta"]=${beta_ref##*/}
branch["stable"]=${stable_ref##*/}
branch["canary"]=${canary_ref##*/}

# Get current branch numbers (number which goes after R).
branch_number["stable"]=$(echo "${branch["stable"]}" | \
  sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p")
branch_number["beta"]=$(echo "${branch["beta"]}" | \
  sed -n -e "s/^release-R\([0-9][0-9]*\).*$/\1/p")
branch_number["canary"]="$((branch_number[beta] + 1))"
for skipped_branch in ${SKIPPED_BRANCHES} ; do
  if [[ ${branch_number["canary"]} == "${skipped_branch}" ]] ; then
    ((branch_number[canary]++))
  fi
done

# Without arguments the script updates all branches.
channels=${1:-"all"}
case "${channels}" in
  stable | canary | beta )
    ;;
  main )
    channels="canary"
    ;;
  all )
    channels="canary beta stable"
    ;;
  --help | help | -h )
    echo "${USAGE}"
    exit 0
    ;;
  * )
    echo "Channel \"${channels}\" is not supported.
Must be main (or canary), beta, stable or all." >&2
    echo "${USAGE}"
    exit 1
esac

# Fetch latest branches.
git -C "${tc_utils_dir}" fetch "${remote_repo}"

worktree_dir=$(mktemp -d)
echo "-> Working in ${worktree_dir}"
# Create a worktree and make changes there.
# This way we don't need to clean-up and sync toolchain_utils before the
# change. Neither we should care about clean-up after the submit.
git -C "${tc_utils_dir}" worktree add --detach "${worktree_dir}"
trap 'git -C "${abs_tc_utils_dir}" worktree remove -f "${worktree_dir}"' EXIT
cd "${worktree_dir}"

for channel in ${channels}
do
  errs=""
  successes=0
  curr_branch_number=${branch_number[${channel}]}
  curr_branch=${branch[${channel}]}
  echo
  echo "Checking \"${channel}\" channel..."
  echo "branch_number=${curr_branch_number} branch=${curr_branch}"
  json="{"
  sep=""
  for kver in ${KVERS}
  do
    # Skip kernels disabled in this branch.
    skipped=false
    for skipped_branch in "${!SKIPPED_KVERS_IN_BRANCHES[@]}"
    do
      if [[ ${curr_branch_number} == "${skipped_branch}" ]]
      then
        # Current branch is in the keys of SKIPPED_KVERS_IN_BRANCHES.
        # Now lets check if $kver is in the list.
        for skipped_kver in ${SKIPPED_KVERS_IN_BRANCHES[${skipped_branch}]}
        do
          if [[ ${kver} == "${skipped_kver}" ]]
          then
            skipped=true
            break
          fi
        done
      fi
    done
    if ${skipped}
    then
      echo "${kver} is skipped in branch ${curr_branch_number}. Skip it."
      continue
    fi
    # Sort the gs output by timestamp (default ordering is by name, so
    # R86-13310.3-1594633089.gcov.xz goes after R86-13310.18-1595237847.gcov.xz)
    latest=$(gsutil.py ls -l "${GS_BASE}/${kver}/" | sort -k2 | \
             grep "R${curr_branch_number}" | tail -1 || true)
    if [[ -z "${latest}" && "${channel}" != "stable" ]]
    then
      # if no profiles exist for the current branch, try the previous branch
      latest=$(gsutil.py ls -l "${GS_BASE}/${kver}/" | sort -k2 | \
        grep "R$((curr_branch_number - 1))" | tail -1)
    fi

    # Verify that the file has the expected date.
    file_time=$(echo "${latest}" | awk '{print $2}')
    file_time_unix=$(date +%s -d "${file_time}")
    if [ "${file_time_unix}" -lt "${expected_time}" ]
    then
      expected=$(env TZ=UTC date +%Y-%m-%dT%H:%M:%SZ -d @"${expected_time}")
      echo "Wrong date for ${kver}: ${file_time} is before ${expected}" >&2
      errs="${errs} ${kver}"
      continue
    fi

    # Generate JSON.
    json_kver=$(echo "${kver}" | tr . _)
    # b/147370213 (migrating profiles from gcov format) may result in the
    # pattern below no longer doing the right thing.
    name="$(basename "${latest%.gcov.*}")"
    # Skip kernels with no AFDO support in the current channel.
    if [[ "${name}" == "" ]]
    then
      continue
    fi
    json=$(cat <<EOT
${json}${sep}
    "chromeos-kernel-${json_kver}": {
        "name": "${name}"
    }
EOT
    )
    sep=","
    successes=$((successes + 1))
  done

  # If we did not succeed for any kvers, exit now.
  if [[ ${successes} -eq 0 ]]
  then
    echo "error: AFDO profiles out of date for all kernel versions" >&2
    failed_channels="${failed_channels} ${channel}"
    continue
  fi

  git reset --hard HEAD
  echo git checkout "${remote_repo}/${curr_branch}"
  git checkout "${remote_repo}/${curr_branch}"

  # Write new JSON file.
  # Don't use `echo` since `json` might have esc characters in it.
  printf "%s\n}\n" "${json}" > "${outfile}"

  # If no changes were made, say so.
  outdir=$(dirname "${outfile}")
  shortstat=$(cd "${outdir}" && git status --short "$(basename "${outfile}")")
  [ -z "${shortstat}" ] && echo "$(basename "${outfile}") is up to date." \
    && continue

  # If we had any errors, warn about them.
  if [[ -n "${errs}" ]]
  then
    echo "warning: failed to update ${errs} in ${channel}" >&2
    failed_channels="${failed_channels} ${channel}"
    continue
  fi

  git add afdo_metadata/kernel_afdo.json
  case "${channel}" in
    canary )
      commit_contents=$'afdo_metadata: Publish the new kernel profiles\n\n'
      for kver in ${KVERS} ; do
        commit_contents="${commit_contents}Update chromeos-kernel-${kver}"$'\n'
      done
      commit_contents="${commit_contents}

BUG=None
TEST=Verified in kernel-release-afdo-verify-orchestrator"
      ;;
    beta | stable )
      commit_contents="afdo_metadata: Publish the new kernel profiles\
 in ${curr_branch}

Have PM pre-approval because this shouldn't break the release branch.

BUG=None
TEST=Verified in kernel-release-afdo-verify-orchestrator"
      ;;
    * )
      echo "internal error: unhandled channel \"${channel}\"" >&2
      exit 2
  esac

  git commit -v -e -m "${commit_contents}"

  commit[${channel}]=$(git -C "${worktree_dir}" rev-parse HEAD)
done

echo
# Array size check doesn't play well with the unbound variable option.
set +u
if [[ ${#commit[@]} -gt 0 ]]
then
  set -u
  echo "The change is applied in ${!commit[*]}."
  echo "Run these commands to upload the change:"
  echo
  for channel in "${!commit[@]}"
  do
    echo -e "\tgit -C ${tc_utils_dir} push ${remote_repo} \
${commit[${channel}]}:refs/for/${branch[${channel}]}"
  done

  # Report failed channels.
  if [[ -n "${failed_channels}" ]]
  then
    echo
    echo "error: failed to update kernel afdo in ${failed_channels}" >&2
    exit 3
  fi
else
  # No commits. Check if it is due to failures.
  if [[ -z "${failed_channels}" ]]
  then
    echo "No changes are applied. It looks like AFDO versions are up to date."
  else
    echo "error: update in ${failed_channels} failed" >&2
    exit 3
  fi
fi