Add cos_xfstests_runner container in cos/tools.
BUG=b/218901513
Change-Id: I1878d5ee47228cfaa6e7834390df3b26f6504980
Reviewed-on: https://cos-review.googlesource.com/c/cos/tools/+/51569
Tested-by: Meena Shanmugam <meenashanmugam@google.com>
Reviewed-by: Arnav Kansal <rnv@google.com>
Cloud-Build: GCB Service account <228075978874@cloudbuild.gserviceaccount.com>
diff --git a/src/cmd/cos_xfstests_runner/Dockerfile b/src/cmd/cos_xfstests_runner/Dockerfile
new file mode 100644
index 0000000..ff0869c
--- /dev/null
+++ b/src/cmd/cos_xfstests_runner/Dockerfile
@@ -0,0 +1,10 @@
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get install -y make git
+RUN apt-get install -y apt-transport-https ca-certificates gnupg curl sudo python3
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+RUN apt-get update && apt-get install -y google-cloud-cli
+COPY ./run_config.sh /run_config.sh
+
+ENTRYPOINT ["/run_config.sh"]
diff --git a/src/cmd/cos_xfstests_runner/README.md b/src/cmd/cos_xfstests_runner/README.md
new file mode 100644
index 0000000..99ca1d7
--- /dev/null
+++ b/src/cmd/cos_xfstests_runner/README.md
@@ -0,0 +1,56 @@
+# COS Xfstests Runner container used in COS xfstests
+
+## Overview
+
+This is an docker image used by COS (Container-Optimized OS)
+xfstests. It includes scripts and necessary dependencies for running
+gce-xfstests under a given xfstests config against a given COS version.
+
+The image is released at `gcr.io/cos-xfstests/cos-xfstests-runner` with a unique
+tag for each release. This container image is not publicly available.
+
+## Building COS Xfstests Runner Image
+
+### Locally (for testing the image)
+
+For testing, you can simply build and test this docker container locally on your
+workstation:
+
+```shell
+ $ docker build -t cos-xfstests-runner:dev .
+```
+
+### Production (push into GCR)
+Since `gcr.io/cos-xfstests/cos-xfstests-runner` is not publicly available, only
+authenticated users can push into the GCR registry.
+
+```shell
+ $ VERSION=<version> # e.g., 20171008
+ $ docker build -t gcr.io/cos-xfstests/cos-xfstests-runner:$VERSION .
+ $ docker build -t gcr.io/cos-xfstests/cos-xfstests-runner:latest .
+ $ gcloud docker -- push gcr.io/cos-xfstests/cos-xfstests-runner:$VERSION
+ $ gcloud docker -- push gcr.io/cos-xfstests/cos-xfstests-runner:latest
+```
+
+## Using COS Xfstests Runner Image
+
+### Locally (for testing the image)
+
+COS xfstests runner container will use the xfstests-bld tarball and kernel image
+generated by COS kernel CI. To run this container locally, you can build the kernel
+image using cos_kerenl_devenv container.
+
+To run the COS xfstests runner locally:
+
+```shell
+ $ docker run --rm -v ~/.config/gcloud:/root/.config/gcloud cos-xfstests-runner:dev -x overlay -n xfs-vm -r gs://xueweiz-play/xfstests/R76-12188.0.0 -p cos-xfstests -z us-west1-c -b generic/269,generic/347,generic/405,generic/500
+```
+
+The container will download xfstests-bld tarball and kernel image from given GCS
+bucket, then run `gce-xfstests` script on the given config. This will create one
+GCE instance and two responding disks in the given project and zone. If they run
+into soft lockup, the developer must manually clean them up.
+
+The container image is set to use
+[test appliance image](https://github.com/tytso/xfstests-bld/blob/master/Documentation/gce-xfstests.md#creating-a-new-gce-test-appliance-image)
+"xfstests-201901211635" by default. Use the `--rootfs_image` to overwrite it.
diff --git a/src/cmd/cos_xfstests_runner/cloudbuild.yaml b/src/cmd/cos_xfstests_runner/cloudbuild.yaml
new file mode 100644
index 0000000..dcb05c9
--- /dev/null
+++ b/src/cmd/cos_xfstests_runner/cloudbuild.yaml
@@ -0,0 +1,24 @@
+options:
+ env:
+ - 'DOCKER_CLI_EXPERIMENTAL=enabled'
+steps:
+# This step is needed to add a new entry to /proc/sys/fs/binfmt_misc. Docker
+# uses QEMU user emulation to run arm64 programs on x86 hosts. A QEMU
+# interpreter needs to be added to /proc/sys/fs/binfmt_misc to run arm64
+# programs.
+- name: 'gcr.io/cloud-builders/docker'
+ args: ['run', '--privileged', 'linuxkit/binfmt:a17941b47f5cb262638cfb49ffc59ac5ac2bf334-amd64']
+# The default builder (which appears to be the Docker daemon that implements
+# the old, familiar `docker build` behavior) doesn't support the --platform
+# flag, so we need to create a new builder.
+- name: 'gcr.io/cloud-builders/docker'
+ args: ['buildx', 'create', '--name', 'builder']
+- name: 'gcr.io/cloud-builders/docker'
+ args: ['buildx', 'use', 'builder']
+# Images produced in this way do not appear in the Docker image registry shown
+# by `docker images`, at least by default. We use the --push flag to push the
+# image after building it, because a subsequent `docker push` won't find the
+# image locally.
+- name: 'gcr.io/cloud-builders/docker'
+ args: ['buildx', 'build', '--platform', 'linux/amd64,linux/arm64', '-f', 'src/cmd/cos_xfstests_runner/Dockerfile', '-t', 'gcr.io/${_OUTPUT_PROJECT}/cos_xfstests_runner:latest', '-t', 'gcr.io/${_OUTPUT_PROJECT}/cos_xfstests_runner:${TAG_NAME}', '--push', 'src/cmd/cos_xfstests_runner']
+timeout: 1800s
diff --git a/src/cmd/cos_xfstests_runner/run_config.sh b/src/cmd/cos_xfstests_runner/run_config.sh
new file mode 100755
index 0000000..7db972a
--- /dev/null
+++ b/src/cmd/cos_xfstests_runner/run_config.sh
@@ -0,0 +1,172 @@
+#!/bin/bash
+# Run xfstests for a given file system configuration.
+
+set -eu
+set -o pipefail
+
+readonly PROG_NAME=$(basename "$0")
+readonly XFSTESTS_BLD_DIR="/home/fstests"
+readonly XFSTESTS_CFG_DIR="test-appliance/files/root/fs/"
+
+CONFIG=""
+INSTANCE_NAME=""
+RESULTS_BUCKET=""
+PROJECT=""
+ZONE=""
+GROUP="auto"
+XFSTESTS_EXCLUDED=""
+ARCH="x86_64"
+
+#
+# usage <exit_code>
+#
+# Print usage and exit.
+#
+usage() {
+ local exit_code="${1}"
+
+ cat <<EOF
+Usage:
+ ${PROG_NAME} [-x <xfstests_config>] [-n <instance_name>]
+ -x, --xfstests_config the configuration to run xfstests on
+ -n, --instance_name the name of the instance running xfstests
+ -r, --result_bucket the GCS bucket to store test result
+ -p, --project the GCP project to run the test VM
+ -z, --zone the GCE zone to run the test VM
+ -g, --group the xfstest group to run
+ -b, --blocked_tests the list of tests to not run
+ -i, --rootfs_image the test appliance image to use
+ -a, --arch the architecture of the image
+
+Note that this script expect kernel to be located at [result_bucket]/bzImage.
+
+Examples:
+ $ ${PROG_NAME} -x overlay -n xfs-vm -r gs://xfstests/R93-11647.62.0 -p cos-xfstests -z us-west1-c -b generic/269,generic/500
+
+EOF
+ exit "${exit_code}"
+}
+
+#
+# parse_args <args...>
+#
+# Parse command line arguments.
+#
+parse_args() {
+ local args
+
+ args=$(getopt \
+ --options "x:n:r:p:z:g:b:a:" \
+ --longoptions "xfstests_config: instance_name: result_bucket: project: zone: group: blocked_tests: arch:" \
+ -- "$@")
+ [[ $? -eq 0 ]] || usage 1
+ eval set -- "${args}"
+
+ while :; do
+ arg="${1}"
+ shift
+ case "${arg}" in
+ -x|--xfstests_config) CONFIG="${1}"; shift ;;
+ -n|--instance_name) INSTANCE_NAME="${1}"; shift ;;
+ -r|--result_bucket) RESULTS_BUCKET="${1}"; shift ;;
+ -p|--project) PROJECT="${1}"; shift ;;
+ -z|--zone) ZONE="${1}"; shift ;;
+ -g|--group) GROUP="${1}"; shift ;;
+ -b|--blocked_tests) XFSTESTS_EXCLUDED="${1}"; shift ;;
+ -i|--rootfs_image) TEST_APPLIANCE_IMAGE="${1}"; shift ;;
+ -a|--arch) ARCH="${1}"; shift ;;
+ --) break ;;
+ *) echo "internal error parsing arguments!"; usage 1 ;;
+ esac
+ done
+
+ if [[ -z "$CONFIG" || -z "$RESULTS_BUCKET" || -z "$PROJECT" || -z "$ZONE" ]] ; then
+ usage 1
+ fi
+
+ if [[ -z "$INSTANCE_NAME" ]] ; then
+ echo "Instance name missing. Randomly generating one."
+ CONFIG_ALNUM=${CONFIG//[^[:alnum:]]/}
+ INSTANCE_NAME="cos-xfstests-$(date +"%Y%m%d%H%M")${RANDOM}-${CONFIG_ALNUM}"
+ echo "Instance name: $INSTANCE_NAME"
+ else
+ echo "Given instance name: $INSTANCE_NAME"
+ fi
+ INSTANCE_NAME=$(echo "${INSTANCE_NAME}" | sed 's/_/-/g')
+ echo "Using instance name: $INSTANCE_NAME"
+}
+
+setup_xfstests_bld() {
+ git clone https://github.com/tytso/xfstests-bld $XFSTESTS_BLD_DIR
+ cd "$XFSTESTS_BLD_DIR" && make
+}
+
+#
+# gce_xfstests_run
+#
+# Run the gce_xfstests command
+#
+gce_xfstests_run() {
+ # The result bucket name without gs:// prefix.
+ local bucket_name="${RESULTS_BUCKET#gs://}"
+ if [[ "${ARCH}" = "x86_64" ]]; then
+ kernel="${RESULTS_BUCKET}/bzImage"
+ elif [[ "${ARCH}" = "arm64" ]]; then
+ kernel="${RESULTS_BUCKET}/Image"
+ else
+ echo "Unknown architecture: ${ARCH}"
+ exit 1
+ fi
+
+ # Sets up the config file for gce-xfstests.
+ mkdir -p /root/.config
+ cat <<EOF > /root/.config/gce-xfstests
+GS_BUCKET=${bucket_name}
+GCE_PROJECT=${PROJECT}
+GCE_IMAGE_PROJECT=${PROJECT}
+GCE_ZONE=${ZONE}
+GCE_KERNEL=${kernel}
+GCE_UPLOAD_SUMMARY=true
+EOF
+
+ local excluded_arg=""
+ local excluded_val=""
+ if [[ ! -z "${XFSTESTS_EXCLUDED}" ]] ; then
+ # In COS xfsquick test suite has many excluded test cases(~120 test cases),
+ # which are passed as command line arguments to gce-xfstests script.
+ # The arguments that we are passing to gce-xfstests is passed as kernel
+ # command line to kexec(This is added recently ). Since cos xfs is having
+ # too many excluded test cases, the kernel command line becomes too big and
+ # kexec fails with "Kernel command line too long for kernel! Cannot load /root/bzImage"
+ # Hence adding the excluded tests in a file instead of passing as an argument.
+
+ if [[ "${CONFIG}" = "xfs" ]] ; then
+ echo "${XFSTESTS_EXCLUDED}" | sed 's/,/\n/g' >> \
+ "${XFSTESTS_BLD_DIR}"/"${XFSTESTS_CFG_DIR}"/"${CONFIG}"/exclude
+ excluded_arg="--update-files"
+ else
+ excluded_arg="-X"
+ excluded_val="${XFSTESTS_EXCLUDED}"
+
+ fi
+ fi
+ echo $XFSTESTS_BLD_DIR/gce-xfstests --instance-name "${INSTANCE_NAME}" \
+ --kernel "${RESULTS_BUCKET}/bzImage" ${excluded_arg} "${excluded_val}" \
+ -c "${CONFIG}" -g "${GROUP}" --arch "${ARCH}"
+ # shellcheck disable=SC2086
+ # putting ${excluded_val} in quotes causes gce-xfstests to fail because it
+ # it takes empty value as an argument.
+ $XFSTESTS_BLD_DIR/gce-xfstests --instance-name "${INSTANCE_NAME}" \
+ --kernel "${kernel}" ${excluded_arg} ${excluded_val} \
+ -c "${CONFIG}" -g "${GROUP}" --arch "${ARCH}"
+}
+
+main() {
+ parse_args "$@"
+
+ setup_xfstests_bld
+
+ gce_xfstests_run
+}
+
+main "$@"
