testing/gpu_test.yaml

# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the License);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:

- name: 'gcr.io/cloud-builders/gcloud'
  entrypoint: '/bin/bash'
  args:
  - '-c'
  - |
    set +o histexpand
    test_list=("gcloud builds submit --config=testing/gpu_test/gpu_test.yaml\
                  --substitutions=_DRIVER_VERSION=470.223.02,_INPUT_IMAGE=cos-97-16919-450-16,_PROJECT_NAME=${_PROJECT_NAME},_ZONE=ZONE_SUB\
                  --async --format='value(ID)' ."
               "gcloud builds submit --config=testing/gpu_test/gpu_test.yaml\
                  --substitutions=_DRIVER_VERSION=470.223.02,_INPUT_IMAGE=cos-101-17162-386-22,_PROJECT_NAME=${_PROJECT_NAME},_ZONE=ZONE_SUB\
                  --async --format='value(ID)' ."
                "gcloud builds submit --config=testing/gpu_test/gpu_test.yaml\
                  --substitutions=_DRIVER_VERSION=470.223.02,_INPUT_IMAGE=cos-105-17412-294-29,_PROJECT_NAME=${_PROJECT_NAME},_ZONE=ZONE_SUB\
                  --async --format='value(ID)' ."
                "gcloud builds submit --config=testing/gpu_test/gpu_test.yaml\
                  --substitutions=_DRIVER_VERSION=535.154.05,_INPUT_IMAGE=cos-109-17800-147-15,_PROJECT_NAME=${_PROJECT_NAME},_ZONE=ZONE_SUB\
                  --async --format='value(ID)' .")

    primary_zone=us-central1-a
    secondary_zone=us-west1-b
    exit_code=0
    build_ids=()
    declare -A build_statuses
    declare -A build_ids_map
    declare -A retries
    for i in "${!test_list[@]}"; do
      test_name=test-"${i}"
      build_id="$(eval "${test_list[$i]//ZONE_SUB/$primary_zone}")"
      build_ids+=($build_id)
      build_ids_map[${build_id}]=${test_name}
    done
    for ((i=0; i < "${#build_ids[@]}"; i++ )); do
      build_id=${build_ids[$i]}
      test_name=${build_ids_map[$build_id]}
      status=""
      while true; do
          status="$(gcloud builds describe "${build_id}" --format='value(status)')"
          case "${status}" in
            "SUCCESS"|"INTERNAL_ERROR"|"TIMEOUT"|"CANCELLED")
              echo "${status}"
              break
              ;;
            "FAILURE")
              echo "Build ${build_id} failed"
              log_url="$(gcloud builds describe "${build_id}" --format='value(logUrl)')"
              echo "Logs: ${log_url}"
              if [[ -v retries[$test_name] ]]; then
                echo "Build failed in all zones, exiting"
                break
              fi
              echo "Retrying with different zone for test $test_name"
              new_build_id="$(eval "${test_list[${test_name##*-}]//ZONE_SUB/$secondary_zone}")"
              build_ids_map[${new_build_id}]=${test_name}
              build_ids+=($new_build_id)
              retries[${test_name}]=1
              break
              ;;
            "QUEUED"|"WORKING")
              sleep 5
              ;;
            "STATUS_UNKNOWN")
              echo "Received STATUS_UNKNOWN for build ${build_id}" 1>&2
              sleep 5
              ;;
            *)
              echo "Unknown status for build ${build_id}: ${status}" 1>&2
              break
              ;;
          esac
        done
      build_statuses[${test_name}]=${status}
    done
    for test in "${!build_statuses[@]}"; do
      status="${build_statuses[$test]}"
      if [[ "${status}" == "SUCCESS" ]]; then
         echo "Test ${test} succeeded"
      else
         echo "Test ${test} failed"
         exit_code=1
      fi
    done
    exit "${exit_code}"
options:
  machineType: 'N1_HIGHCPU_32'
timeout: "7200s"