testing/gpu_test/preload_test.cfg

#cloud-config
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

write_files:
  - path: /tmp/preloader-test/test.sh
    permissions: 0644
    owner: root
    content: |
      set -o errexit
      set -o pipefail

      # Templated by cloudbuild config that runs this test.
      EXPECTED_DRIVER_VERSION=%s

      trap 'fail exiting due to errors' EXIT

      fail() {
        echo "TestFail: $@"
      }

      testGpuDirExists() {
        if [[ ! -d /var/lib/nvidia ]]; then
          echo "/var/lib/nvidia is not a valid directory (does it exist?)"
          echo "testGpuDirExists fail"
          RESULT="fail"
          return
        fi
        echo "testGpuDirExists pass"
      }

      testDockerImageCached() {
        actual="$(docker images --format {{.Repository}} | sort)"
        if [[ "${actual}" != *ubuntu* ]]; then
          echo "ubuntu not found in docker images"
          echo "actual docker images: ${actual}"
          echo "testDockerImageCached fail"
          RESULT="fail"
          return
        fi
        if [[ "${actual}" != *gcr.io/cos-cloud/cos-gpu-installer* ]]; then
          echo "gcr.io/cos-cloud/cos-gpu-installer not found in docker images"
          echo "actual docker images: ${actual}"
          echo "testDockerImageCached fail"
          RESULT="fail"
          return
        fi
        echo "testDockerImageCached pass"
      }

      testActivateDrivers() {
        local activate_drivers="pass"
        /bin/bash /var/lib/nvidia/setup_gpu.sh
        if [[ ! -c "/dev/nvidia-uvm" ]]; then
          echo "Cannot find character device /dev/nvidia-uvm"
          activate_drivers="fail"
          RESULT="fail"
        fi
        if [[ ! -c "/dev/nvidia-uvm-tools" ]]; then
          echo "Cannot find character device /dev/nvidia-uvm-tools"
          activate_drivers="fail"
          RESULT="fail"
        fi
        if [[ ! -c "/dev/nvidia0" ]]; then
          echo "Cannot find character device /dev/nvidia0"
          activate_drivers="fail"
          RESULT="fail"
        fi
        if [[ ! -c "/dev/nvidiactl" ]]; then
          echo "Cannot find character device /dev/nvidiactl"
          activate_drivers="fail"
          RESULT="fail"
        fi
        if ! pgrep -f nvidia-persistenced > /dev/null; then
          echo "nvidia-persistenced is not running"
          activate_drivers="fail"
          RESULT="fail"
        fi
        if [[ "$(cat /proc/sys/kernel/softlockup_panic)" != "1" ]]; then
          echo "softlockup_panic is not set"
          activate_drivers="fail"
          RESULT="fail"
        fi
        if [[ "${activate_drivers}" == "fail" ]]; then
          echo "testActivateDrivers fail"
          return
        fi
        echo "testActivateDrivers pass"
      }

      testDriverVersion() {
        source /var/lib/nvidia/.cache
        if [[ "${EXPECTED_DRIVER_VERSION}" != "${DRIVER_VERSION}" ]]; then
          echo "Wrong driver version"
          echo "Expected: ${EXPECTED_DRIVER_VERSION}"
          echo "Actual: ${DRIVER_VERSION}"
          echo "testDriverVersion fail"
          RESULT="fail"
          return
        fi
        echo "testDriverVersion pass"
      }

      main() {
        RESULT="pass"
        testGpuDirExists
        testDockerImageCached
        testActivateDrivers
        testDriverVersion
        if [[ "${RESULT}" == "fail" ]]; then
          exit 1
        fi
      }

      main 2>&1 | sed "s/^/TestStatus: /"
      trap - EXIT
      echo "TestPass: all tests passed"

  - path: /etc/systemd/system/preloader-test.service
    permissions: 0644
    owner: root
    content: |
      [Unit]
      Description=Preloader test
      Wants=network-online.target gcr-online.target docker.service
      After=network-online.target gcr-online.target docker.service

      [Service]
      Type=oneshot
      RemainAfterExit=yes
      User=root
      ExecStart=/bin/bash /tmp/preloader-test/test.sh
      StandardOutput=tty
      StandardError=tty
      TTYPath=/dev/ttyS1

runcmd:
  - systemctl daemon-reload
  - systemctl --no-block start preloader-test.service