client/site_tests/graphics_GLBench/graphics_GLBench.py

# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import os

from autotest_lib.client.bin import test, utils
from autotest_lib.client.common_lib import error
from autotest_lib.client.cros import perf
from autotest_lib.client.cros import service_stopper
from autotest_lib.client.cros.graphics import graphics_utils


class graphics_GLBench(graphics_utils.GraphicsTest):
  """Run glbench, a benchmark that times graphics intensive activities."""
  version = 1
  preserve_srcdir = True
  _services = None

  # Good images.
  reference_images_file = 'deps/glbench/glbench_reference_images.txt'
  # Images that are bad but for which the bug has not been fixed yet.
  knownbad_images_file = 'deps/glbench/glbench_knownbad_images.txt'
  # Images that are bad and for which a fix has been submitted.
  fixedbad_images_file = 'deps/glbench/glbench_fixedbad_images.txt'

  # These tests do not draw anything, they can only be used to check
  # performance.
  no_checksum_tests = set([
      'compositing_no_fill',
      'pixel_read',
      'texture_reuse_luminance_teximage2d',
      'texture_reuse_luminance_texsubimage2d',
      'texture_reuse_rgba_teximage2d',
      'texture_reuse_rgba_texsubimage2d',
      'context_glsimple',
      'swap_glsimple',
  ])

  blacklist = ''

  unit_higher_is_better = {
      'mpixels_sec': True,
      'mtexel_sec': True,
      'mtri_sec': True,
      'mvtx_sec': True,
      'us': False,
      '1280x768_fps': True
  }

  def setup(self):
    self.job.setup_dep(['glbench'])

  def initialize(self):
    super(graphics_GLBench, self).initialize()
    # If UI is running, we must stop it and restore later.
    self._services = service_stopper.ServiceStopper(['ui'])
    self._services.stop_services()

  def cleanup(self):
    if self._services:
      self._services.restore_services()
    super(graphics_GLBench, self).cleanup()

  def report_temperature(self, keyname):
    """Report current max observed temperature with given keyname.

    @param keyname: key to be used when reporting perf value.
    """
    temperature = utils.get_temperature_input_max()
    logging.info('%s = %f degree Celsius', keyname, temperature)
    self.output_perf_value(
        description=keyname,
        value=temperature,
        units='Celsius',
        higher_is_better=False)

  def report_temperature_critical(self, keyname):
    """Report temperature at which we will see throttling with given keyname.

    @param keyname: key to be used when reporting perf value.
    """
    temperature = utils.get_temperature_critical()
    logging.info('%s = %f degree Celsius', keyname, temperature)
    self.output_perf_value(
        description=keyname,
        value=temperature,
        units='Celsius',
        higher_is_better=False)

  def is_no_checksum_test(self, testname):
    """Check if given test requires no screenshot checksum.

    @param testname: name of test to check.
    """
    for prefix in self.no_checksum_tests:
      if testname.startswith(prefix):
        return True
    return False

  def load_imagenames(self, filename):
    """Loads text file with MD5 file names.

    @param filename: name of file to load.
    """
    imagenames = os.path.join(self.autodir, filename)
    with open(imagenames, 'r') as f:
      imagenames = f.read()
      return imagenames

  @graphics_utils.GraphicsTest.failure_report_decorator('graphics_GLBench')
  def run_once(self, options='', hasty=False):
    dep = 'glbench'
    dep_dir = os.path.join(self.autodir, 'deps', dep)
    self.job.install_pkg(dep, 'dep', dep_dir)

    options += self.blacklist

    # Run the test, saving is optional and helps with debugging
    # and reference image management. If unknown images are
    # encountered one can take them from the outdir and copy
    # them (after verification) into the reference image dir.
    exefile = os.path.join(self.autodir, 'deps/glbench/glbench')
    outdir = self.outputdir
    options += ' -save -outdir=' + outdir
    # Using the -hasty option we run only a subset of tests without waiting
    # for thermals to normalize. Test should complete in 15-20 seconds.
    if hasty:
      options += ' -hasty'

    cmd = '%s %s' % (exefile, options)
    summary = None
    pc_error_reason = None
    try:
      if hasty:
        # On BVT the test will not monitor thermals so we will not verify its
        # correct status using PerfControl
        summary = utils.run(cmd,
                            stderr_is_expected=False,
                            stdout_tee=utils.TEE_TO_LOGS,
                            stderr_tee=utils.TEE_TO_LOGS).stdout
      else:
        self.report_temperature_critical('temperature_critical')
        self.report_temperature('temperature_1_start')
        # Wrap the test run inside of a PerfControl instance to make machine
        # behavior more consistent.
        with perf.PerfControl() as pc:
          if not pc.verify_is_valid():
            raise error.TestFail('Failed: %s' % pc.get_error_reason())
          self.report_temperature('temperature_2_before_test')

          # Run the test. If it gets the CPU too hot pc should notice.
          summary = utils.run(cmd,
                              stderr_is_expected=False,
                              stdout_tee=utils.TEE_TO_LOGS,
                              stderr_tee=utils.TEE_TO_LOGS).stdout
          if not pc.verify_is_valid():
            # Defer error handling until after perf report.
            pc_error_reason = pc.get_error_reason()
    except error.CmdError:
      raise error.TestFail('Failed: CmdError running %s' % cmd)
    except error.CmdTimeoutError:
      raise error.TestFail('Failed: CmdTimeout running %s' % cmd)

    # Write a copy of stdout to help debug failures.
    results_path = os.path.join(self.outputdir, 'summary.txt')
    f = open(results_path, 'w+')
    f.write('# ---------------------------------------------------\n')
    f.write('# [' + cmd + ']\n')
    f.write(summary)
    f.write('\n# -------------------------------------------------\n')
    f.write('# [graphics_GLBench.py postprocessing]\n')

    # Analyze the output. Sample:
    ## board_id: NVIDIA Corporation - Quadro FX 380/PCI/SSE2
    ## Running: ../glbench -save -outdir=img
    #swap_swap = 221.36 us [swap_swap.pixmd5-20dbc...f9c700d2f.png]
    results = summary.splitlines()
    if not results:
      f.close()
      raise error.TestFail('Failed: No output from test. Check /tmp/' +
                           'test_that_latest/graphics_GLBench/summary.txt' +
                           ' for details.')

    # The good images, the silenced and the zombie/recurring failures.
    reference_imagenames = self.load_imagenames(self.reference_images_file)
    knownbad_imagenames = self.load_imagenames(self.knownbad_images_file)
    fixedbad_imagenames = self.load_imagenames(self.fixedbad_images_file)

    # Check if we saw GLBench end as expected (without crashing).
    test_ended_normal = False
    for line in results:
      if line.strip().startswith('@TEST_END'):
        test_ended_normal = True

    # Analyze individual test results in summary.
    # TODO(pwang): Raise TestFail if an error is detected during glbench.
    keyvals = {}
    failed_tests = {}
    for line in results:
      if not line.strip().startswith('@RESULT: '):
        continue
      keyval, remainder = line[9:].split('[')
      key, val = keyval.split('=')
      testname = key.strip()
      score, unit = val.split()
      testrating = float(score)
      imagefile = remainder.split(']')[0]

      higher = self.unit_higher_is_better.get(unit)
      if higher is None:
        raise error.TestFail('Failed: Unknown test unit "%s" for %s' %
                             (unit, testname))

      if not hasty:
        # Prepend unit to test name to maintain backwards compatibility with
        # existing per data.
        perf_value_name = '%s_%s' % (unit, testname)
        self.output_perf_value(
            description=perf_value_name,
            value=testrating,
            units=unit,
            higher_is_better=higher,
            graph=perf_value_name)
        # Add extra value to the graph distinguishing different boards.
        variant = utils.get_board_with_frequency_and_memory()
        desc = '%s-%s' % (perf_value_name, variant)
        self.output_perf_value(
            description=desc,
            value=testrating,
            units=unit,
            higher_is_better=higher,
            graph=perf_value_name)

      # Classify result image.
      if testrating == -1.0:
        # Tests that generate GL Errors.
        glerror = imagefile.split('=')[1]
        f.write('# GLError ' + glerror + ' during test (perf set to -3.0)\n')
        keyvals[testname] = -3.0
        failed_tests[testname] = 'GLError'
      elif testrating == 0.0:
        # Tests for which glbench does not generate a meaningful perf score.
        f.write('# No score for test\n')
        keyvals[testname] = 0.0
      elif imagefile in fixedbad_imagenames:
        # We know the image looked bad at some point in time but we thought
        # it was fixed. Throw an exception as a reminder.
        keyvals[testname] = -2.0
        f.write('# fixedbad [' + imagefile + '] (setting perf as -2.0)\n')
        failed_tests[testname] = imagefile
      elif imagefile in knownbad_imagenames:
        # We have triaged the failure and have filed a tracking bug.
        # Don't throw an exception and remind there is a problem.
        keyvals[testname] = -1.0
        f.write('# knownbad [' + imagefile + '] (setting perf as -1.0)\n')
        # This failure is whitelisted so don't add to failed_tests.
      elif imagefile in reference_imagenames:
        # Known good reference images (default).
        keyvals[testname] = testrating
      elif imagefile == 'none':
        # Tests that do not write images can't fail because of them.
        keyvals[testname] = testrating
      elif self.is_no_checksum_test(testname):
        # TODO(ihf): these really should not write any images
        keyvals[testname] = testrating
      else:
        # Completely unknown images. Raise a failure.
        keyvals[testname] = -2.0
        failed_tests[testname] = imagefile
        f.write('# unknown [' + imagefile + '] (setting perf as -2.0)\n')
    f.close()
    if not hasty:
      self.report_temperature('temperature_3_after_test')
      self.write_perf_keyval(keyvals)

    # Raise exception if images don't match.
    if failed_tests:
      logging.info('Some images are not matching their reference in %s.',
                   self.reference_images_file)
      logging.info('Please verify that the output images are correct '
                   'and if so copy them to the reference directory.')
      raise error.TestFail('Failed: Some images are not matching their '
                           'references. Check /tmp/'
                           'test_that_latest/graphics_GLBench/summary.txt'
                           ' for details.')

    if not test_ended_normal:
      raise error.TestFail(
          'Failed: No end marker. Presumed crash/missing images.')
    if pc_error_reason:
      raise error.TestFail('Failed: %s' % pc_error_reason)