crosperf/results_organizer.py

# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Parse data from benchmark_runs for tabulator."""

from __future__ import print_function

import errno
import json
import os
import re
import sys

from cros_utils import misc

_TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json'
_DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}')


def _AdjustIteration(benchmarks, max_dup, bench):
  """Adjust the interation numbers if they have keys like ABCD{i}."""
  for benchmark in benchmarks:
    if benchmark.name != bench or benchmark.iteration_adjusted:
      continue
    benchmark.iteration_adjusted = True
    benchmark.iterations *= (max_dup + 1)


def _GetMaxDup(data):
  """Find the maximum i inside ABCD{i}.

  data should be a [[[Key]]], where Key is a string that may look like
  ABCD{i}.
  """
  max_dup = 0
  for label in data:
    for run in label:
      for key in run:
        match = _DUP_KEY_REGEX.match(key)
        if match:
          max_dup = max(max_dup, int(match.group(2)))
  return max_dup


def _Repeat(func, times):
  """Returns the result of running func() n times."""
  return [func() for _ in range(times)]


def _DictWithReturnValues(retval, pass_fail):
  """Create a new dictionary pre-populated with success/fail values."""
  new_dict = {}
  # Note: 0 is a valid retval; test to make sure it's not None.
  if retval is not None:
    new_dict['retval'] = retval
  if pass_fail:
    new_dict[''] = pass_fail
  return new_dict


def _GetNonDupLabel(max_dup, runs):
  """Create new list for the runs of the same label.

  Specifically, this will split out keys like foo{0}, foo{1} from one run into
  their own runs. For example, given a run like:
    {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5}

  You'll get:
    [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}]

  Hands back the lists of transformed runs, all concatenated together.
  """
  new_runs = []
  for run in runs:
    run_retval = run.get('retval', None)
    run_pass_fail = run.get('', None)
    new_run = {}
    # pylint: disable=cell-var-from-loop
    added_runs = _Repeat(
        lambda: _DictWithReturnValues(run_retval, run_pass_fail), max_dup)
    for key, value in run.items():
      match = _DUP_KEY_REGEX.match(key)
      if not match:
        new_run[key] = value
      else:
        new_key, index_str = match.groups()
        added_runs[int(index_str) - 1][new_key] = str(value)
    new_runs.append(new_run)
    new_runs += added_runs
  return new_runs


def _DuplicatePass(result, benchmarks):
  """Properly expands keys like `foo{1}` in `result`."""
  for bench, data in result.items():
    max_dup = _GetMaxDup(data)
    # If there's nothing to expand, there's nothing to do.
    if not max_dup:
      continue
    for i, runs in enumerate(data):
      data[i] = _GetNonDupLabel(max_dup, runs)
    _AdjustIteration(benchmarks, max_dup, bench)


def _ReadSummaryFile(filename):
  """Reads the summary file at filename."""
  dirname, _ = misc.GetRoot(filename)
  fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE)
  try:
    # Slurp the summary file into a dictionary. The keys in the dictionary are
    # the benchmark names. The value for a key is a list containing the names
    # of all the result fields that should be returned in a 'default' report.
    with open(fullname) as in_file:
      return json.load(in_file)
  except IOError as e:
    # ENOENT means "no such file or directory"
    if e.errno == errno.ENOENT:
      return {}
    raise


def _MakeOrganizeResultOutline(benchmark_runs, labels):
  """Creates the "outline" of the OrganizeResults result for a set of runs.

  Report generation returns lists of different sizes, depending on the input
  data. Depending on the order in which we iterate through said input data, we
  may populate the Nth index of a list, then the N-1st, then the N+Mth, ...

  It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't
  have to worry about resizing while computing results.
  """
  # Count how many iterations exist for each benchmark run.
  # We can't simply count up, since we may be given an incomplete set of
  # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3])
  iteration_count = {}
  for run in benchmark_runs:
    name = run.benchmark.name
    old_iterations = iteration_count.get(name, -1)
    # N.B. run.iteration starts at 1, not 0.
    iteration_count[name] = max(old_iterations, run.iteration)

  # Result structure: {benchmark_name: [[{key: val}]]}
  result = {}
  for run in benchmark_runs:
    name = run.benchmark.name
    num_iterations = iteration_count[name]
    # default param makes cros lint be quiet about defining num_iterations in a
    # loop.
    make_dicts = lambda n=num_iterations: _Repeat(dict, n)
    result[name] = _Repeat(make_dicts, len(labels))
  return result


def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False):
  """Create a dict from benchmark_runs.

  The structure of the output dict is as follows:
  {"benchmark_1":[
    [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}]
    #one label
    []
    #the other label
    ]
   "benchmark_2":
    [
    ]}.
  """
  result = _MakeOrganizeResultOutline(benchmark_runs, labels)
  label_names = [label.name for label in labels]
  label_indices = {name: i for i, name in enumerate(label_names)}
  summary_file = _ReadSummaryFile(sys.argv[0])

  if benchmarks is None:
    benchmarks = []

  for benchmark_run in benchmark_runs:
    if not benchmark_run.result:
      continue
    benchmark = benchmark_run.benchmark
    label_index = label_indices[benchmark_run.label.name]
    cur_label_list = result[benchmark.name][label_index]
    cur_dict = cur_label_list[benchmark_run.iteration - 1]

    show_all_results = json_report or benchmark.show_all_results
    if not show_all_results:
      summary_list = summary_file.get(benchmark.name)
      if summary_list:
        for key in benchmark_run.result.keyvals.keys():
          if any(
              key.startswith(added_key)
              for added_key in ['retval', 'cpufreq', 'cputemp']):
            summary_list.append(key)
      else:
        # Did not find test_name in json file; show everything.
        show_all_results = True
    if benchmark_run.result.cwp_dso:
      # If we are in cwp approximation mode, we only care about samples
      if 'samples' in benchmark_run.result.keyvals:
        cur_dict['samples'] = benchmark_run.result.keyvals['samples']
      cur_dict['retval'] = benchmark_run.result.keyvals['retval']
      for key, value in benchmark_run.result.keyvals.items():
        if any(
            key.startswith(cpustat_keyword)
            for cpustat_keyword in ['cpufreq', 'cputemp']):
          cur_dict[key] = value
    else:
      for test_key in benchmark_run.result.keyvals:
        if show_all_results or test_key in summary_list:
          cur_dict[test_key] = benchmark_run.result.keyvals[test_key]
    # Occasionally Telemetry tests will not fail but they will not return a
    # result, either.  Look for those cases, and force them to be a fail.
    # (This can happen if, for example, the test has been disabled.)
    if len(cur_dict) == 1 and cur_dict['retval'] == 0:
      cur_dict['retval'] = 1
      benchmark_run.result.keyvals['retval'] = 1
      # TODO: This output should be sent via logger.
      print(
          "WARNING: Test '%s' appears to have succeeded but returned"
          ' no results.' % benchmark.name,
          file=sys.stderr)
    if json_report and benchmark_run.machine:
      cur_dict['machine'] = benchmark_run.machine.name
      cur_dict['machine_checksum'] = benchmark_run.machine.checksum
      cur_dict['machine_string'] = benchmark_run.machine.checksum_string
  _DuplicatePass(result, benchmarks)
  return result