blob: ff6c4f96e86e38c914b5f57b8204e430e9952feb [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A module to handle the report format."""
from __future__ import print_function
import datetime
import functools
import itertools
import json
import os
import re
import time
from cros_utils.tabulator import AmeanResult
from cros_utils.tabulator import Cell
from cros_utils.tabulator import CoeffVarFormat
from cros_utils.tabulator import CoeffVarResult
from cros_utils.tabulator import Column
from cros_utils.tabulator import SamplesTableGenerator
from cros_utils.tabulator import Format
from cros_utils.tabulator import IterationResult
from cros_utils.tabulator import GmeanRatioResult
from cros_utils.tabulator import LiteralResult
from cros_utils.tabulator import MaxResult
from cros_utils.tabulator import MinResult
from cros_utils.tabulator import PValueFormat
from cros_utils.tabulator import PValueResult
from cros_utils.tabulator import RatioFormat
from cros_utils.tabulator import RawResult
from cros_utils.tabulator import StdResult
from cros_utils.tabulator import TableFormatter
from cros_utils.tabulator import TableGenerator
from cros_utils.tabulator import TablePrinter
from update_telemetry_defaults import TelemetryDefaults
from column_chart import ColumnChart
from results_organizer import OrganizeResults
import results_report_templates as templates
def ParseChromeosImage(chromeos_image):
"""Parse the chromeos_image string for the image and version.
The chromeos_image string will probably be in one of two formats:
1: <path-to-chroot>/src/build/images/<board>/<ChromeOS-version>.<datetime>/ \
2: <path-to-chroot>/chroot/tmp/<buildbot-build>/<ChromeOS-version>/ \
We parse these strings to find the 'chromeos_version' to store in the
json archive (without the .datatime bit in the first case); and also
the 'chromeos_image', which would be all of the first case, but only the
part after '/chroot/tmp' in the second case.
chromeos_image: string containing the path to the chromeos_image that
crosperf used for the test.
version, image: The results of parsing the input string, as explained
# Find the Chromeos Version, e.g. R45-2345.0.0.....
# chromeos_image should have been something like:
# <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin"
if chromeos_image.endswith('/chromiumos_test_image.bin'):
full_version = chromeos_image.split('/')[-2]
# Strip the date and time off of local builds (which have the format
# "").
version, _ = os.path.splitext(full_version)
version = ''
# Find the chromeos image. If it's somewhere in .../chroot/tmp/..., then
# it's an official image that got downloaded, so chop off the download path
# to make the official image name more clear.
official_image_path = '/chroot/tmp'
if official_image_path in chromeos_image:
image = chromeos_image.split(official_image_path, 1)[1]
image = chromeos_image
return version, image
def _AppendUntilLengthIs(gen, the_list, target_len):
"""Appends to `list` until `list` is `target_len` elements long.
Uses `gen` to generate elements.
the_list.extend(gen() for _ in range(target_len - len(the_list)))
return the_list
def _FilterPerfReport(event_threshold, report):
"""Filters out entries with `< event_threshold` percent in a perf report."""
def filter_dict(m):
return {
fn_name: pct for fn_name, pct in m.items() if pct >= event_threshold
return {event: filter_dict(m) for event, m in report.items()}
class _PerfTable(object):
"""Generates dicts from a perf table.
Dicts look like:
{'benchmark_name': {'perf_event_name': [LabelData]}}
where LabelData is a list of perf dicts, each perf dict coming from the same
Each perf dict looks like {'function_name': 0.10, ...} (where 0.10 is the
percentage of time spent in function_name).
def __init__(self,
read_perf_report is a function that takes a label name, benchmark name, and
benchmark iteration, and returns a dictionary describing the perf output for
that given run.
self.event_threshold = event_threshold
self._label_indices = {name: i for i, name in enumerate(label_names)}
self.perf_data = {}
for label in label_names:
for bench_name, bench_iterations in benchmark_names_and_iterations:
for i in range(bench_iterations):
report = read_perf_report(label, bench_name, i)
self._ProcessPerfReport(report, label, bench_name, i)
def _ProcessPerfReport(self, perf_report, label, benchmark_name, iteration):
"""Add the data from one run to the dict."""
perf_of_run = perf_report
if self.event_threshold is not None:
perf_of_run = _FilterPerfReport(self.event_threshold, perf_report)
if benchmark_name not in self.perf_data:
self.perf_data[benchmark_name] = {event: [] for event in perf_of_run}
ben_data = self.perf_data[benchmark_name]
label_index = self._label_indices[label]
for event in ben_data:
_AppendUntilLengthIs(list, ben_data[event], label_index + 1)
data_for_label = ben_data[event][label_index]
_AppendUntilLengthIs(dict, data_for_label, iteration + 1)
data_for_label[iteration] = perf_of_run[event] if perf_of_run else {}
def _GetResultsTableHeader(ben_name, iterations):
benchmark_info = ('Benchmark: {0}; Iterations: {1}'.format(
ben_name, iterations))
cell = Cell()
cell.string_value = benchmark_info
cell.header = True
return [[cell]]
def _GetDSOHeader(cwp_dso):
info = 'CWP_DSO: %s' % cwp_dso
cell = Cell()
cell.string_value = info
cell.header = False
return [[cell]]
def _ParseColumn(columns, iteration):
new_column = []
for column in columns:
if column.result.__class__.__name__ != 'RawResult':
Column(LiteralResult(i), Format(), str(i + 1))
for i in range(iteration))
return new_column
def _GetTables(benchmark_results, columns, table_type):
iter_counts = benchmark_results.iter_counts
result = benchmark_results.run_keyvals
tables = []
for bench_name, runs in result.items():
iterations = iter_counts[bench_name]
ben_table = _GetResultsTableHeader(bench_name, iterations)
all_runs_empty = all(not dict for label in runs for dict in label)
if all_runs_empty:
cell = Cell()
cell.string_value = ('This benchmark contains no result.'
' Is the benchmark name valid?')
cell_table = [[cell]]
table = TableGenerator(runs, benchmark_results.label_names).GetTable()
parsed_columns = _ParseColumn(columns, iterations)
tf = TableFormatter(table, parsed_columns)
cell_table = tf.GetCellTable(table_type)
return tables
def _GetPerfTables(benchmark_results, columns, table_type):
p_table = _PerfTable(benchmark_results.benchmark_names_and_iterations,
tables = []
for benchmark in p_table.perf_data:
iterations = benchmark_results.iter_counts[benchmark]
ben_table = _GetResultsTableHeader(benchmark, iterations)
benchmark_data = p_table.perf_data[benchmark]
table = []
for event in benchmark_data:
tg = TableGenerator(
table = tg.GetTable(ResultsReport.PERF_ROWS)
parsed_columns = _ParseColumn(columns, iterations)
tf = TableFormatter(table, parsed_columns)
table = tf.GetCellTable(table_type, headers=False)
return tables
def _GetSamplesTables(benchmark_results, columns, table_type):
tables = []
dso_header_table = _GetDSOHeader(benchmark_results.cwp_dso)
(table, new_keyvals, iter_counts) = SamplesTableGenerator(
benchmark_results.run_keyvals, benchmark_results.label_names,
benchmark_results.iter_counts, benchmark_results.weights).GetTable()
parsed_columns = _ParseColumn(columns, 1)
tf = TableFormatter(table, parsed_columns, samples_table=True)
cell_table = tf.GetCellTable(table_type)
return (tables, new_keyvals, iter_counts)
class ResultsReport(object):
"""Class to handle the report format."""
def __init__(self, results):
self.benchmark_results = results
def _GetTablesWithColumns(self, columns, table_type, summary_type):
if summary_type == 'perf':
get_tables = _GetPerfTables
elif summary_type == 'samples':
get_tables = _GetSamplesTables
get_tables = _GetTables
ret = get_tables(self.benchmark_results, columns, table_type)
# If we are generating a samples summary table, the return value of
# get_tables will be a tuple, and we will update the benchmark_results for
# composite benchmark so that full table can use it.
if isinstance(ret, tuple):
self.benchmark_results.run_keyvals = ret[1]
self.benchmark_results.iter_counts = ret[2]
ret = ret[0]
return ret
def GetFullTables(self, perf=False):
ignore_min_max = self.benchmark_results.ignore_min_max
columns = [
Column(RawResult(), Format()),
Column(MinResult(), Format()),
Column(MaxResult(), Format()),
Column(AmeanResult(ignore_min_max), Format()),
Column(StdResult(ignore_min_max), Format(), 'StdDev'),
Column(CoeffVarResult(ignore_min_max), CoeffVarFormat(), 'StdDev/Mean'),
Column(GmeanRatioResult(ignore_min_max), RatioFormat(), 'GmeanSpeedup'),
Column(PValueResult(ignore_min_max), PValueFormat(), 'p-value')
return self._GetTablesWithColumns(columns, 'full', perf)
def GetSummaryTables(self, summary_type=''):
ignore_min_max = self.benchmark_results.ignore_min_max
columns = []
if summary_type == 'samples':
columns += [Column(IterationResult(), Format(), 'Iterations [Pass:Fail]')]
columns += [
AmeanResult(ignore_min_max), Format(),
'Weighted Samples Amean' if summary_type == 'samples' else ''),
Column(StdResult(ignore_min_max), Format(), 'StdDev'),
Column(CoeffVarResult(ignore_min_max), CoeffVarFormat(), 'StdDev/Mean'),
Column(GmeanRatioResult(ignore_min_max), RatioFormat(), 'GmeanSpeedup'),
Column(PValueResult(ignore_min_max), PValueFormat(), 'p-value')
return self._GetTablesWithColumns(columns, 'summary', summary_type)
def _PrintTable(tables, out_to):
# tables may be None.
if not tables:
return ''
if out_to == 'HTML':
out_type = TablePrinter.HTML
elif out_to == 'PLAIN':
out_type = TablePrinter.PLAIN
elif out_to == 'CONSOLE':
out_type = TablePrinter.CONSOLE
elif out_to == 'TSV':
out_type = TablePrinter.TSV
elif out_to == 'EMAIL':
out_type = TablePrinter.EMAIL
raise ValueError('Invalid out_to value: %s' % (out_to,))
printers = (TablePrinter(table, out_type) for table in tables)
return ''.join(printer.Print() for printer in printers)
class TextResultsReport(ResultsReport):
"""Class to generate text result report."""
H1_STR = '==========================================='
H2_STR = '-------------------------------------------'
def __init__(self, results, email=False, experiment=None):
super(TextResultsReport, self).__init__(results) = email
self.experiment = experiment
def _MakeTitle(title):
header_line = TextResultsReport.H1_STR
# '' at the end gives one newline.
return '\n'.join([header_line, title, header_line, ''])
def _MakeSection(title, body):
header_line = TextResultsReport.H2_STR
# '\n' at the end gives us two newlines.
return '\n'.join([header_line, title, header_line, body, '\n'])
def FromExperiment(experiment, email=False):
results = BenchmarkResults.FromExperiment(experiment)
return TextResultsReport(results, email, experiment)
def GetStatusTable(self):
"""Generate the status table by the tabulator."""
table = [['', '']]
columns = [
Column(LiteralResult(iteration=0), Format(), 'Status'),
Column(LiteralResult(iteration=1), Format(), 'Failing Reason')
for benchmark_run in self.experiment.benchmark_runs:
status = [,
[benchmark_run.timeline.GetLastEvent(), benchmark_run.failure_reason]
cell_table = TableFormatter(table, columns).GetCellTable('status')
return [cell_table]
def GetTotalWaitCooldownTime(self):
"""Get cooldown wait time in seconds from experiment benchmark runs.
Dictionary {'dut': int(wait_time_in_seconds)}
waittime_dict = {}
for dut in self.experiment.machine_manager.GetMachines():
waittime_dict[] = dut.GetCooldownWaitTime()
return waittime_dict
def GetReport(self):
"""Generate the report for email and console."""
output_type = 'EMAIL' if else 'CONSOLE'
experiment = self.experiment
sections = []
if experiment is not None:
title_contents = "Results report for '%s'" % (,)
title_contents = 'Results report'
if not self.benchmark_results.cwp_dso:
summary_table = _PrintTable(self.GetSummaryTables(), output_type)
summary_table = _PrintTable(
self.GetSummaryTables(summary_type='samples'), output_type)
sections.append(self._MakeSection('Summary', summary_table))
if experiment is not None:
table = _PrintTable(self.GetStatusTable(), output_type)
sections.append(self._MakeSection('Benchmark Run Status', table))
if not self.benchmark_results.cwp_dso:
perf_table = _PrintTable(
self.GetSummaryTables(summary_type='perf'), output_type)
sections.append(self._MakeSection('Perf Data', perf_table))
if experiment is not None:
experiment_file = experiment.experiment_file
sections.append(self._MakeSection('Experiment File', experiment_file))
cpu_info = experiment.machine_manager.GetAllCPUInfo(experiment.labels)
sections.append(self._MakeSection('CPUInfo', cpu_info))
totaltime = (
time.time() - experiment.start_time) if experiment.start_time else 0
totaltime_str = 'Total experiment time:\n%d min' % (totaltime // 60)
cooldown_waittime_list = ['Cooldown wait time:']
# When running experiment on multiple DUTs cooldown wait time may vary
# on different devices. In addition its combined time may exceed total
# experiment time which will look weird but it is reasonable.
# For this matter print cooldown time per DUT.
for dut, waittime in sorted(self.GetTotalWaitCooldownTime().items()):
cooldown_waittime_list.append('DUT %s: %d min' % (dut, waittime // 60))
cooldown_waittime_str = '\n'.join(cooldown_waittime_list)
self._MakeSection('Duration', '\n\n'.join(
[totaltime_str, cooldown_waittime_str])))
return '\n'.join(sections)
def _GetHTMLCharts(label_names, test_results):
charts = []
for item, runs in test_results.items():
# Fun fact: label_names is actually *entirely* useless as a param, since we
# never add headers. We still need to pass it anyway.
table = TableGenerator(runs, label_names).GetTable()
columns = [
Column(AmeanResult(), Format()),
Column(MinResult(), Format()),
Column(MaxResult(), Format())
tf = TableFormatter(table, columns)
data_table = tf.GetCellTable('full', headers=False)
for cur_row_data in data_table:
test_key = cur_row_data[0].string_value
title = '{0}: {1}'.format(item, test_key.replace('/', ''))
chart = ColumnChart(title, 300, 200)
chart.AddColumn('Label', 'string')
chart.AddColumn('Average', 'number')
chart.AddColumn('Min', 'number')
chart.AddColumn('Max', 'number')
chart.AddSeries('Min', 'line', 'black')
chart.AddSeries('Max', 'line', 'black')
cur_index = 1
for label in label_names:
label, cur_row_data[cur_index].value,
cur_row_data[cur_index + 1].value, cur_row_data[cur_index + 2].value
if isinstance(cur_row_data[cur_index].value, str):
chart = None
cur_index += 3
if chart:
return charts
class HTMLResultsReport(ResultsReport):
"""Class to generate html result report."""
def __init__(self, benchmark_results, experiment=None):
super(HTMLResultsReport, self).__init__(benchmark_results)
self.experiment = experiment
def FromExperiment(experiment):
return HTMLResultsReport(
BenchmarkResults.FromExperiment(experiment), experiment=experiment)
def GetReport(self):
label_names = self.benchmark_results.label_names
test_results = self.benchmark_results.run_keyvals
charts = _GetHTMLCharts(label_names, test_results)
chart_javascript = ''.join(chart.GetJavascript() for chart in charts)
chart_divs = ''.join(chart.GetDiv() for chart in charts)
if not self.benchmark_results.cwp_dso:
summary_table = self.GetSummaryTables()
perf_table = self.GetSummaryTables(summary_type='perf')
summary_table = self.GetSummaryTables(summary_type='samples')
perf_table = None
full_table = self.GetFullTables()
experiment_file = ''
if self.experiment is not None:
experiment_file = self.experiment.experiment_file
# Use kwargs for sanity, and so that testing is a bit easier.
return templates.GenerateHTMLPage(
def ParseStandardPerfReport(report_data):
"""Parses the output of `perf report`.
It'll parse the following:
# Samples: 1234M of event 'foo'
1.23% command shared_object location function::name
1.22% command shared_object location function2::name
# Samples: 999K of event 'bar'
0.23% command shared_object location function3::name
{'foo': {'function::name': 1.23, 'function2::name': 1.22},
'bar': {'function3::name': 0.23, etc.}}
# This function fails silently on its if it's handed a string (as opposed to a
# list of lines). So, auto-split if we do happen to get a string.
if isinstance(report_data, str):
report_data = report_data.splitlines()
# When switching to python3 catch the case when bytes are passed.
elif isinstance(report_data, bytes):
raise TypeError()
# Samples: N{K,M,G} of event 'event-name'
samples_regex = re.compile(r"#\s+Samples: \d+\S? of event '([^']+)'")
# We expect lines like:
# N.NN% command samples shared_object [location] symbol
# Note that we're looking at stripped lines, so there is no space at the
# start.
perf_regex = re.compile(r'^(\d+(?:.\d*)?)%' # N.NN%
r'\s*\d+' # samples count (ignored)
r'\s*\S+' # command (ignored)
r'\s*\S+' # shared_object (ignored)
r'\s*\[.\]' # location (ignored)
r'\s*(\S.+)' # function
stripped_lines = (l.strip() for l in report_data)
nonempty_lines = (l for l in stripped_lines if l)
# Ignore all lines before we see samples_regex
interesting_lines = itertools.dropwhile(lambda x: not samples_regex.match(x),
first_sample_line = next(interesting_lines, None)
# Went through the entire file without finding a 'samples' header. Quit.
if first_sample_line is None:
return {}
sample_name = samples_regex.match(first_sample_line).group(1)
current_result = {}
results = {sample_name: current_result}
for line in interesting_lines:
samples_match = samples_regex.match(line)
if samples_match:
sample_name =
current_result = {}
results[sample_name] = current_result
match = perf_regex.match(line)
if not match:
percentage_str, func_name = match.groups()
percentage = float(percentage_str)
except ValueError:
# Couldn't parse it; try to be "resilient".
current_result[func_name] = percentage
return results
def _ReadExperimentPerfReport(results_directory, label_name, benchmark_name,
"""Reads a perf report for the given benchmark. Returns {} on failure.
The result should be a map of maps; it should look like:
{perf_event_name: {function_name: pct_time_spent}}, e.g.
{'cpu_cycles': {'_malloc': 10.0, '_free': 0.3, ...}}
raw_dir_name = label_name + benchmark_name + str(benchmark_iteration + 1)
dir_name = ''.join(c for c in raw_dir_name if c.isalnum())
file_name = os.path.join(results_directory, dir_name, '')
with open(file_name) as in_file:
return ParseStandardPerfReport(in_file)
except IOError:
# Yes, we swallow any IO-related errors.
return {}
# Split out so that testing (specifically: mocking) is easier
def _ExperimentToKeyvals(experiment, for_json_report):
"""Converts an experiment to keyvals."""
return OrganizeResults(
experiment.benchmark_runs, experiment.labels, json_report=for_json_report)
class BenchmarkResults(object):
"""The minimum set of fields that any ResultsReport will take."""
def __init__(self,
if read_perf_report is None:
def _NoPerfReport(*_args, **_kwargs):
return {}
read_perf_report = _NoPerfReport
self.label_names = label_names
self.benchmark_names_and_iterations = benchmark_names_and_iterations
self.iter_counts = dict(benchmark_names_and_iterations)
self.run_keyvals = run_keyvals
self.ignore_min_max = ignore_min_max
self.read_perf_report = read_perf_report
self.cwp_dso = cwp_dso
self.weights = dict(weights) if weights else None
def FromExperiment(experiment, for_json_report=False):
label_names = [ for label in experiment.labels]
benchmark_names_and_iterations = [(, benchmark.iterations)
for benchmark in experiment.benchmarks]
run_keyvals = _ExperimentToKeyvals(experiment, for_json_report)
ignore_min_max = experiment.ignore_min_max
read_perf_report = functools.partial(_ReadExperimentPerfReport,
cwp_dso = experiment.cwp_dso
weights = [(, benchmark.weight)
for benchmark in experiment.benchmarks]
return BenchmarkResults(label_names, benchmark_names_and_iterations,
run_keyvals, ignore_min_max, read_perf_report,
cwp_dso, weights)
def _GetElemByName(name, from_list):
"""Gets an element from the given list by its name field.
Raises an error if it doesn't find exactly one match.
elems = [e for e in from_list if == name]
if len(elems) != 1:
raise ValueError('Expected 1 item named %s, found %d' % (name, len(elems)))
return elems[0]
def _Unlist(l):
"""If l is a list, extracts the first element of l. Otherwise, returns l."""
return l[0] if isinstance(l, list) else l
class JSONResultsReport(ResultsReport):
"""Class that generates JSON reports for experiments."""
def __init__(self,
"""Construct a JSONResultsReport.
json_args is the dict of arguments we pass to json.dumps in GetReport().
super(JSONResultsReport, self).__init__(benchmark_results)
defaults = TelemetryDefaults()
summary_field_defaults = defaults.GetDefault()
if summary_field_defaults is None:
summary_field_defaults = {}
self.summary_field_defaults = summary_field_defaults
if json_args is None:
json_args = {}
self.json_args = json_args
self.experiment = experiment
if not benchmark_date:
timestamp = datetime.datetime.strftime(,
'%Y-%m-%d %H:%M:%S')
benchmark_date, benchmark_time = timestamp.split(' ') = benchmark_date
self.time = benchmark_time
def FromExperiment(experiment,
benchmark_results = BenchmarkResults.FromExperiment(
experiment, for_json_report=True)
return JSONResultsReport(benchmark_results, benchmark_date, benchmark_time,
experiment, json_args)
def GetReportObjectIgnoringExperiment(self):
"""Gets the JSON report object specifically for the output data.
Ignores any experiment-specific fields (e.g. board, machine checksum, ...).
benchmark_results = self.benchmark_results
label_names = benchmark_results.label_names
summary_field_defaults = self.summary_field_defaults
final_results = []
for test, test_results in benchmark_results.run_keyvals.items():
for label_name, label_results in zip(label_names, test_results):
for iter_results in label_results:
passed = iter_results.get('retval') == 0
json_results = {
'time': self.time,
'label': label_name,
'test_name': test,
'pass': passed,
if not passed:
# Get overall results.
summary_fields = summary_field_defaults.get(test)
if summary_fields is not None:
value = []
json_results['overall_result'] = value
for f in summary_fields:
v = iter_results.get(f)
if v is None:
# New telemetry results format: sometimes we get a list of lists
# now.
v = _Unlist(_Unlist(v))
value.append((f, float(v)))
# Get detailed results.
detail_results = {}
json_results['detailed_results'] = detail_results
for k, v in iter_results.items():
if k == 'retval' or k == 'PASS' or k == ['PASS'] or v == 'PASS':
v = _Unlist(v)
if 'machine' in k:
json_results[k] = v
elif v is not None:
if isinstance(v, list):
detail_results[k] = [float(d) for d in v]
detail_results[k] = float(v)
return final_results
def GetReportObject(self):
"""Generate the JSON report, returning it as a python object."""
report_list = self.GetReportObjectIgnoringExperiment()
if self.experiment is not None:
return report_list
def _AddExperimentSpecificFields(self, report_list):
"""Add experiment-specific data to the JSON report."""
board = self.experiment.labels[0].board
manager = self.experiment.machine_manager
for report in report_list:
label_name = report['label']
label = _GetElemByName(label_name, self.experiment.labels)
img_path = os.path.realpath(os.path.expanduser(label.chromeos_image))
ver, img = ParseChromeosImage(img_path)
'board': board,
'chromeos_image': img,
'chromeos_version': ver,
'chrome_version': label.chrome_version,
'compiler': label.compiler
if not report['pass']:
if 'machine_checksum' not in report:
report['machine_checksum'] = manager.machine_checksum[label_name]
if 'machine_string' not in report:
report['machine_string'] = manager.machine_checksum_string[label_name]
def GetReport(self):
"""Dump the results of self.GetReportObject() to a string as JSON."""
# This exists for consistency with the other GetReport methods.
# Specifically, they all return strings, so it's a bit awkward if the JSON
# results reporter returns an object.
return json.dumps(self.GetReportObject(), **self.json_args)