blob: 531245e03083554eea7296822da7806bdb118899 [file] [log] [blame] [edit]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A module providing the summary for multiple test results.
This firmware_summary module is used to collect the test results of
multiple rounds from the logs generated by different firmware versions.
The test results of the various validators of every gesture are displayed.
In addition, the test results of every validator across all gestures are
also summarized.
Usage:
$ python firmware_summary log_directory
A typical summary output looks like
Test Summary (by gesture) : fw_2.41 fw_2.42 count
---------------------------------------------------------------------
one_finger_tracking
CountTrackingIDValidator : 1.00 0.90 12
LinearityBothEndsValidator : 0.97 0.89 12
LinearityMiddleValidator : 1.00 1.00 12
NoGapValidator : 0.74 0.24 12
NoReversedMotionBothEndsValidator : 0.68 0.34 12
NoReversedMotionMiddleValidator : 1.00 1.00 12
ReportRateValidator : 1.00 1.00 12
one_finger_to_edge
CountTrackingIDValidator : 1.00 1.00 4
LinearityBothEndsValidator : 0.88 0.89 4
LinearityMiddleValidator : 1.00 1.00 4
NoGapValidator : 0.50 0.00 4
NoReversedMotionMiddleValidator : 1.00 1.00 4
RangeValidator : 1.00 1.00 4
...
Test Summary (by validator) : fw_2.4 fw_2.4.a count
---------------------------------------------------------------------
CountPacketsValidator : 1.00 0.82 6
CountTrackingIDValidator : 0.92 0.88 84
...
"""
import getopt
import os
import sys
import firmware_log
import test_conf as conf
from collections import defaultdict
from common_util import print_and_exit
from firmware_constants import OPTIONS
from test_conf import (log_root_dir, merged_validators, segment_weights,
validator_weights)
from validators import BaseValidator, get_parent_validators
class OptionsDisplayMetrics:
"""The options of displaying metrics."""
# Defining the options of displaying metrics
HIDE_SOME_METRICS_STATS = '0'
DISPLAY_ALL_METRICS_STATS = '1'
DISPLAY_ALL_METRICS_WITH_RAW_VALUES = '2'
DISPLAY_METRICS_OPTIONS = [HIDE_SOME_METRICS_STATS,
DISPLAY_ALL_METRICS_STATS,
DISPLAY_ALL_METRICS_WITH_RAW_VALUES]
DISPLAY_METRICS_DEFAULT = DISPLAY_ALL_METRICS_WITH_RAW_VALUES
def __init__(self, option):
"""Initialize with the level value.
@param option: the option of display metrics
"""
if option not in self.DISPLAY_METRICS_OPTIONS:
option = self.DISPLAY_METRICS_DEFAULT
# To display all metrics statistics grouped by validators?
self.display_all_stats = (
option == self.DISPLAY_ALL_METRICS_STATS or
option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES)
# To display the raw metrics values in details on file basis?
self.display_raw_values = (
option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES)
class FirmwareSummary:
"""Summary for touch device firmware tests."""
def __init__(self, log_dir, display_metrics=False, debug_flag=False,
display_scores=False, individual_round_flag=False,
segment_weights=segment_weights,
validator_weights=validator_weights):
""" segment_weights and validator_weights are passed as arguments
so that it is possible to assign arbitrary weights in unit tests.
"""
if os.path.isdir(log_dir):
self.log_dir = log_dir
else:
error_msg = 'Error: The test result directory does not exist: %s'
print error_msg % log_dir
sys.exit(1)
self.display_metrics = display_metrics
self.display_scores = display_scores
self.slog = firmware_log.SummaryLog(log_dir,
segment_weights,
validator_weights,
individual_round_flag,
debug_flag)
def _print_summary_title(self, summary_title_str):
"""Print the summary of the test results by gesture."""
# Create a flexible column title format according to the number of
# firmware versions which could be 1, 2, or more.
#
# A typical summary title looks like
# Test Summary () : fw_11.26 fw_11.23
# mean ssd count mean ssd count
# ----------------------------------------------------------------------
#
# The 1st line above is called title_fw.
# The 2nd line above is called title_statistics.
#
# As an example for 2 firmwares, title_fw_format looks like:
# '{0:<37}: {1:>12} {2:>21}'
title_fw_format_list = ['{0:<37}:',]
for i in range(len(self.slog.fws)):
format_space = 12 if i == 0 else (12 + 9)
title_fw_format_list.append('{%d:>%d}' % (i + 1, format_space))
title_fw_format = ' '.join(title_fw_format_list)
# As an example for 2 firmwares, title_statistics_format looks like:
# '{0:>47} {1:>6} {2:>5} {3:>8} {4:>6} {5:>5}'
title_statistics_format_list = []
for i in range(len(self.slog.fws)):
format_space = (12 + 35) if i == 0 else 8
title_statistics_format_list.append('{%d:>%d}' % (3 * i,
format_space))
title_statistics_format_list.append('{%d:>%d}' % (3 * i + 1 , 6))
title_statistics_format_list.append('{%d:>%d}' % (3 * i + 2 , 5))
title_statistics_format = ' '.join(title_statistics_format_list)
# Create title_fw_list
# As an example for two firmware versions, it looks like
# ['Test Summary (by gesture)', 'fw_2.4', 'fw_2.5']
title_fw_list = [summary_title_str,] + self.slog.fws
# Create title_statistics_list
# As an example for two firmware versions, it looks like
# ['mean', 'ssd', 'count', 'mean', 'ssd', 'count', ]
title_statistics_list = ['mean', 'ssd', 'count'] * len(self.slog.fws)
# Print the title.
title_fw = title_fw_format.format(*title_fw_list)
title_statistics = title_statistics_format.format(
*title_statistics_list)
print '\n\n', title_fw
print title_statistics
print '-' * len(title_statistics)
def _print_result_stats(self, gesture=None):
"""Print the result statistics of validators."""
for validator in self.slog.validators:
stat_scores_data = []
statistics_format_list = []
for fw in self.slog.fws:
result = self.slog.get_result(fw=fw, gesture=gesture,
validators=validator)
scores_data = result.stat_scores.all_data
if scores_data:
stat_scores_data += scores_data
statistics_format_list.append('{:>8.2f} {:>6.2f} {:>5}')
else:
stat_scores_data.append('')
statistics_format_list.append('{:>21}')
# Print the score statistics of all firmwares on the same row.
if any(stat_scores_data):
stat_scores_data.insert(0, validator)
statistics_format_list.insert(0,' {:<35}:')
statistics_format = ' '.join(statistics_format_list)
print statistics_format.format(*tuple(stat_scores_data))
def _print_result_stats_by_gesture(self):
"""Print the summary of the test results by gesture."""
self._print_summary_title('Test Summary (by gesture)')
for gesture in self.slog.gestures:
print gesture
self._print_result_stats(gesture=gesture)
def _print_result_stats_by_validator(self):
"""Print the summary of the test results by validator. The validator
results of all gestures are combined to compute the statistics.
"""
self._print_summary_title('Test Summary (by validator)')
self._print_result_stats()
def _get_metric_name_for_display(self, metric_name):
"""Get the metric name for display.
We would like to shorten the metric name when displayed.
@param metric_name: a metric name
"""
return metric_name.split('--')[0]
def _get_merged_validators(self):
merged = defaultdict(list)
for validator_name in self.slog.validators:
parents = get_parent_validators(validator_name)
for parent in parents:
if parent in merged_validators:
merged[parent].append(validator_name)
break
else:
merged[validator_name] = [validator_name,]
return sorted(merged.values())
def _print_statistics_of_metrics(self, detailed=True, gesture=None):
"""Print the statistics of metrics by gesture or by validator.
@param gesture: print the statistics grouped by gesture
if this argument is specified; otherwise, by validator.
@param detailed: print statistics for all derived validators if True;
otherwise, print the merged statistics, e.g.,
both StationaryFingerValidator and StationaryTapValidator
are merged into StationaryValidator.
"""
# Print the complete title which looks like:
# <title_str> <fw1> <fw2> ... <description>
fws = self.slog.fws
num_fws = len(fws)
fws_str_max_width = max(map(len, fws))
fws_str_width = max(fws_str_max_width + 1, 10)
table_name = ('Detailed table (for debugging)' if detailed else
'Summary table')
title_str = ('Metrics statistics by gesture: ' + gesture if gesture else
'Metrics statistics by validator')
description_str = 'description (lower is better)'
fw_format = '{:>%d}' % fws_str_width
complete_title = ('{:<37}: '.format(title_str) +
(fw_format * num_fws).format(*fws) +
' {:<40}'.format(description_str))
print '\n' * 2
print table_name
print complete_title
print '-' * len(complete_title)
# Print the metric name and the metric stats values of every firmwares
name_format = ' ' * 6 + '{:<31}:'
description_format = ' {:<40}'
float_format = '{:>%d.2f}' % fws_str_width
blank_format = '{:>%d}' % fws_str_width
validators = (self.slog.validators if detailed else
self._get_merged_validators())
for validator in validators:
fw_stats_values = defaultdict(dict)
for fw in fws:
result = self.slog.get_result(fw=fw, gesture=gesture,
validators=validator)
stat_metrics = result.stat_metrics
for metric_name in stat_metrics.metrics_values:
fw_stats_values[metric_name][fw] = \
stat_metrics.stats_values[metric_name]
fw_stats_values_printed = False
for metric_name, fw_values_dict in sorted(fw_stats_values.items()):
values = []
values_format = ''
for fw in fws:
value = fw_values_dict.get(fw, '')
values.append(value)
values_format += float_format if value else blank_format
# The metrics of some special validators will not be shown
# unless the display_all_stats flag is True or any stats values
# are non-zero.
if (validator not in conf.validators_hidden_when_no_failures or
self.display_metrics.display_all_stats or any(values)):
if not fw_stats_values_printed:
fw_stats_values_printed = True
if isinstance(validator, list):
print (' ' + ' {}' * len(validator)).format(*validator)
else:
print ' ' + validator
disp_name = self._get_metric_name_for_display(metric_name)
print name_format.format(disp_name),
print values_format.format(*values),
print description_format.format(
stat_metrics.metrics_props[metric_name].description)
def _print_raw_metrics_values(self):
"""Print the raw metrics values."""
# The subkey() below extracts (gesture, variation, round) from
# metric.key which is (fw, round, gesture, variation, validator)
subkey = lambda key: (key[2], key[3], key[1])
# The sum_len() below is used to calculate the sum of the length
# of the elements in the subkey.
sum_len = lambda lst: sum([len(str(l)) if l else 0 for l in lst])
mnprops = firmware_log.MetricNameProps()
print '\n\nRaw metrics values'
print '-' * 80
for fw in self.slog.fws:
print '\n', fw
for validator in self.slog.validators:
result = self.slog.get_result(fw=fw, validators=validator)
metrics_dict = result.stat_metrics.metrics_dict
if metrics_dict:
print '\n' + ' ' * 3 + validator
for metric_name, metrics in sorted(metrics_dict.items()):
disp_name = self._get_metric_name_for_display(metric_name)
print ' ' * 6 + disp_name
metric_note = mnprops.metrics_props[metric_name].note
if metric_note:
msg = '** Note: value below represents '
print ' ' * 9 + msg + metric_note
# Make a metric value list sorted by
# (gesture, variation, round)
value_list = sorted([(subkey(metric.key), metric.value)
for metric in metrics])
max_len = max([sum_len(value[0]) for value in value_list])
template_prefix = ' ' * 9 + '{:<%d}: ' % (max_len + 5)
for (gesture, variation, round), value in value_list:
template = template_prefix + (
'{}' if isinstance(value, tuple) else '{:.2f}')
gvr_str = '%s.%s (%s)' % (gesture, variation, round)
print template.format(gvr_str, value)
def _print_final_weighted_averages(self):
"""Print the final weighted averages of all validators."""
title_str = 'Test Summary (final weighted averages)'
print '\n\n' + title_str
print '-' * len(title_str)
weighted_average = self.slog.get_final_weighted_average()
for fw in self.slog.fws:
print '%s: %4.3f' % (fw, weighted_average[fw])
def print_result_summary(self):
"""Print the summary of the test results."""
print self.slog.test_version
if self.display_metrics:
self._print_statistics_of_metrics(detailed=False)
self._print_statistics_of_metrics(detailed=True)
if self.display_metrics.display_raw_values:
self._print_raw_metrics_values()
if self.display_scores:
self._print_result_stats_by_gesture()
self._print_result_stats_by_validator()
self._print_final_weighted_averages()
def _usage_and_exit():
"""Print the usage message and exit."""
prog = sys.argv[0]
print 'Usage: $ python %s [options]\n' % prog
print 'options:'
print ' -D, --%s' % OPTIONS.DEBUG
print ' enable debug flag'
print ' -d, --%s <directory>' % OPTIONS.DIR
print ' specify which log directory to derive the summary'
print ' -h, --%s' % OPTIONS.HELP
print ' show this help'
print ' -i, --%s' % OPTIONS.INDIVIDUAL
print ' Calculate statistics of every individual round separately'
print ' -m, --%s <verbose_level>' % OPTIONS.METRICS
print ' display the summary metrics.'
print ' verbose_level:'
print ' 0: hide some metrics statistics if they passed'
print ' 1: display all metrics statistics'
print ' 2: display all metrics statistics and ' \
'the detailed raw metrics values (default)'
print ' -s, --%s' % OPTIONS.SCORES
print ' display the scores (0.0 ~ 1.0)'
print
print 'Examples:'
print ' Specify the log root directory.'
print ' $ python %s -d /tmp' % prog
print ' Hide some metrics statistics.'
print ' $ python %s -m 0' % prog
print ' Display all metrics statistics.'
print ' $ python %s -m 1' % prog
print ' Display all metrics statistics with detailed raw metrics values.'
print ' $ python %s # or' % prog
print ' $ python %s -m 2' % prog
sys.exit(1)
def _parsing_error(msg):
"""Print the usage and exit when encountering parsing error."""
print 'Error: %s' % msg
_usage_and_exit()
def _parse_options():
"""Parse the options."""
# Set the default values of options.
options = {OPTIONS.DEBUG: False,
OPTIONS.DIR: log_root_dir,
OPTIONS.INDIVIDUAL: False,
OPTIONS.METRICS: OptionsDisplayMetrics(None),
OPTIONS.SCORES: False,
}
try:
short_opt = 'Dd:him:s'
long_opt = [OPTIONS.DEBUG,
OPTIONS.DIR + '=',
OPTIONS.HELP,
OPTIONS.INDIVIDUAL,
OPTIONS.METRICS + '=',
OPTIONS.SCORES,
]
opts, args = getopt.getopt(sys.argv[1:], short_opt, long_opt)
except getopt.GetoptError, err:
_parsing_error(str(err))
for opt, arg in opts:
if opt in ('-h', '--%s' % OPTIONS.HELP):
_usage_and_exit()
elif opt in ('-D', '--%s' % OPTIONS.DEBUG):
options[OPTIONS.DEBUG] = True
elif opt in ('-d', '--%s' % OPTIONS.DIR):
options[OPTIONS.DIR] = arg
if not os.path.isdir(arg):
print 'Error: the log directory %s does not exist.' % arg
_usage_and_exit()
elif opt in ('-i', '--%s' % OPTIONS.INDIVIDUAL):
options[OPTIONS.INDIVIDUAL] = True
elif opt in ('-m', '--%s' % OPTIONS.METRICS):
options[OPTIONS.METRICS] = OptionsDisplayMetrics(arg)
elif opt in ('-s', '--%s' % OPTIONS.SCORES):
options[OPTIONS.SCORES] = True
else:
msg = 'This option "%s" is not supported.' % opt
_parsing_error(opt)
return options
if __name__ == '__main__':
options = _parse_options()
summary = FirmwareSummary(options[OPTIONS.DIR],
display_metrics=options[OPTIONS.METRICS],
individual_round_flag=options[OPTIONS.INDIVIDUAL],
display_scores=options[OPTIONS.SCORES],
debug_flag=options[OPTIONS.DEBUG])
summary.print_result_summary()