[autotest] telemetry_Benchmarks upload perf data using output_perf_value
telemetry_Benchmarks now upload perf data using output_perf_value.
It will not write to the keyval file anymore.
After the change lands, we should stop the scripts extract_perf.py
and generate_perf_graphs that are currently kicked off
by cron to upload the perf data.
Also makes output_perf_value accept empty unit.
CQ-DEPEND=I0b746f4ab3162eb7ba4aefa12294b7a60e326513
TEST=1)Locally kicked off the tests from the AFE,
confirmed that the perf data was uploaded to the testing
perf dashboard.
2)test_unittest.py telemetry_runner_unittest.py
BUG=chromium:280634
Change-Id: Icf6dca08cab0268b41f8359eb73084dfee3e4b9d
Reviewed-on: https://chromium-review.googlesource.com/176746
Reviewed-by: Fang Deng <fdeng@chromium.org>
Tested-by: Fang Deng <fdeng@chromium.org>
Commit-Queue: Fang Deng <fdeng@chromium.org>
diff --git a/client/common_lib/test.py b/client/common_lib/test.py
index 9f6d20d..31e4cbc 100644
--- a/client/common_lib/test.py
+++ b/client/common_lib/test.py
@@ -86,7 +86,7 @@
return new_dict
- def output_perf_value(self, description, value, units,
+ def output_perf_value(self, description, value, units=None,
higher_is_better=True, graph=None):
"""
Records a measured performance value in an output file.
@@ -124,10 +124,11 @@
raise ValueError('The units must be at most 32 characters.')
string_regex = re.compile(r'^[-\.\w]+$')
if (not string_regex.search(description) or
- not string_regex.search(units)):
+ (units and not string_regex.search(units))):
raise ValueError('Invalid description or units string. May only '
'contain letters, numbers, periods, dashes, and '
- 'underscores.')
+ 'underscores. description: %s, units: %s' %
+ (description, units))
entry = {
'description': description,
diff --git a/server/cros/telemetry_runner.py b/server/cros/telemetry_runner.py
index ddffd80..82429af 100644
--- a/server/cros/telemetry_runner.py
+++ b/server/cros/telemetry_runner.py
@@ -30,10 +30,6 @@
'(?P<VALUE>[\{\[]?[-\d\., ]+[\}\]]?)('
' ?(?P<UNITS>.+))?')
-# Constants pertaining to perf keys generated from Telemetry test results.
-PERF_KEY_TELEMETRY_PREFIX = 'TELEMETRY'
-PERF_KEY_DELIMITER = '--'
-
class TelemetryResult(object):
"""Class to represent the results of a telemetry run.
@@ -55,7 +51,10 @@
else:
self.status = FAILED_STATUS
- self.perf_keyvals = {}
+ # A list of perf values, e.g.
+ # [{'graph': 'graphA', 'trace': 'page_load_time',
+ # 'units': 'secs', 'value':0.5}, ...]
+ self.perf_data = []
self._stdout = stdout
self._stderr = stderr
self.output = '\n'.join([stdout, stderr])
@@ -172,13 +171,16 @@
# In this example, we'd get 34.2.
value_list = [float(x) for x in value.strip('{},').split(',')]
value = value_list[0] # Position 0 is the value.
+ elif re.search('^\d+$', value):
+ value = int(value)
+ else:
+ value = float(value)
- perf_key = PERF_KEY_DELIMITER.join(
- [PERF_KEY_TELEMETRY_PREFIX, graph_name, trace_name, units])
- self.perf_keyvals[perf_key] = str(value)
+ self.perf_data.append({'graph':graph_name, 'trace': trace_name,
+ 'units': units, 'value': value})
pp = pprint.PrettyPrinter(indent=2)
- logging.debug('Perf Keyvals: %s', pp.pformat(self.perf_keyvals))
+ logging.debug('Perf values: %s', pp.pformat(self.perf_data))
if self.status is SUCCESS_STATUS:
return
@@ -312,14 +314,38 @@
return self._run_test(TELEMETRY_RUN_CROS_TESTS_SCRIPT, test)
- def run_telemetry_benchmark(self, benchmark, keyval_writer=None):
+ @staticmethod
+ def _output_perf_value(perf_value_writer, perf_data):
+ """Output perf values to result dir.
+
+ The perf values will be output to the result dir and
+ be subsequently uploaded to perf dashboard.
+
+ @param perf_value_writer: Should be an instance with the function
+ output_perf_value(), if None, no perf value
+ will be written. Typically this will be the
+ job object from an autotest test.
+ @param perf_data: A list of perf values, each value is
+ a dictionary that looks like
+ {'graph':'GraphA', 'trace':'metric1',
+ 'units':'secs', 'value':0.5}
+ """
+ for perf_value in perf_data:
+ perf_value_writer.output_perf_value(
+ description=perf_value['trace'],
+ value=perf_value['value'],
+ units=perf_value['units'],
+ graph=perf_value['graph'])
+
+
+ def run_telemetry_benchmark(self, benchmark, perf_value_writer=None):
"""Runs a telemetry benchmark on a dut.
@param benchmark: Benchmark we want to run.
- @param keyval_writer: Should be a instance with the function
- write_perf_keyval(), if None, no keyvals will be
- written. Typically this will be the job object
- from a autotest test.
+ @param perf_value_writer: Should be an instance with the function
+ output_perf_value(), if None, no perf value
+ will be written. Typically this will be the
+ job object from an autotest test.
@returns A TelemetryResult Instance with the results of this telemetry
execution.
@@ -330,8 +356,8 @@
result = self._run_telemetry(telemetry_script, benchmark)
result.parse_benchmark_results()
- if keyval_writer:
- keyval_writer.write_perf_keyval(result.perf_keyvals)
+ if perf_value_writer:
+ self._output_perf_value(perf_value_writer, result.perf_data)
if result.status is WARNING_STATUS:
raise error.TestWarn('Telemetry Benchmark: %s'
diff --git a/server/cros/telemetry_runner_unittest.py b/server/cros/telemetry_runner_unittest.py
index 2d1c679..d5376dd 100644
--- a/server/cros/telemetry_runner_unittest.py
+++ b/server/cros/telemetry_runner_unittest.py
@@ -21,12 +21,16 @@
'[614,527,523,471,530,523,577,625,614,538] ms\n'
'RESULT graph_name: test_name= {3.14, 0.98} units')
- EXPECTED_KEYVALS = {
- 'TELEMETRY--average_commit_time_by_url--http___www.ebay.com--ms':
- '8.86528',
- 'TELEMETRY--CodeLoad--CodeLoad--score__bigger_is_better_': '6343',
- 'TELEMETRY--ai-astar--ai-astar--ms': '554.2',
- 'TELEMETRY--graph_name--test_name--units': '3.14'}
+ EXPECTED_PERF_DATA = [
+ {'graph': 'average_commit_time_by_url', 'trace': 'http___www.ebay.com',
+ 'units': 'ms', 'value': 8.86528},
+ {'graph': 'CodeLoad', 'trace': 'CodeLoad',
+ 'units': 'score__bigger_is_better_', 'value': 6343},
+ {'graph': 'ai-astar', 'trace': 'ai-astar',
+ 'units': 'ms', 'value': 554.2},
+ {'graph': 'graph_name', 'trace': 'test_name',
+ 'units': 'units', 'value': 3.14}]
+
def testEmptyStdout(self):
"""Test when the test exits with 0 but there is no output."""
@@ -41,7 +45,7 @@
exit_code=0, stdout=self.SAMPLE_RESULT_LINES)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
- self.assertEquals(self.EXPECTED_KEYVALS, result.perf_keyvals)
+ self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testOnlyResultLinesWithWarnings(self):
@@ -54,7 +58,7 @@
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.WARNING_STATUS)
- self.assertEquals(self.EXPECTED_KEYVALS, result.perf_keyvals)
+ self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testOnlyResultLinesWithWarningsAndTraceback(self):
@@ -71,7 +75,7 @@
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.FAILED_STATUS)
- self.assertEquals(self.EXPECTED_KEYVALS, result.perf_keyvals)
+ self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoBeforeResultLines(self):
@@ -84,7 +88,7 @@
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.WARNING_STATUS)
- self.assertEquals(self.EXPECTED_KEYVALS, result.perf_keyvals)
+ self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoAfterResultLines(self):
@@ -98,7 +102,7 @@
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
- self.assertEquals(self.EXPECTED_KEYVALS, result.perf_keyvals)
+ self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoBeforeAndAfterResultLines(self):
@@ -113,7 +117,7 @@
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
- self.assertEquals(self.EXPECTED_KEYVALS, result.perf_keyvals)
+ self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testNoResultLines(self):
@@ -127,7 +131,7 @@
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
- self.assertEquals({}, result.perf_keyvals)
+ self.assertEquals([], result.perf_data)
def testBadCharactersInResultStringComponents(self):
@@ -139,18 +143,22 @@
'RESULT ai-astar: ai-astar= '
'[614,527,523,471,530,523,577,625,614,538] ~~ms\n'
'RESULT !!graph_name: &&test_name= {3.14, 0.98} units!')
- expected_keyvals = {
- 'TELEMETRY--average_commit_time_by_url_--http___www.__ebay.com--ms':
- '8.86528',
- 'TELEMETRY--CodeLoad_--CodeLoad--score': '6343',
- 'TELEMETRY--ai-astar--ai-astar--__ms': '554.2',
- 'TELEMETRY--__graph_name--__test_name--units_': '3.14'}
+ expected_perf_data = [
+ {'graph': 'average_commit_time_by_url_',
+ 'trace': 'http___www.__ebay.com',
+ 'units': 'ms', 'value': 8.86528},
+ {'graph': 'CodeLoad_', 'trace': 'CodeLoad',
+ 'units': 'score', 'value': 6343},
+ {'graph': 'ai-astar', 'trace': 'ai-astar',
+ 'units': '__ms', 'value': 554.2},
+ {'graph': '__graph_name', 'trace': '__test_name',
+ 'units': 'units_', 'value': 3.14}]
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
- self.assertEquals(expected_keyvals, result.perf_keyvals)
+ self.assertEquals(expected_perf_data, result.perf_data)
def testCleanupUnitsString(self):
diff --git a/server/site_tests/telemetry_Benchmarks/telemetry_Benchmarks.py b/server/site_tests/telemetry_Benchmarks/telemetry_Benchmarks.py
index 896d1ce..3c31793 100644
--- a/server/site_tests/telemetry_Benchmarks/telemetry_Benchmarks.py
+++ b/server/site_tests/telemetry_Benchmarks/telemetry_Benchmarks.py
@@ -17,4 +17,4 @@
@param benchmark: benchmark we want to run.
"""
telemetry = telemetry_runner.TelemetryRunner(host)
- telemetry.run_telemetry_benchmark(benchmark, keyval_writer=self)
+ telemetry.run_telemetry_benchmark(benchmark, perf_value_writer=self)
diff --git a/tko/perf_upload/perf_dashboard_config.json b/tko/perf_upload/perf_dashboard_config.json
index 337e3db..212e05b 100644
--- a/tko/perf_upload/perf_dashboard_config.json
+++ b/tko/perf_upload/perf_dashboard_config.json
@@ -18,5 +18,45 @@
{
"autotest_name": "video_VideoDecodeMemeoryUsage",
"master_name": "ChromeOSVideo"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.dromaeo.domcoreattr",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "dromaeo.domcoreattr"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.dromaeo.domcoremodify",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "dromaeo.domcoremodify"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.dromaeo.domcorequery",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "dromaeo.domcorequery"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.dromaeo.domcoretraverse",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "dromaeo.domcoretraverse"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.media.tough_media_cases",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "media.tough_media_cases"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.memory.top_25",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "memory.top_25"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.robohornet_pro",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "robohornet_pro"
+ },
+ {
+ "autotest_name": "telemetry_Benchmarks.smoothness.top_25",
+ "master_name": "ChromiumPerf",
+ "dashboard_test_name": "smoothness.top_25"
}
]
