blob: d5376dd3cdf212cdf6cf611d5b57ab262e333f9d [file] [log] [blame]
#!/usr/bin/python
#
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unit tests for server/cros/dynamic_suite/telemetry_runner.py."""
import mox
import common
from autotest_lib.server.cros import telemetry_runner
class TelemetryResultTest(mox.MoxTestBase):
"""Unit tests for telemetry_runner.TelemetryResult."""
SAMPLE_RESULT_LINES = (
'RESULT average_commit_time_by_url: http___www.ebay.com= 8.86528 ms\n'
'RESULT CodeLoad: CodeLoad= 6343 score (bigger is better)\n'
'RESULT ai-astar: ai-astar= '
'[614,527,523,471,530,523,577,625,614,538] ms\n'
'RESULT graph_name: test_name= {3.14, 0.98} units')
EXPECTED_PERF_DATA = [
{'graph': 'average_commit_time_by_url', 'trace': 'http___www.ebay.com',
'units': 'ms', 'value': 8.86528},
{'graph': 'CodeLoad', 'trace': 'CodeLoad',
'units': 'score__bigger_is_better_', 'value': 6343},
{'graph': 'ai-astar', 'trace': 'ai-astar',
'units': 'ms', 'value': 554.2},
{'graph': 'graph_name', 'trace': 'test_name',
'units': 'units', 'value': 3.14}]
def testEmptyStdout(self):
"""Test when the test exits with 0 but there is no output."""
result = telemetry_runner.TelemetryResult()
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.FAILED_STATUS)
def testOnlyResultLines(self):
"""Test when the stdout is only Result lines."""
result = telemetry_runner.TelemetryResult(
exit_code=0, stdout=self.SAMPLE_RESULT_LINES)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testOnlyResultLinesWithWarnings(self):
"""Test when the stderr has Warnings."""
stdout = self.SAMPLE_RESULT_LINES
stderr = ('WARNING: Page failed to load http://www.facebook.com\n'
'WARNING: Page failed to load http://www.yahoo.com\n')
result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout,
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.WARNING_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testOnlyResultLinesWithWarningsAndTraceback(self):
"""Test when the stderr has Warnings and Traceback."""
stdout = self.SAMPLE_RESULT_LINES
stderr = ('WARNING: Page failed to load http://www.facebook.com\n'
'WARNING: Page failed to load http://www.yahoo.com\n'
'Traceback (most recent call last):\n'
'File "../../utils/unittest_suite.py", line 238, in '
'<module>\n'
'main()')
result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout,
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.FAILED_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoBeforeResultLines(self):
"""Test when there is info before the Result lines."""
stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' +
self.SAMPLE_RESULT_LINES)
stderr = 'WARNING: Page failed to load http://www.facebook.com\n'
result = telemetry_runner.TelemetryResult(exit_code=1, stdout=stdout,
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.WARNING_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoAfterResultLines(self):
"""Test when there is info after the Result lines."""
stdout = (self.SAMPLE_RESULT_LINES + '\n'
'stderr:WARNING:root:Found (system), but you do not have '
'a DISPLAY environment set.\n\n'
'04/16 12:51:23.312 DEBUG|telemetry_:0139|')
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoBeforeAndAfterResultLines(self):
"""Test when there is info before and after the Result lines."""
stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' +
self.SAMPLE_RESULT_LINES + '\n'
'stderr:WARNING:root:Found (system), but you do not have '
'a DISPLAY environment set.\n\n'
'04/16 12:51:23.312 DEBUG|telemetry_:0139|')
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testNoResultLines(self):
"""Test when Result lines are missing from stdout."""
stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n'
'stderr:WARNING:root:Found (system), but you do not have '
'a DISPLAY environment set.\n\n'
'04/16 12:51:23.312 DEBUG|telemetry_:0139|')
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals([], result.perf_data)
def testBadCharactersInResultStringComponents(self):
"""Test bad characters are cleaned up in RESULT string components."""
stdout = (
'RESULT average_commit_time_by_url!: '
'http___www.^^ebay.com= 8.86528 ms\n'
'RESULT CodeLoad*: CodeLoad= 6343 score\n'
'RESULT ai-astar: ai-astar= '
'[614,527,523,471,530,523,577,625,614,538] ~~ms\n'
'RESULT !!graph_name: &&test_name= {3.14, 0.98} units!')
expected_perf_data = [
{'graph': 'average_commit_time_by_url_',
'trace': 'http___www.__ebay.com',
'units': 'ms', 'value': 8.86528},
{'graph': 'CodeLoad_', 'trace': 'CodeLoad',
'units': 'score', 'value': 6343},
{'graph': 'ai-astar', 'trace': 'ai-astar',
'units': '__ms', 'value': 554.2},
{'graph': '__graph_name', 'trace': '__test_name',
'units': 'units_', 'value': 3.14}]
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(expected_perf_data, result.perf_data)
def testCleanupUnitsString(self):
"""Test that special characters in units strings are cleaned up."""
result = telemetry_runner.TelemetryResult()
self.assertEquals(result._cleanup_units_string('score/unit'),
'score_per_unit')
self.assertEquals(result._cleanup_units_string('score / unit'),
'score__per__unit')
self.assertEquals(result._cleanup_units_string('%'),
'percent')
self.assertEquals(result._cleanup_units_string('unit%'),
'unitpercent')
self.assertEquals(result._cleanup_units_string('^^un!ts##'),
'__un_ts__')
self.assertEquals(result._cleanup_units_string('^^un!ts##/time %'),
'__un_ts___per_time_percent')