| #!/usr/bin/python | |
| # | |
| # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """Unit tests for server/cros/dynamic_suite/telemetry_runner.py.""" | |
| import mox | |
| import common | |
| from autotest_lib.server.cros import telemetry_runner | |
| class TelemetryResultTest(mox.MoxTestBase): | |
| """Unit tests for telemetry_runner.TelemetryResult.""" | |
| SAMPLE_RESULT_LINES = ( | |
| 'RESULT average_commit_time_by_url: http___www.ebay.com= 8.86528 ms\n' | |
| 'RESULT CodeLoad: CodeLoad= 6343 score (bigger is better)\n' | |
| 'RESULT ai-astar: ai-astar= ' | |
| '[614,527,523,471,530,523,577,625,614,538] ms\n' | |
| 'RESULT graph_name: test_name= {3.14, 0.98} units') | |
| EXPECTED_PERF_DATA = [ | |
| {'graph': 'average_commit_time_by_url', 'trace': 'http___www.ebay.com', | |
| 'units': 'ms', 'value': 8.86528}, | |
| {'graph': 'CodeLoad', 'trace': 'CodeLoad', | |
| 'units': 'score__bigger_is_better_', 'value': 6343}, | |
| {'graph': 'ai-astar', 'trace': 'ai-astar', | |
| 'units': 'ms', 'value': 554.2}, | |
| {'graph': 'graph_name', 'trace': 'test_name', | |
| 'units': 'units', 'value': 3.14}] | |
| def testEmptyStdout(self): | |
| """Test when the test exits with 0 but there is no output.""" | |
| result = telemetry_runner.TelemetryResult() | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.FAILED_STATUS) | |
| def testOnlyResultLines(self): | |
| """Test when the stdout is only Result lines.""" | |
| result = telemetry_runner.TelemetryResult( | |
| exit_code=0, stdout=self.SAMPLE_RESULT_LINES) | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) | |
| self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) | |
| def testOnlyResultLinesWithWarnings(self): | |
| """Test when the stderr has Warnings.""" | |
| stdout = self.SAMPLE_RESULT_LINES | |
| stderr = ('WARNING: Page failed to load http://www.facebook.com\n' | |
| 'WARNING: Page failed to load http://www.yahoo.com\n') | |
| result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout, | |
| stderr=stderr) | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.WARNING_STATUS) | |
| self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) | |
| def testOnlyResultLinesWithWarningsAndTraceback(self): | |
| """Test when the stderr has Warnings and Traceback.""" | |
| stdout = self.SAMPLE_RESULT_LINES | |
| stderr = ('WARNING: Page failed to load http://www.facebook.com\n' | |
| 'WARNING: Page failed to load http://www.yahoo.com\n' | |
| 'Traceback (most recent call last):\n' | |
| 'File "../../utils/unittest_suite.py", line 238, in ' | |
| '<module>\n' | |
| 'main()') | |
| result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout, | |
| stderr=stderr) | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.FAILED_STATUS) | |
| self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) | |
| def testInfoBeforeResultLines(self): | |
| """Test when there is info before the Result lines.""" | |
| stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' + | |
| self.SAMPLE_RESULT_LINES) | |
| stderr = 'WARNING: Page failed to load http://www.facebook.com\n' | |
| result = telemetry_runner.TelemetryResult(exit_code=1, stdout=stdout, | |
| stderr=stderr) | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.WARNING_STATUS) | |
| self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) | |
| def testInfoAfterResultLines(self): | |
| """Test when there is info after the Result lines.""" | |
| stdout = (self.SAMPLE_RESULT_LINES + '\n' | |
| 'stderr:WARNING:root:Found (system), but you do not have ' | |
| 'a DISPLAY environment set.\n\n' | |
| '04/16 12:51:23.312 DEBUG|telemetry_:0139|') | |
| result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, | |
| stderr='') | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) | |
| self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) | |
| def testInfoBeforeAndAfterResultLines(self): | |
| """Test when there is info before and after the Result lines.""" | |
| stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' + | |
| self.SAMPLE_RESULT_LINES + '\n' | |
| 'stderr:WARNING:root:Found (system), but you do not have ' | |
| 'a DISPLAY environment set.\n\n' | |
| '04/16 12:51:23.312 DEBUG|telemetry_:0139|') | |
| result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, | |
| stderr='') | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) | |
| self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) | |
| def testNoResultLines(self): | |
| """Test when Result lines are missing from stdout.""" | |
| stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' | |
| 'stderr:WARNING:root:Found (system), but you do not have ' | |
| 'a DISPLAY environment set.\n\n' | |
| '04/16 12:51:23.312 DEBUG|telemetry_:0139|') | |
| result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, | |
| stderr='') | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) | |
| self.assertEquals([], result.perf_data) | |
| def testBadCharactersInResultStringComponents(self): | |
| """Test bad characters are cleaned up in RESULT string components.""" | |
| stdout = ( | |
| 'RESULT average_commit_time_by_url!: ' | |
| 'http___www.^^ebay.com= 8.86528 ms\n' | |
| 'RESULT CodeLoad*: CodeLoad= 6343 score\n' | |
| 'RESULT ai-astar: ai-astar= ' | |
| '[614,527,523,471,530,523,577,625,614,538] ~~ms\n' | |
| 'RESULT !!graph_name: &&test_name= {3.14, 0.98} units!') | |
| expected_perf_data = [ | |
| {'graph': 'average_commit_time_by_url_', | |
| 'trace': 'http___www.__ebay.com', | |
| 'units': 'ms', 'value': 8.86528}, | |
| {'graph': 'CodeLoad_', 'trace': 'CodeLoad', | |
| 'units': 'score', 'value': 6343}, | |
| {'graph': 'ai-astar', 'trace': 'ai-astar', | |
| 'units': '__ms', 'value': 554.2}, | |
| {'graph': '__graph_name', 'trace': '__test_name', | |
| 'units': 'units_', 'value': 3.14}] | |
| result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, | |
| stderr='') | |
| result.parse_benchmark_results() | |
| self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) | |
| self.assertEquals(expected_perf_data, result.perf_data) | |
| def testCleanupUnitsString(self): | |
| """Test that special characters in units strings are cleaned up.""" | |
| result = telemetry_runner.TelemetryResult() | |
| self.assertEquals(result._cleanup_units_string('score/unit'), | |
| 'score_per_unit') | |
| self.assertEquals(result._cleanup_units_string('score / unit'), | |
| 'score__per__unit') | |
| self.assertEquals(result._cleanup_units_string('%'), | |
| 'percent') | |
| self.assertEquals(result._cleanup_units_string('unit%'), | |
| 'unitpercent') | |
| self.assertEquals(result._cleanup_units_string('^^un!ts##'), | |
| '__un_ts__') | |
| self.assertEquals(result._cleanup_units_string('^^un!ts##/time %'), | |
| '__un_ts___per_time_percent') |