Script to collect test statuses and results for browser_tests.
BUG=chromium:356857
TEST=Run using 'tests' file at go/btr-data
Change-Id: I18c7ed7fe37ec6b4d54938591e76f3cc63a39c4e
Reviewed-on: https://chromium-review.googlesource.com/191749
Tested-by: Scott Cunningham <scunningham@chromium.org>
Reviewed-by: Matthew Montgomery <mmontgomery@chromium.org>
Reviewed-by: Scott Cunningham <scunningham@chromium.org>
Commit-Queue: Scott Cunningham <scunningham@chromium.org>
diff --git a/provingground/browsertest_status.py b/provingground/browsertest_status.py
new file mode 100755
index 0000000..998e1ca
--- /dev/null
+++ b/provingground/browsertest_status.py
@@ -0,0 +1,485 @@
+#!/usr/bin/python
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Script to display latest test run status and test results for a user-
+# specified list of browser_tests. The names of the desired brower_tests are
+# read from a file located (by default) in the same directory as this script.
+#
+# Latest test run status is fetched from the build.chromium.org build server,
+# read from the 'stdio' text file located at:
+# http://build.chromium.org/p/chromium.chromiumos/builders/TR_BUILDER/
+# builds/BUILD_NUMBER/steps/browser_tests/logs/stdio/text
+#
+# Recent test results are fetched from the test-results.appspot.com server,
+# read from the results.json file located at:
+# https://test-results.appspot.com/testfile?master=ChromiumChromiumOS&
+# builder=Linux ChromiumOS Tests (dbg)(2)&testtype=browser_tests&
+# name=results.json
+#
+
+"""Script to report test status and results of user-specified browsertests."""
+__author__ = ('scunningham@google.com Scott Cunningham')
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib2
+
+# Chromium builder url parameter defaults.
+_BUILD_HOST = 'build.chromium.org'
+_BUILD_PROJECT = 'chromium.chromiumos'
+_BUILD_NUMBER = '-2'
+
+# TestResults server url parameter defaults.
+_TR_HOST = 'test-results.appspot.com' # URI to TestResults server.
+_TR_MASTER = 'ChromiumChromiumOS' # Test-results build master repository.
+_TR_BUILDER = 'Linux ChromiumOS Tests (dbg)(2)' # TestResults builder name.
+
+# Input file and report directory parameter defaults.
+_TESTS_FILE = './tests' # Path to the file that contains the tests names.
+_REPORT_DIR = os.getcwd() # Path to the directory to store the results report.
+
+# Contents of test result types.
+_RESULT_TYPES = {
+ 'A': 'AUDIO',
+ 'C': 'CRASH',
+ 'F': 'TEXT',
+ 'I': 'IMAGE',
+ 'L': 'FLAKY',
+ 'O': 'MISSING',
+ 'N': 'NO DATA',
+ 'Q': 'FAIL',
+ 'P': 'PASS',
+ 'T': 'TIMEOUT',
+ 'Y': 'NOTRUN',
+ 'X': 'SKIP',
+ 'Z': 'IMAGE+TEXT'
+ }
+
+# Report header result types.
+_NOTRUN = 'NotRun'
+_FAILED = 'Failed'
+_PASSED = 'Passed'
+_MISSING = 'Missing'
+
+
+def _FindLatestCompletedBuildNumber(build_number):
+ """Find the latest completed build number from the given build number.
+
+ Check if build with build_number completed successfully. If not, iteratively
+ check earlier versions, until a successfully completed build is found.
+
+ Args:
+ build_number: build number to start search.
+
+ Returns:
+ Build number of most recent successfully completed build.
+ """
+ # TODO(scunningham): Implement a real find function in later version.
+ return build_number
+
+
+def _GetStdioLogUrl(builder, build_num):
+ """Get url to Stdio Log file from builder for build number.
+
+ Args:
+ builder: Builder name.
+ build_num: Build number.
+
+ Returns:
+ Url to the Stdio log text file.
+ """
+ # Generate percent-encoded build url.
+ build_url = (('http://%s/p/%s/json/builders/%s/builds?'
+ 'select=%s/steps/browser_tests/') %
+ (urllib2.quote(_BUILD_HOST), urllib2.quote(_BUILD_PROJECT),
+ urllib2.quote(builder), urllib2.quote(build_num)))
+
+ # Fetch build status file from build url.
+ print '\nFetching build status file from: %s' % build_url
+ url = urllib2.urlopen(build_url)
+ build_status_file = url.read()
+ url.close()
+
+ # Convert json build status file to dictionary.
+ build_status_dict = json.loads(build_status_file)
+
+ # Extract stdio log url from build status dictionary.
+ print '\n Contents of build status file: %s' % build_status_dict
+ return build_status_dict[build_num]['steps']['browser_tests']['logs'][0][1]
+
+
+def _GetStdioLogTestsDict(stdio_url):
+ """Get Stdio Log browser_tests from the given url.
+
+ Args:
+ stdio_url: url to stdio log browser_tests file.
+
+ Returns:
+ Dictionary of tests from stdio log browser_tests text file.
+ """
+ # Fetch builder stdio log text file from url.
+ stdio_text_url = stdio_url+'/text'
+ print '\nFetching builder stdio log file from: %s' % stdio_text_url
+ stdio_text_file = urllib2.urlopen(stdio_text_url)
+
+ # Extract test lines from stdio text file.
+ pattern = r'\[\d+/\d+\] '
+ test_lines = [line for line in stdio_text_file if re.match(pattern, line)]
+ stdio_text_file.close()
+ print ' Total run tests extracted: %s' % len(test_lines)
+ if test_lines:
+ print ' Last run test line: "%s"' % test_lines[-1].strip()
+
+ # Extract test data and pack into stdio tests dictionary.
+ stdio_tests_dict = {}
+ pattern = r'(\[\d*/\d*\]) (.*?) (\(.*\))'
+ for i, line in enumerate(test_lines):
+ m = re.match(pattern, line)
+ if m:
+ stdio_tests_dict[m.group(2)] = (m.group(1), m.group(3))
+ else:
+ print 'Error: Invalid test line %s) %s' % (i, line.strip())
+
+ return stdio_tests_dict
+
+
+def _GetUserSpecifiedTests(tests_file):
+ """Get list of user-specified tests from tests file.
+
+ File must be a text file, formatted with one line per test. Leading
+ and trailing spaces and blanklines are stripped from the test list.
+
+ Args:
+ tests_file: Path and name of tests file.
+
+ Returns:
+ List of user tests read from lines in the tests_file.
+ """
+ print '\nFetching user-specified tests from: %s' % tests_file
+ content = open(tests_file, 'r').read().strip()
+ return [line.strip() for line in content.splitlines() if line.strip()]
+
+
+def _RunAndNotrunTests(stdio_tests, user_tests):
+ """Return lists of individual test instances of user-specified tests.
+
+ The first list is of test instances present in the stdio tests list.
+ Presence indicates that the test instance was run on the build. Second list
+ is tests that are absent from the stdio tests list. Absence means that no
+ instance of the test was run on the build.
+
+ Note that there can be multiple instances of a user-specified test run on a
+ build if a) The test belongs to test group, and b) the test was run with
+ multiple test data values. This function uses a regex to search for multiple
+ instances of tests that match the user-specifed test name.
+
+ Args:
+ stdio_tests: List of test instances run on build.
+ user_tests: List of test names specified by user.
+
+ Returns:
+ 1) run_tests: list of test instances of user tests run on build.
+ 2) notrun_tests: list of user tests not run on build.
+ """
+ run_user_tests = []
+ notrun_user_tests = []
+ for user_test in user_tests:
+ pattern = r'(.*/)?%s(/\d*)?$' % user_test
+ found_run_test = False
+ for stdio_test in stdio_tests:
+ if re.search(pattern, stdio_test):
+ found_run_test = True
+ run_user_tests.append(stdio_test)
+ if not found_run_test:
+ notrun_user_tests.append(user_test)
+ print ' Run instances of user tests: %s' % len(run_user_tests)
+ print ' Not run user tests: %s\n' % len(notrun_user_tests)
+ return run_user_tests, notrun_user_tests
+
+
+def _GetResultsDict(master, builder):
+ """Get results dictionary from builder results.json file.
+
+ The results dictionary contains information about recent tests run,
+ test results, build numbers, chrome revision numbers, etc, for the
+ last 500 builds on the specified builder.
+
+ Args:
+ master: Master repo (e.g., 'ChromiumChromiumOS')
+ builder: Builder name (e.g., 'Linux ChromiumOS Tests (dbg)(2)')
+
+ Returns:
+ Dictionary of builder results.
+ """
+ # Generate percent-encoded builder results url.
+ results_url = (('https://%s/testfile?master=%s&builder=%s'
+ '&testtype=browser_tests&name=results.json') %
+ (urllib2.quote(_TR_HOST), urllib2.quote(master),
+ urllib2.quote(builder)))
+
+ # Fetch results file from builder results url.
+ print 'Fetching builder results file from %s' % results_url
+ url = urllib2.urlopen(results_url)
+ results_json = url.read()
+ url.close()
+
+ # Convert json results to native Python object.
+ builder_results_dict = json.loads(results_json)
+ return builder_results_dict[builder]
+
+
+def _CreateTestsResultsDictionary(builder_tests_dict):
+ """Create dictionary of all tests+results from builder tests dictionary.
+
+ Parse individual tests and results from the builder tests dictionary,
+ and place into a flattened tests results dictionary. Most tests are
+ standalone, and keyed by thier test name. Some tests belong to a
+ 'testinstance' group, and are keyed by their testinstance name, the
+ test data value (e.g., '0', '1', '2'), and the test name.
+
+ For example, a standalone test:result is formatted thus:
+ "BookmarksTest.CommandOpensBookmarksTab": {
+ "results": [...]
+ "times": [...]
+ }
+
+ Tests grouped under a testinstance, are formatted thus:
+ "MediaStreamDevicesControllerBrowserTestInstance": {
+ "MediaStreamDevicesControllerBrowserTest.AudioCaptureAllowed": {
+ "1": {
+ "results": [...],
+ "times": [...]
+ }
+ "MediaStreamDevicesControllerBrowserTest.VideoCaptureAllowed": {
+ "0": {
+ "results": [...],
+ "times": [...]
+ }
+ }
+
+ Args:
+ builder_tests_dict: Dictionary of test groups & tests on builder.
+
+ Returns:
+ Dictionary of flattened tests and their results on builder.
+ """
+ tests_results_dict = {}
+ standalone = 0
+ group = 0
+ subtest = 0
+
+ for group_name in builder_tests_dict:
+ test_group = builder_tests_dict[group_name]
+ if '.' in group_name:
+ standalone += 1
+ test_result = test_group['results']
+ tests_results_dict[group_name] = test_result
+ else:
+ group += 1
+ for test_name in test_group.keys():
+ test_values = test_group[test_name]
+ for value in test_values.keys():
+ subtest += 1
+ test_result = test_values[value]['results']
+ long_test_name = '%s/%s/%s' % (group_name, test_name, value)
+ tests_results_dict[long_test_name] = test_result
+
+ print ' Number of standalone tests: %s' % standalone
+ print ' Number of instance tests (in %s groups): %s' % (group, subtest)
+ print ' Total tests results: %s\n' % len(tests_results_dict)
+
+ return tests_results_dict
+
+
+def _CreateUserTestsResultsDict(test_results_dict, run_user_tests):
+ """Create dictionary of tests:results for all user-specified tests.
+
+ If a user specified test is missing from builder test+results, then default
+ the test result to the code for missing test: 'O'.
+
+ Args:
+ test_results_dict: Builder tests+results dictionary
+ run_user_tests: List of run instances of user specified tests.
+
+ Returns:
+ Dictionary of tests to results for all user specified tests.
+ """
+ user_tests_results_dict = {}
+ # Iterate over run user-specified tests.
+ for test_name in run_user_tests:
+ if test_name in test_results_dict: # Test has results.
+ test_result = test_results_dict[test_name]
+ else:
+ test_result = [[999, u'O']] # Set result to missing.
+ user_tests_results_dict[test_name] = test_result
+
+ return user_tests_results_dict
+
+
+def _CreateResultOfTestsDict(user_tests_results_dict):
+ """Create dictionary of user tests keyed by result.
+
+ Args:
+ user_tests_results_dict: dictionary of user tests to results.
+
+ Returns:
+ Dictionary of results of tests.
+ """
+ # Test result type lists.
+ missing = ['O']
+ passed = ['P']
+ fails = [key for key in _RESULT_TYPES if key not in missing+passed]
+
+ failed_tests = []
+ passed_tests = []
+ missing_tests = []
+ for test in user_tests_results_dict:
+ result = user_tests_results_dict[test][0][1]
+ if result in fails:
+ failed_tests.append(test)
+ elif result in passed:
+ passed_tests.append(test)
+ elif result in missing:
+ missing_tests.append(test)
+ return {_FAILED: failed_tests,
+ _PASSED: passed_tests,
+ _MISSING: missing_tests}
+
+
+def _ReportTestsByResult(result_of_tests_dict, tr_dict, report_dir):
+ """Print and write report of tests, grouped by result type.
+
+ Args:
+ result_of_tests_dict: Dictionary of results for tests.
+ tr_dict: Dictionary of tests to results for builder.
+ report_dir: Directory where to save report.
+ """
+ # Test report result types and section headers.
+ report_results_headers = {
+ _NOTRUN: 'Test Status: Not Run',
+ _FAILED: 'Test Result: Fail or other Error',
+ _PASSED: 'Test Result: Passed recently',
+ _MISSING: 'Test Result: Passing long-term'
+ }
+ report_section_order = [_NOTRUN, _FAILED, _PASSED, _MISSING]
+
+ ofile = open(report_dir+'/report', 'w')
+ for result_type in report_section_order:
+ header = report_results_headers[result_type]
+ tests = result_of_tests_dict[result_type]
+ print '%s (%s)' % (header, len(tests))
+ ofile.write('%s (%s)\n' % (header, len(tests)))
+ for num, test_name in enumerate(sorted(tests)):
+ if test_name in tr_dict:
+ print ' %s) %s: %s' % (num+1, test_name, tr_dict[test_name][0:2])
+ ofile.write(' %s) %s: %s\n' % (num+1, test_name, tr_dict[test_name]))
+ else:
+ print ' %s) %s' % (num+1, test_name)
+ ofile.write(' %s) %s\n' % (num+1, test_name))
+ ofile.close()
+
+
+def main():
+ """Report test results of specified browsertests."""
+ parser = argparse.ArgumentParser(
+ description=('Report run status and test results for a user-specified '
+ 'list of browser_test tests.'),
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument('--tests_file', dest='tests_file', default=_TESTS_FILE,
+ help=('Specify tests path/file '
+ '(default is %s).' % _TESTS_FILE))
+ parser.add_argument('--report_dir', dest='report_dir', default=_REPORT_DIR,
+ help=('Specify path to report directory '
+ '(default is %s).' % _REPORT_DIR))
+ parser.add_argument('--build_num', dest='build_num', default=_BUILD_NUMBER,
+ help=('Specify builder build number '
+ '(default is %s).' % _BUILD_NUMBER))
+ parser.add_argument('--master', dest='master', default=_TR_MASTER,
+ help=('Specify build master repository '
+ '(default is %s).' % _TR_MASTER))
+ parser.add_argument('--builder', dest='builder', default=_TR_BUILDER,
+ help=('Specify test builder '
+ '(default is %s).' % _TR_BUILDER))
+ parser.add_argument('--print_types', dest='print_types',
+ action='store_true', help='Print test result types.')
+ arguments = parser.parse_args()
+
+ # Set parameters from command line arguments.
+ report_dir = arguments.report_dir
+ tests_file = arguments.tests_file
+ master = arguments.master
+ builder = arguments.builder
+ build_num = arguments.build_num
+ print_types = arguments.print_types
+
+ # Print map of test result types.
+ if print_types:
+ print 'Test result types:'
+ print json.dumps(_RESULT_TYPES, indent=4)
+ sys.exit()
+
+ # Ensure default or user defined tests file points to a real file.
+ if not os.path.isfile(tests_file):
+ print 'Error: Could not find tests file. Try passing in --tests_file.'
+ sys.exit(2)
+
+ # Ensure default or user-defined report folder points to a real dir.
+ if not os.path.exists(report_dir):
+ print ('Error: Could not find report directory. '
+ 'Try passing in --report_dir.')
+ sys.exit(2)
+
+ # Get builder stdio log test info for build number.
+ # Find latest completed build number.
+ if build_num == '0' or build_num == '-1':
+ print ('Error: Invalid build number: %s. '
+ 'Using %s instead.' % (build_num, _BUILD_NUMBER))
+ build_num = _BUILD_NUMBER
+ completed_build_num = _FindLatestCompletedBuildNumber(build_num)
+
+ # Get list of test instances run on builder for build number.
+ stdio_log_url = _GetStdioLogUrl(builder, completed_build_num)
+ stdio_tests_dict = _GetStdioLogTestsDict(stdio_log_url)
+
+ # Read list of user-specified tests from tests file.
+ user_tests = _GetUserSpecifiedTests(tests_file)
+ if not user_tests:
+ print 'Error: tests file is empty.'
+ sys.exit(2)
+ print ' Number of user tests: %s' % len(user_tests)
+
+ # Get list of instances of run and not run user tests.
+ run_user_tests, notrun_user_tests = (
+ _RunAndNotrunTests(stdio_tests_dict, user_tests))
+
+ # Get run user tests and results data from the specified builder.
+ if run_user_tests:
+ # Fetch builder results dictionary from test-results server.
+ builder_results_dict = _GetResultsDict(master, builder)
+ builder_tests_dict = builder_results_dict['tests']
+ else:
+ builder_tests_dict = {}
+
+ # Extract tests to results dictionary from builder tests dictionary.
+ tests_results_dict = _CreateTestsResultsDictionary(builder_tests_dict)
+
+ # Create dictionary of run user test instances and results.
+ user_tests_results_dict = (
+ _CreateUserTestsResultsDict(tests_results_dict, run_user_tests))
+
+ # Create dictionary of run tests that are failed, passed, and missing.
+ result_of_tests_dict = _CreateResultOfTestsDict(user_tests_results_dict)
+
+ # Add list of not run tests to the result of tests dictionary.
+ result_of_tests_dict[_NOTRUN] = notrun_user_tests
+
+ # Output tests by result type: notrun, failed, passed, and missing.
+ _ReportTestsByResult(result_of_tests_dict, tests_results_dict, report_dir)
+
+if __name__ == '__main__':
+ main()