Script to collect test statuses and results for browser_tests.

BUG=chromium:356857
TEST=Run using 'tests' file at go/btr-data

Change-Id: I18c7ed7fe37ec6b4d54938591e76f3cc63a39c4e
Reviewed-on: https://chromium-review.googlesource.com/191749
Tested-by: Scott Cunningham <scunningham@chromium.org>
Reviewed-by: Matthew Montgomery <mmontgomery@chromium.org>
Reviewed-by: Scott Cunningham <scunningham@chromium.org>
Commit-Queue: Scott Cunningham <scunningham@chromium.org>
diff --git a/provingground/browsertest_status.py b/provingground/browsertest_status.py
new file mode 100755
index 0000000..998e1ca
--- /dev/null
+++ b/provingground/browsertest_status.py
@@ -0,0 +1,485 @@
+#!/usr/bin/python
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Script to display latest test run status and test results for a user-
+# specified list of browser_tests. The names of the desired brower_tests are
+# read from a file located (by default) in the same directory as this script.
+#
+# Latest test run status is fetched from the build.chromium.org build server,
+# read from the 'stdio' text file located at:
+#   http://build.chromium.org/p/chromium.chromiumos/builders/TR_BUILDER/
+#   builds/BUILD_NUMBER/steps/browser_tests/logs/stdio/text
+#
+# Recent test results are fetched from the test-results.appspot.com server,
+# read from the results.json file located at:
+#   https://test-results.appspot.com/testfile?master=ChromiumChromiumOS&
+#   builder=Linux ChromiumOS Tests (dbg)(2)&testtype=browser_tests&
+#   name=results.json
+#
+
+"""Script to report test status and results of user-specified browsertests."""
+__author__ = ('scunningham@google.com Scott Cunningham')
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib2
+
+# Chromium builder url parameter defaults.
+_BUILD_HOST = 'build.chromium.org'
+_BUILD_PROJECT = 'chromium.chromiumos'
+_BUILD_NUMBER = '-2'
+
+# TestResults server url parameter defaults.
+_TR_HOST = 'test-results.appspot.com'  # URI to TestResults server.
+_TR_MASTER = 'ChromiumChromiumOS'  # Test-results build master repository.
+_TR_BUILDER = 'Linux ChromiumOS Tests (dbg)(2)'  # TestResults builder name.
+
+# Input file and report directory parameter defaults.
+_TESTS_FILE = './tests'  # Path to the file that contains the tests names.
+_REPORT_DIR = os.getcwd()  # Path to the directory to store the results report.
+
+# Contents of test result types.
+_RESULT_TYPES = {
+    'A': 'AUDIO',
+    'C': 'CRASH',
+    'F': 'TEXT',
+    'I': 'IMAGE',
+    'L': 'FLAKY',
+    'O': 'MISSING',
+    'N': 'NO DATA',
+    'Q': 'FAIL',
+    'P': 'PASS',
+    'T': 'TIMEOUT',
+    'Y': 'NOTRUN',
+    'X': 'SKIP',
+    'Z': 'IMAGE+TEXT'
+    }
+
+# Report header result types.
+_NOTRUN = 'NotRun'
+_FAILED = 'Failed'
+_PASSED = 'Passed'
+_MISSING = 'Missing'
+
+
+def _FindLatestCompletedBuildNumber(build_number):
+  """Find the latest completed build number from the given build number.
+
+  Check if build with build_number completed successfully. If not, iteratively
+  check earlier versions, until a successfully completed build is found.
+
+  Args:
+    build_number: build number to start search.
+
+  Returns:
+    Build number of most recent successfully completed build.
+  """
+  # TODO(scunningham): Implement a real find function in later version.
+  return build_number
+
+
+def _GetStdioLogUrl(builder, build_num):
+  """Get url to Stdio Log file from builder for build number.
+
+  Args:
+    builder: Builder name.
+    build_num: Build number.
+
+  Returns:
+    Url to the Stdio log text file.
+  """
+  # Generate percent-encoded build url.
+  build_url = (('http://%s/p/%s/json/builders/%s/builds?'
+                'select=%s/steps/browser_tests/') %
+               (urllib2.quote(_BUILD_HOST), urllib2.quote(_BUILD_PROJECT),
+                urllib2.quote(builder), urllib2.quote(build_num)))
+
+  # Fetch build status file from build url.
+  print '\nFetching build status file from: %s' % build_url
+  url = urllib2.urlopen(build_url)
+  build_status_file = url.read()
+  url.close()
+
+  # Convert json build status file to dictionary.
+  build_status_dict = json.loads(build_status_file)
+
+  # Extract stdio log url from build status dictionary.
+  print '\n  Contents of build status file: %s' % build_status_dict
+  return build_status_dict[build_num]['steps']['browser_tests']['logs'][0][1]
+
+
+def _GetStdioLogTestsDict(stdio_url):
+  """Get Stdio Log browser_tests from the given url.
+
+  Args:
+    stdio_url: url to stdio log browser_tests file.
+
+  Returns:
+    Dictionary of tests from stdio log browser_tests text file.
+  """
+  # Fetch builder stdio log text file from url.
+  stdio_text_url = stdio_url+'/text'
+  print '\nFetching builder stdio log file from: %s' % stdio_text_url
+  stdio_text_file = urllib2.urlopen(stdio_text_url)
+
+  # Extract test lines from stdio text file.
+  pattern = r'\[\d+/\d+\] '
+  test_lines = [line for line in stdio_text_file if re.match(pattern, line)]
+  stdio_text_file.close()
+  print '  Total run tests extracted: %s' % len(test_lines)
+  if test_lines:
+    print '  Last run test line: "%s"' % test_lines[-1].strip()
+
+  # Extract test data and pack into stdio tests dictionary.
+  stdio_tests_dict = {}
+  pattern = r'(\[\d*/\d*\]) (.*?) (\(.*\))'
+  for i, line in enumerate(test_lines):
+    m = re.match(pattern, line)
+    if m:
+      stdio_tests_dict[m.group(2)] = (m.group(1), m.group(3))
+    else:
+      print 'Error: Invalid test line %s) %s' % (i, line.strip())
+
+  return stdio_tests_dict
+
+
+def _GetUserSpecifiedTests(tests_file):
+  """Get list of user-specified tests from tests file.
+
+  File must be a text file, formatted with one line per test. Leading
+  and trailing spaces and blanklines are stripped from the test list.
+
+  Args:
+    tests_file: Path and name of tests file.
+
+  Returns:
+    List of user tests read from lines in the tests_file.
+  """
+  print '\nFetching user-specified tests from: %s' % tests_file
+  content = open(tests_file, 'r').read().strip()
+  return [line.strip() for line in content.splitlines() if line.strip()]
+
+
+def _RunAndNotrunTests(stdio_tests, user_tests):
+  """Return lists of individual test instances of user-specified tests.
+
+  The first list is of test instances present in the stdio tests list.
+  Presence indicates that the test instance was run on the build. Second list
+  is tests that are absent from the stdio tests list. Absence means that no
+  instance of the test was run on the build.
+
+  Note that there can be multiple instances of a user-specified test run on a
+  build if a) The test belongs to test group, and b) the test was run with
+  multiple test data values. This function uses a regex to search for multiple
+  instances of tests that match the user-specifed test name.
+
+  Args:
+    stdio_tests: List of test instances run on build.
+    user_tests: List of test names specified by user.
+
+  Returns:
+  1) run_tests: list of test instances of user tests run on build.
+  2) notrun_tests: list of user tests not run on build.
+  """
+  run_user_tests = []
+  notrun_user_tests = []
+  for user_test in user_tests:
+    pattern = r'(.*/)?%s(/\d*)?$' % user_test
+    found_run_test = False
+    for stdio_test in stdio_tests:
+      if re.search(pattern, stdio_test):
+        found_run_test = True
+        run_user_tests.append(stdio_test)
+    if not found_run_test:
+      notrun_user_tests.append(user_test)
+  print '  Run instances of user tests: %s' % len(run_user_tests)
+  print '  Not run user tests: %s\n' % len(notrun_user_tests)
+  return run_user_tests, notrun_user_tests
+
+
+def _GetResultsDict(master, builder):
+  """Get results dictionary from builder results.json file.
+
+  The results dictionary contains information about recent tests run,
+  test results, build numbers, chrome revision numbers, etc, for the
+  last 500 builds on the specified builder.
+
+  Args:
+    master: Master repo (e.g., 'ChromiumChromiumOS')
+    builder: Builder name (e.g., 'Linux ChromiumOS Tests (dbg)(2)')
+
+  Returns:
+    Dictionary of builder results.
+  """
+  # Generate percent-encoded builder results url.
+  results_url = (('https://%s/testfile?master=%s&builder=%s'
+                  '&testtype=browser_tests&name=results.json') %
+                 (urllib2.quote(_TR_HOST), urllib2.quote(master),
+                  urllib2.quote(builder)))
+
+  # Fetch results file from builder results url.
+  print 'Fetching builder results file from %s' % results_url
+  url = urllib2.urlopen(results_url)
+  results_json = url.read()
+  url.close()
+
+  # Convert json results to native Python object.
+  builder_results_dict = json.loads(results_json)
+  return builder_results_dict[builder]
+
+
+def _CreateTestsResultsDictionary(builder_tests_dict):
+  """Create dictionary of all tests+results from builder tests dictionary.
+
+  Parse individual tests and results from the builder tests dictionary,
+  and place into a flattened tests results dictionary. Most tests are
+  standalone, and keyed by thier test name. Some tests belong to a
+  'testinstance' group, and are keyed by their testinstance name, the
+  test data value (e.g., '0', '1', '2'), and the test name.
+
+  For example, a standalone test:result is formatted thus:
+  "BookmarksTest.CommandOpensBookmarksTab": {
+    "results": [...]
+    "times": [...]
+  }
+
+  Tests grouped under a testinstance, are formatted thus:
+  "MediaStreamDevicesControllerBrowserTestInstance": {
+    "MediaStreamDevicesControllerBrowserTest.AudioCaptureAllowed": {
+      "1": {
+        "results": [...],
+        "times": [...]
+      }
+    "MediaStreamDevicesControllerBrowserTest.VideoCaptureAllowed": {
+      "0": {
+        "results": [...],
+        "times": [...]
+      }
+    }
+
+  Args:
+    builder_tests_dict: Dictionary of test groups & tests on builder.
+
+  Returns:
+    Dictionary of flattened tests and their results on builder.
+  """
+  tests_results_dict = {}
+  standalone = 0
+  group = 0
+  subtest = 0
+
+  for group_name in builder_tests_dict:
+    test_group = builder_tests_dict[group_name]
+    if '.' in group_name:
+      standalone += 1
+      test_result = test_group['results']
+      tests_results_dict[group_name] = test_result
+    else:
+      group += 1
+      for test_name in test_group.keys():
+        test_values = test_group[test_name]
+        for value in test_values.keys():
+          subtest += 1
+          test_result = test_values[value]['results']
+          long_test_name = '%s/%s/%s' % (group_name, test_name, value)
+          tests_results_dict[long_test_name] = test_result
+
+  print '  Number of standalone tests: %s' % standalone
+  print '  Number of instance tests (in %s groups): %s' % (group, subtest)
+  print '  Total tests results: %s\n' % len(tests_results_dict)
+
+  return tests_results_dict
+
+
+def _CreateUserTestsResultsDict(test_results_dict, run_user_tests):
+  """Create dictionary of tests:results for all user-specified tests.
+
+  If a user specified test is missing from builder test+results, then default
+  the test result to the code for missing test: 'O'.
+
+  Args:
+    test_results_dict: Builder tests+results dictionary
+    run_user_tests: List of run instances of user specified tests.
+
+  Returns:
+    Dictionary of tests to results for all user specified tests.
+  """
+  user_tests_results_dict = {}
+  # Iterate over run user-specified tests.
+  for test_name in run_user_tests:
+    if test_name in test_results_dict:  # Test has results.
+      test_result = test_results_dict[test_name]
+    else:
+      test_result = [[999, u'O']]  # Set result to missing.
+    user_tests_results_dict[test_name] = test_result
+
+  return user_tests_results_dict
+
+
+def _CreateResultOfTestsDict(user_tests_results_dict):
+  """Create dictionary of user tests keyed by result.
+
+  Args:
+    user_tests_results_dict: dictionary of user tests to results.
+
+  Returns:
+    Dictionary of results of tests.
+  """
+  # Test result type lists.
+  missing = ['O']
+  passed = ['P']
+  fails = [key for key in _RESULT_TYPES if key not in missing+passed]
+
+  failed_tests = []
+  passed_tests = []
+  missing_tests = []
+  for test in user_tests_results_dict:
+    result = user_tests_results_dict[test][0][1]
+    if result in fails:
+      failed_tests.append(test)
+    elif result in passed:
+      passed_tests.append(test)
+    elif result in missing:
+      missing_tests.append(test)
+  return {_FAILED: failed_tests,
+          _PASSED: passed_tests,
+          _MISSING: missing_tests}
+
+
+def _ReportTestsByResult(result_of_tests_dict, tr_dict, report_dir):
+  """Print and write report of tests, grouped by result type.
+
+  Args:
+    result_of_tests_dict: Dictionary of results for tests.
+    tr_dict: Dictionary of tests to results for builder.
+    report_dir: Directory where to save report.
+  """
+  # Test report result types and section headers.
+  report_results_headers = {
+      _NOTRUN: 'Test Status: Not Run',
+      _FAILED: 'Test Result: Fail or other Error',
+      _PASSED: 'Test Result: Passed recently',
+      _MISSING: 'Test Result: Passing long-term'
+  }
+  report_section_order = [_NOTRUN, _FAILED, _PASSED, _MISSING]
+
+  ofile = open(report_dir+'/report', 'w')
+  for result_type in report_section_order:
+    header = report_results_headers[result_type]
+    tests = result_of_tests_dict[result_type]
+    print '%s (%s)' % (header, len(tests))
+    ofile.write('%s (%s)\n' % (header, len(tests)))
+    for num, test_name in enumerate(sorted(tests)):
+      if test_name in tr_dict:
+        print '  %s) %s: %s' % (num+1, test_name, tr_dict[test_name][0:2])
+        ofile.write('  %s) %s: %s\n' % (num+1, test_name, tr_dict[test_name]))
+      else:
+        print '  %s) %s' % (num+1, test_name)
+        ofile.write('  %s) %s\n' % (num+1, test_name))
+  ofile.close()
+
+
+def main():
+  """Report test results of specified browsertests."""
+  parser = argparse.ArgumentParser(
+      description=('Report run status and test results for a user-specified '
+                   'list of browser_test tests.'),
+      formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.add_argument('--tests_file', dest='tests_file', default=_TESTS_FILE,
+                      help=('Specify tests path/file '
+                            '(default is %s).' % _TESTS_FILE))
+  parser.add_argument('--report_dir', dest='report_dir', default=_REPORT_DIR,
+                      help=('Specify path to report directory '
+                            '(default is %s).' % _REPORT_DIR))
+  parser.add_argument('--build_num', dest='build_num', default=_BUILD_NUMBER,
+                      help=('Specify builder build number '
+                            '(default is %s).' % _BUILD_NUMBER))
+  parser.add_argument('--master', dest='master', default=_TR_MASTER,
+                      help=('Specify build master repository '
+                            '(default is %s).' % _TR_MASTER))
+  parser.add_argument('--builder', dest='builder', default=_TR_BUILDER,
+                      help=('Specify test builder '
+                            '(default is %s).' % _TR_BUILDER))
+  parser.add_argument('--print_types', dest='print_types',
+                      action='store_true', help='Print test result types.')
+  arguments = parser.parse_args()
+
+  # Set parameters from command line arguments.
+  report_dir = arguments.report_dir
+  tests_file = arguments.tests_file
+  master = arguments.master
+  builder = arguments.builder
+  build_num = arguments.build_num
+  print_types = arguments.print_types
+
+  # Print map of test result types.
+  if print_types:
+    print 'Test result types:'
+    print json.dumps(_RESULT_TYPES, indent=4)
+    sys.exit()
+
+  # Ensure default or user defined tests file points to a real file.
+  if not os.path.isfile(tests_file):
+    print 'Error: Could not find tests file. Try passing in --tests_file.'
+    sys.exit(2)
+
+  # Ensure default or user-defined report folder points to a real dir.
+  if not os.path.exists(report_dir):
+    print ('Error: Could not find report directory. '
+           'Try passing in --report_dir.')
+    sys.exit(2)
+
+  # Get builder stdio log test info for build number.
+  # Find latest completed build number.
+  if build_num == '0' or build_num == '-1':
+    print ('Error: Invalid build number: %s. '
+           'Using %s instead.' % (build_num, _BUILD_NUMBER))
+    build_num = _BUILD_NUMBER
+  completed_build_num = _FindLatestCompletedBuildNumber(build_num)
+
+  # Get list of test instances run on builder for build number.
+  stdio_log_url = _GetStdioLogUrl(builder, completed_build_num)
+  stdio_tests_dict = _GetStdioLogTestsDict(stdio_log_url)
+
+  # Read list of user-specified tests from tests file.
+  user_tests = _GetUserSpecifiedTests(tests_file)
+  if not user_tests:
+    print 'Error: tests file is empty.'
+    sys.exit(2)
+  print '  Number of user tests: %s' % len(user_tests)
+
+  # Get list of instances of run and not run user tests.
+  run_user_tests, notrun_user_tests = (
+      _RunAndNotrunTests(stdio_tests_dict, user_tests))
+
+  # Get run user tests and results data from the specified builder.
+  if run_user_tests:
+    # Fetch builder results dictionary from test-results server.
+    builder_results_dict = _GetResultsDict(master, builder)
+    builder_tests_dict = builder_results_dict['tests']
+  else:
+    builder_tests_dict = {}
+
+  # Extract tests to results dictionary from builder tests dictionary.
+  tests_results_dict = _CreateTestsResultsDictionary(builder_tests_dict)
+
+  # Create dictionary of run user test instances and results.
+  user_tests_results_dict = (
+      _CreateUserTestsResultsDict(tests_results_dict, run_user_tests))
+
+  # Create dictionary of run tests that are failed, passed, and missing.
+  result_of_tests_dict = _CreateResultOfTestsDict(user_tests_results_dict)
+
+  # Add list of not run tests to the result of tests dictionary.
+  result_of_tests_dict[_NOTRUN] = notrun_user_tests
+
+  # Output tests by result type: notrun, failed, passed, and missing.
+  _ReportTestsByResult(result_of_tests_dict, tests_results_dict, report_dir)
+
+if __name__ == '__main__':
+  main()