provingground/browsertest_status.py

#!/usr/bin/python
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to display latest test run status and test results for a user-
# specified list of browser_tests. The names of the desired brower_tests are
# read from a file located (by default) in the same directory as this script.
#
# Latest test run status is fetched from the build.chromium.org build server,
# read from the 'stdio' text file located at:
#   http://build.chromium.org/p/chromium.chromiumos/builders/TR_BUILDER/
#   builds/BUILD_NUMBER/steps/browser_tests/logs/stdio/text
#
# Recent test results are fetched from the test-results.appspot.com server,
# read from the results.json file located at:
#   https://test-results.appspot.com/testfile?master=ChromiumChromiumOS&
#   builder=Linux ChromiumOS Tests (2)&testtype=browser_tests&
#   name=results.json
#

"""Script to report test status and results of user-specified browsertests."""
__author__ = ('scunningham@google.com Scott Cunningham')

import argparse
import json
import os
import re
import sys
import urllib2

# Chromium builder url parameter defaults.
_BUILD_HOST = 'build.chromium.org'
_BUILD_PROJECT = 'chromium.chromiumos'
_BUILD_NUMBER = '-2'

# TestResults server url parameter defaults.
_TR_HOST = 'test-results.appspot.com'  # URI to TestResults server.
_TR_MASTER = 'ChromiumChromiumOS'  # Test-results build master repository.
_TR_BUILDER = 'Linux ChromiumOS Tests (2)'  # TestResults builder name.

# Input file and report directory parameter defaults.
_TESTS_FILE = './tests'  # Path to the file that contains the tests names.
_REPORT_DIR = os.getcwd()  # Path to the directory to store the results report.

# Contents of test result types.
_RESULT_TYPES = {
    'A': 'AUDIO',
    'C': 'CRASH',
    'F': 'TEXT',
    'I': 'IMAGE',
    'L': 'FLAKY',
    'O': 'MISSING',
    'N': 'NO DATA',
    'Q': 'FAIL',
    'P': 'PASS',
    'T': 'TIMEOUT',
    'Y': 'NOTRUN',
    'X': 'SKIP',
    'Z': 'IMAGE+TEXT'
    }

# Report header result types.
_NOTRUN = 'NotRun'
_FAILED = 'Failed'
_PASSED = 'Passed'
_MISSING = 'Missing'


def _FindLatestCompletedBuildNumber(build_number):
  """Find the latest completed build number from the given build number.

  Check if build with build_number completed successfully. If not, iteratively
  check earlier versions, until a successfully completed build is found.

  Args:
    build_number: build number to start search.

  Returns:
    Build number of most recent successfully completed build.
  """
  # TODO(scunningham): Implement a real find function in later version.
  return build_number


def _GetStdioLogUrl(builder, build_num):
  """Get url to Stdio Log file from builder for build number.

  Args:
    builder: Builder name.
    build_num: Build number.

  Returns:
    Url to the Stdio log text file.
  """
  # Generate percent-encoded build url.
  build_url = (('http://%s/p/%s/json/builders/%s/builds?'
                'select=%s/steps/browser_tests/') %
               (urllib2.quote(_BUILD_HOST), urllib2.quote(_BUILD_PROJECT),
                urllib2.quote(builder), urllib2.quote(build_num)))

  # Fetch build status file from build url.
  print '\nFetching build status file from: %s' % build_url
  url = urllib2.urlopen(build_url)
  build_status_file = url.read()
  url.close()

  # Convert json build status file to dictionary.
  build_status_dict = json.loads(build_status_file)

  # Extract stdio log url from build status dictionary.
  print '\n  Contents of build status file: %s' % build_status_dict
  return build_status_dict[build_num]['steps']['browser_tests']['logs'][0][1]


def _GetStdioLogTestsDict(stdio_url):
  """Get Stdio Log browser_tests from the given url.

  Args:
    stdio_url: url to stdio log browser_tests file.

  Returns:
    Dictionary of tests from stdio log browser_tests text file.
  """
  # Fetch builder stdio log text file from url.
  stdio_text_url = stdio_url+'/text'
  print '\nFetching builder stdio log file from: %s' % stdio_text_url
  stdio_text_file = urllib2.urlopen(stdio_text_url)

  # Extract test lines from stdio text file.
  pattern = r'\[\d+/\d+\] '
  test_lines = [line for line in stdio_text_file if re.match(pattern, line)]
  stdio_text_file.close()
  print '  Total run tests extracted: %s' % len(test_lines)
  if test_lines:
    print '  Last run test line: "%s"' % test_lines[-1].strip()

  # Extract test data and pack into stdio tests dictionary.
  stdio_tests_dict = {}
  pattern = r'(\[\d*/\d*\]) (.*?) (\(.*\))'
  for i, line in enumerate(test_lines):
    m = re.match(pattern, line)
    if m:
      stdio_tests_dict[m.group(2)] = (m.group(1), m.group(3))
    else:
      print 'Error: Invalid test line %s) %s' % (i, line.strip())

  return stdio_tests_dict


def _GetUserSpecifiedTests(tests_file):
  """Get list of user-specified tests from tests file.

  File must be a text file, formatted with one line per test. Leading
  and trailing spaces and blanklines are stripped from the test list.

  Args:
    tests_file: Path and name of tests file.

  Returns:
    List of user tests read from lines in the tests_file.
  """
  print '\nFetching user-specified tests from: %s' % tests_file
  content = open(tests_file, 'r').read().strip()
  return [line.strip() for line in content.splitlines() if line.strip()]


def _RunAndNotrunTests(stdio_tests, user_tests):
  """Return lists of individual test instances of user-specified tests.

  The first list is of test instances present in the stdio tests list.
  Presence indicates that the test instance was run on the build. Second list
  is tests that are absent from the stdio tests list. Absence means that no
  instance of the test was run on the build.

  Note that there can be multiple instances of a user-specified test run on a
  build if a) The test belongs to test group, and b) the test was run with
  multiple test data values. This function uses a regex to search for multiple
  instances of tests that match the user-specifed test name.

  Args:
    stdio_tests: List of test instances run on build.
    user_tests: List of test names specified by user.

  Returns:
  1) run_tests: list of test instances of user tests run on build.
  2) notrun_tests: list of user tests not run on build.
  """
  run_user_tests = []
  notrun_user_tests = []
  for user_test in user_tests:
    pattern = r'(.*/)?%s(/\d*)?$' % user_test
    found_run_test = False
    for stdio_test in stdio_tests:
      if re.search(pattern, stdio_test):
        found_run_test = True
        run_user_tests.append(stdio_test)
    if not found_run_test:
      notrun_user_tests.append(user_test)
  print '  Run instances of user tests: %s' % len(run_user_tests)
  print '  Not run user tests: %s\n' % len(notrun_user_tests)
  return run_user_tests, notrun_user_tests


def _GetResultsDict(master, builder):
  """Get results dictionary from builder results.json file.

  The results dictionary contains information about recent tests run,
  test results, build numbers, chrome revision numbers, etc, for the
  last 500 builds on the specified builder.

  Args:
    master: Master repo (e.g., 'ChromiumChromiumOS')
    builder: Builder name (e.g., 'Linux ChromiumOS Tests (2)')

  Returns:
    Dictionary of builder results.
  """
  # Generate percent-encoded builder results url.
  results_url = (('https://%s/testfile?master=%s&builder=%s'
                  '&testtype=browser_tests&name=results.json') %
                 (urllib2.quote(_TR_HOST), urllib2.quote(master),
                  urllib2.quote(builder)))

  # Fetch results file from builder results url.
  print 'Fetching builder results file from %s' % results_url
  url = urllib2.urlopen(results_url)
  results_json = url.read()
  url.close()

  # Convert json results to native Python object.
  builder_results_dict = json.loads(results_json)
  return builder_results_dict[builder]


def _CreateTestsResultsDictionary(builder_tests_dict):
  """Create dictionary of all tests+results from builder tests dictionary.

  Parse individual tests and results from the builder tests dictionary,
  and place into a flattened tests results dictionary. Most tests are
  standalone, and keyed by thier test name. Some tests belong to a
  'testinstance' group, and are keyed by their testinstance name, the
  test data value (e.g., '0', '1', '2'), and the test name.

  For example, a standalone test:result is formatted thus:
  "BookmarksTest.CommandOpensBookmarksTab": {
    "results": [...]
    "times": [...]
  }

  Tests grouped under a testinstance, are formatted thus:
  "MediaStreamDevicesControllerBrowserTestInstance": {
    "MediaStreamDevicesControllerBrowserTest.AudioCaptureAllowed": {
      "1": {
        "results": [...],
        "times": [...]
      }
    "MediaStreamDevicesControllerBrowserTest.VideoCaptureAllowed": {
      "0": {
        "results": [...],
        "times": [...]
      }
    }

  Args:
    builder_tests_dict: Dictionary of test groups & tests on builder.

  Returns:
    Dictionary of flattened tests and their results on builder.
  """
  tests_results_dict = {}
  standalone = 0
  group = 0
  subtest = 0

  for group_name in builder_tests_dict:
    test_group = builder_tests_dict[group_name]
    if '.' in group_name:
      standalone += 1
      test_result = test_group['results']
      tests_results_dict[group_name] = test_result
    else:
      group += 1
      for test_name in test_group.keys():
        test_values = test_group[test_name]
        for value in test_values.keys():
          subtest += 1
          test_result = test_values[value]['results']
          long_test_name = '%s/%s/%s' % (group_name, test_name, value)
          tests_results_dict[long_test_name] = test_result

  print '  Number of standalone tests: %s' % standalone
  print '  Number of instance tests (in %s groups): %s' % (group, subtest)
  print '  Total tests results: %s\n' % len(tests_results_dict)

  return tests_results_dict


def _CreateUserTestsResultsDict(test_results_dict, run_user_tests):
  """Create dictionary of tests:results for all user-specified tests.

  If a user specified test is missing from builder test+results, then default
  the test result to the code for missing test: 'O'.

  Args:
    test_results_dict: Builder tests+results dictionary
    run_user_tests: List of run instances of user specified tests.

  Returns:
    Dictionary of tests to results for all user specified tests.
  """
  user_tests_results_dict = {}
  # Iterate over run user-specified tests.
  for test_name in run_user_tests:
    if test_name in test_results_dict:  # Test has results.
      test_result = test_results_dict[test_name]
    else:
      test_result = [[999, u'O']]  # Set result to missing.
    user_tests_results_dict[test_name] = test_result

  return user_tests_results_dict


def _CreateResultOfTestsDict(user_tests_results_dict):
  """Create dictionary of user tests keyed by result.

  Args:
    user_tests_results_dict: dictionary of user tests to results.

  Returns:
    Dictionary of results of tests.
  """
  # Test result type lists.
  missing = ['O']
  passed = ['P']
  fails = [key for key in _RESULT_TYPES if key not in missing+passed]

  failed_tests = []
  passed_tests = []
  missing_tests = []
  for test in user_tests_results_dict:
    result = user_tests_results_dict[test][0][1]
    if result in fails:
      failed_tests.append(test)
    elif result in passed:
      passed_tests.append(test)
    elif result in missing:
      missing_tests.append(test)
  return {_FAILED: failed_tests,
          _PASSED: passed_tests,
          _MISSING: missing_tests}


def _ReportTestsByResult(result_of_tests_dict, tr_dict, report_dir):
  """Print and write report of tests, grouped by result type.

  Args:
    result_of_tests_dict: Dictionary of results for tests.
    tr_dict: Dictionary of tests to results for builder.
    report_dir: Directory where to save report.
  """
  # Test report result types and section headers.
  report_results_headers = {
      _NOTRUN: 'Test Status: Not Run',
      _FAILED: 'Test Result: Fail or other Error',
      _PASSED: 'Test Result: Passed recently',
      _MISSING: 'Test Result: Passing long-term'
  }
  report_section_order = [_NOTRUN, _FAILED, _PASSED, _MISSING]

  ofile = open(report_dir+'/report', 'w')
  for result_type in report_section_order:
    header = report_results_headers[result_type]
    tests = result_of_tests_dict[result_type]
    print '%s (%s)' % (header, len(tests))
    ofile.write('%s (%s)\n' % (header, len(tests)))
    for num, test_name in enumerate(sorted(tests)):
      if test_name in tr_dict:
        print '  %s) %s: %s' % (num+1, test_name, tr_dict[test_name][0:2])
        ofile.write('  %s) %s: %s\n' % (num+1, test_name, tr_dict[test_name]))
      else:
        print '  %s) %s' % (num+1, test_name)
        ofile.write('  %s) %s\n' % (num+1, test_name))
  ofile.close()


def main():
  """Report test results of specified browsertests."""
  parser = argparse.ArgumentParser(
      description=('Report run status and test results for a user-specified '
                   'list of browser_test tests.'),
      formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('--tests_file', dest='tests_file', default=_TESTS_FILE,
                      help=('Specify tests path/file '
                            '(default is %s).' % _TESTS_FILE))
  parser.add_argument('--report_dir', dest='report_dir', default=_REPORT_DIR,
                      help=('Specify path to report directory '
                            '(default is %s).' % _REPORT_DIR))
  parser.add_argument('--build_num', dest='build_num', default=_BUILD_NUMBER,
                      help=('Specify builder build number '
                            '(default is %s).' % _BUILD_NUMBER))
  parser.add_argument('--master', dest='master', default=_TR_MASTER,
                      help=('Specify build master repository '
                            '(default is %s).' % _TR_MASTER))
  parser.add_argument('--builder', dest='builder', default=_TR_BUILDER,
                      help=('Specify test builder '
                            '(default is %s).' % _TR_BUILDER))
  parser.add_argument('--print_types', dest='print_types',
                      action='store_true', help='Print test result types.')
  arguments = parser.parse_args()

  # Set parameters from command line arguments.
  report_dir = arguments.report_dir
  tests_file = arguments.tests_file
  master = arguments.master
  builder = arguments.builder
  build_num = arguments.build_num
  print_types = arguments.print_types

  # Print map of test result types.
  if print_types:
    print 'Test result types:'
    print json.dumps(_RESULT_TYPES, indent=4)
    sys.exit()

  # Ensure default or user defined tests file points to a real file.
  if not os.path.isfile(tests_file):
    print 'Error: Could not find tests file. Try passing in --tests_file.'
    sys.exit(2)

  # Ensure default or user-defined report folder points to a real dir.
  if not os.path.exists(report_dir):
    print ('Error: Could not find report directory. '
           'Try passing in --report_dir.')
    sys.exit(2)

  # Get builder stdio log test info for build number.
  # Find latest completed build number.
  if build_num == '0' or build_num == '-1':
    print ('Error: Invalid build number: %s. '
           'Using %s instead.' % (build_num, _BUILD_NUMBER))
    build_num = _BUILD_NUMBER
  completed_build_num = _FindLatestCompletedBuildNumber(build_num)

  # Get list of test instances run on builder for build number.
  stdio_log_url = _GetStdioLogUrl(builder, completed_build_num)
  stdio_tests_dict = _GetStdioLogTestsDict(stdio_log_url)

  # Read list of user-specified tests from tests file.
  user_tests = _GetUserSpecifiedTests(tests_file)
  if not user_tests:
    print 'Error: tests file is empty.'
    sys.exit(2)
  print '  Number of user tests: %s' % len(user_tests)

  # Get list of instances of run and not run user tests.
  run_user_tests, notrun_user_tests = (
      _RunAndNotrunTests(stdio_tests_dict, user_tests))

  # Get run user tests and results data from the specified builder.
  if run_user_tests:
    # Fetch builder results dictionary from test-results server.
    builder_results_dict = _GetResultsDict(master, builder)
    builder_tests_dict = builder_results_dict['tests']
  else:
    builder_tests_dict = {}

  # Extract tests to results dictionary from builder tests dictionary.
  tests_results_dict = _CreateTestsResultsDictionary(builder_tests_dict)

  # Create dictionary of run user test instances and results.
  user_tests_results_dict = (
      _CreateUserTestsResultsDict(tests_results_dict, run_user_tests))

  # Create dictionary of run tests that are failed, passed, and missing.
  result_of_tests_dict = _CreateResultOfTestsDict(user_tests_results_dict)

  # Add list of not run tests to the result of tests dictionary.
  result_of_tests_dict[_NOTRUN] = notrun_user_tests

  # Output tests by result type: notrun, failed, passed, and missing.
  _ReportTestsByResult(result_of_tests_dict, tests_results_dict, report_dir)

if __name__ == '__main__':
  main()