Update browsertest_status.py to work with ToT and branch builders.
This update allows the browsertest_status.py to work with both ToT and
Branch builders. To accommodate this, it adds two command line flags
to specify:
- Build Host
- Build Project
- Chromium Version
In an unrelated change, it adds a flag to enable creating a report
file, which is now disabled by default.
BUG=chromium:375395
TEST=Put a tests file in the same directory as the script, that lists
the browsertests you want to have a status/results report. Then run:
$ ./browsertest_status.py --tests_file=tests
Change-Id: I7fd5a1516aaacc2422ea6ff784400843ea6bbe23
Reviewed-on: https://chromium-review.googlesource.com/262432
Reviewed-by: Scott Cunningham <scunningham@chromium.org>
Commit-Queue: Scott Cunningham <scunningham@chromium.org>
Tested-by: Scott Cunningham <scunningham@chromium.org>
diff --git a/provingground/browsertest_status.py b/provingground/browsertest_status.py
index 5dae11c..7bac916 100755
--- a/provingground/browsertest_status.py
+++ b/provingground/browsertest_status.py
@@ -1,26 +1,26 @@
-#!/usr/bin/python
-# Copyright (c) 2015 The Chromium OS Authors. All rights reserved.
+#!/usr/bin/python2
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Script to display latest test run status and test results for a user-
-# specified list of browser_tests. The names of the desired brower_tests are
+# specified list of browsertests. The names of the desired browertests are
# read from a file located (by default) in the same directory as this script.
#
-# Latest test run status is fetched from the build.chromium.org build server,
-# read from the 'stdio' text file located at:
-# http://build.chromium.org/p/chromium.chromiumos/builders/TR_BUILDER/
-# builds/BUILD_NUMBER/steps/browser_tests/logs/stdio/text
+# Latest test run status for a build is fetched from the builder, read from
+# the 'stdio' text file located at:
+# http://BUILDER_HOST/p/BUILDER_PROJECT/builders/BUILDER_NAME/
+# builds/BUILD_NUMBER/steps/TEST_TYPE/logs/stdio/text
#
-# Recent test results are fetched from the test-results.appspot.com server,
-# read from the results.json file located at:
-# https://test-results.appspot.com/testfile?master=ChromiumChromiumOS&
-# builder=Linux ChromiumOS Tests (2)&testtype=browser_tests&
-# name=results.json
+# Recent test results history are fetched from the test-results server, read
+# from the results.json file located at:
+# https://TR_HOST/testfile?master=TR_MASTER&builder=BUILDER_NAME
+# &testtype=TEST_TYPE&name=results.json
#
"""Script to report test status and results of user-specified browsertests."""
-__author__ = ('scunningham@google.com Scott Cunningham')
+
+from __future__ import print_function
import argparse
import json
@@ -29,16 +29,18 @@
import sys
import urllib2
-# Chromium builder url parameter defaults.
-_BUILD_HOST = 'build.chromium.org'
-_BUILD_PROJECT = 'chromium.chromiumos'
-_BUILD_NUMBER = '-2'
+__author__ = 'scunningham@google.com (Scott Cunningham)'
+
+# Builder url parameter defaults.
+_BUILDER_HOST = 'chromegw.corp.google.com' # Host name of Builder.
+_BUILDER_PROJECT = 'chromeos.chrome' # Project name of Builder.
+_BUILDER_NAME = 'Linux ChromeOS Buildspec Tests' # Builder name.
+_BUILD_NUMBER = -1
_TEST_TYPE = 'browser_tests'
-# TestResults server url parameter defaults.
-_TR_HOST = 'test-results.appspot.com' # URI to TestResults server.
-_TR_MASTER = 'ChromiumChromiumOS' # Test-results build master repository.
-_TR_BUILDER = 'Linux ChromiumOS Tests (dbg)(1)' # TestResults builder name.
+# Test-results server url parameter defaults.
+_TR_HOST = 'test-results.appspot.com' # Host name of test-results server.
+_TR_MASTER = 'ChromiumChromiumOS' # Project master of test-results server.
# Input file and report directory parameter defaults.
_TESTS_FILE = './tests' # Path to the file that contains the tests names.
@@ -68,91 +70,8 @@
_MISSING = 'Missing'
-def _FindLatestCompletedBuildNumber(build_number):
- """Find the latest completed build number from the given build number.
-
- Check if build with build_number completed successfully. If not, iteratively
- check earlier versions, until a successfully completed build is found.
-
- Args:
- build_number: build number to start search.
-
- Returns:
- Build number of most recent successfully completed build.
- """
- # TODO(scunningham): Implement a real find function in later version.
- return build_number
-
-
-def _GetStdioLogUrl(builder, build_num, test_type):
- """Get url to Stdio Log file from builder for build number.
-
- Args:
- builder: Builder name.
- build_num: Build number.
- test_type: Type of browser test.
-
- Returns:
- Url to the Stdio log text file.
- """
- # Generate percent-encoded build url.
- build_url = (('http://%s/p/%s/json/builders/%s/builds?'
- 'select=%s/steps/%s/') %
- (urllib2.quote(_BUILD_HOST), urllib2.quote(_BUILD_PROJECT),
- urllib2.quote(builder), urllib2.quote(build_num),
- urllib2.quote(test_type)))
-
- # Fetch build status file from build url.
- print '\nFetching build status file from: %s' % build_url
- url = urllib2.urlopen(build_url)
- build_status_file = url.read()
- url.close()
-
- # Convert json build status file to dictionary.
- build_status_dict = json.loads(build_status_file)
-
- # Extract stdio log url from build status dictionary.
- print '\n Contents of build status file: %s' % build_status_dict
- return build_status_dict[build_num]['steps'][test_type]['logs'][0][1]
-
-
-def _GetStdioLogTestsDict(stdio_url):
- """Get Stdio Log browser_tests from the given url.
-
- Args:
- stdio_url: url to stdio log browser_tests file.
-
- Returns:
- Dictionary of tests from stdio log browser_tests text file.
- """
- # Fetch builder stdio log text file from url.
- stdio_text_url = stdio_url+'/text'
- print '\nFetching builder stdio log file from: %s' % stdio_text_url
- stdio_text_file = urllib2.urlopen(stdio_text_url)
-
- # Extract test lines from stdio text file.
- pattern = r'\[\d+/\d+\] '
- test_lines = [line for line in stdio_text_file if re.match(pattern, line)]
- stdio_text_file.close()
- print ' Total run tests extracted: %s' % len(test_lines)
- if test_lines:
- print ' Last run test line: "%s"' % test_lines[-1].strip()
-
- # Extract test data and pack into stdio tests dictionary.
- stdio_tests_dict = {}
- pattern = r'(\[\d*/\d*\]) (.*?) (\(.*\))'
- for i, line in enumerate(test_lines):
- m = re.match(pattern, line)
- if m:
- stdio_tests_dict[m.group(2)] = (m.group(1), m.group(3))
- else:
- print 'Error: Invalid test line %s) %s' % (i, line.strip())
-
- return stdio_tests_dict
-
-
def _GetUserSpecifiedTests(tests_file):
- """Get list of user-specified tests from tests file.
+ """Get list of user-specified tests from the given tests_file.
File must be a text file, formatted with one line per test. Leading and
trailing spaces and blanklines are stripped from the test list. If a line
@@ -164,19 +83,426 @@
Returns:
List of user tests read from lines in the tests_file.
"""
- print '\nFetching user-specified tests from: %s' % tests_file
+ print('\nReading user-specified tests from: %s' % tests_file)
content = open(tests_file, 'r').read().strip()
- return [line.strip().split()[0] for line in content.splitlines()
- if line.strip()]
+ user_tests = [line.strip().split()[0] for line in content.splitlines()
+ if line.strip()]
+ if not user_tests:
+ print('Error: tests file is empty.')
+ sys.exit(2)
+ print(' Number of user tests: %s' % len(user_tests))
+
+ return user_tests
+
+
+def _GetBuildsList(build_dict):
+ """Get list of available (cached) builds on the builder.
+
+ Args:
+ build_dict: build info: builder host, project, and name.
+
+ Returns:
+ List of builds available on builder.
+ """
+ # Generate percent-encoded builder url.
+ builder_url = ('https://%s/p/%s/json/builders/%s' %
+ (urllib2.quote(build_dict['builder_host']),
+ urllib2.quote(build_dict['builder_proj']),
+ urllib2.quote(build_dict['builder_name'])))
+
+ # Fetch builder status file from builder url.
+ print('\nFetching builder status file from: %s' % builder_url)
+ try:
+ response = urllib2.urlopen(builder_url)
+ builder_status_file = response.read()
+ response.close()
+ except urllib2.HTTPError as err:
+ print('HTTP error: %s' % err)
+ sys.exit(2)
+ except urllib2.URLError as err:
+ print('Notice: Builder was not available: %s' % err)
+ sys.exit(2)
+
+ # Convert json builder status file to dictionary.
+ builder_status_dict = json.loads(builder_status_file)
+ build_list = builder_status_dict['cachedBuilds']
+ print('\nList of avaiable builds: %s\n' % build_list)
+ return build_list
+
+
+def _FindBuildByChromiumVersion(build_dict):
+ """Find the latest completed build with the given chromium version.
+
+ Search from the top of the cached build list for a build with the given
+ chromium |version| number as it's buildspec_version. If found, return the
+ build number. If no build is found containing the |version|, return None.
+
+ Args:
+ build_dict: build info, with chromium version number.
+
+ Returns:
+ Number of most recent build containing the given chromium version.
+ """
+ # TODO(scunningham): Implement real function in later version.
+ cr_version = build_dict['cr_version']
+ if cr_version:
+ pass
+ return _BUILD_NUMBER
+
+
+def _GetNominalBuildNumber(build_num, builds_list):
+ """Verify the build number is in the available builds list.
+
+ If the user-specified |build_num| is negative, then verify it is a valid
+ ordinal index, and get the nominal build number for that index. If it is
+ positive, then verify the nominal build number exists in the |builds_list|.
+ Return the nominal build number. Note that if a build number is available
+ does not mean that the build is valid or completed.
+
+ Args:
+ build_num: nominal build number (positive) or ordinal index (negative).
+ builds_list: List of available build numbers on the builder.
+
+ Returns:
+ Nominal build number in |builds_list|.
+ """
+ # Verify that build number exists in the builds list.
+ if build_num < 0: # User entered an index, not a build number.
+ if len(builds_list) < -build_num:
+ print('Error: The build_num index %s is too large.' % build_num)
+ sys.exit(2)
+ build_number = builds_list[build_num]
+ print('\n Index %s selects build %s' % (build_num, build_number))
+ build_num = build_number
+ elif build_num not in builds_list:
+ print('Error: build %s is not available.' % build_num)
+ sys.exit(2)
+ return build_num
+
+
+def _LatestCompletedBuild(build_dict, builds_list):
+ """Find the latest completed build number from the given list of builds.
+
+ Check each build in the |builds_list|, starting with the build number in
+ the given |build_dict|, to determine if the build completed successfully.
+ If completed successfully, return the number of the build. If not, continue
+ checking. If none of the builds completed successfully, exits.
+
+ Args:
+ build_dict: build info dictionary, with build number to start search.
+ builds_list: List of cached build numbers on the builder.
+
+ Returns:
+ Build number of the latest successfully completed build, and the build
+ test status dictionary of that build.
+ """
+ # Find the latest completed build, starting from build_num.
+ build_num = build_dict['build_num']
+ starting_build_num = build_num
+ starting_build_num_failed = False
+ starting_build_index = builds_list.index(starting_build_num)
+ for build_num in reversed(builds_list[0:starting_build_index+1]):
+ build_test_status_dict = _BuildIsCompleted(build_dict, build_num)
+ if build_test_status_dict is not None:
+ break
+ starting_build_num_failed = True
+ else:
+ print('No completed builds are available.')
+ sys.exit(2)
+ if starting_build_num_failed:
+ print('\nError: Requested build_num %s was not completed successfully.' %
+ starting_build_num)
+ print('Using latest successfully completed build: %s\n' % build_num)
+ return build_num, build_test_status_dict
+
+
+def _BuildIsCompleted(build_dict, build_num):
+ """Determine whether build was completed successfully.
+
+ Get the build test status. Check whether the build given by |build_num]
+ was terminated by an error, or is not finished building. If so, return
+ None. Otherwise, return the build test status dictionary.
+
+ Args:
+ build_dict: build info dictionary.
+ build_num: build number to check.
+
+ Returns:
+ Dictionary of build test status if build is completed. Otherwise, None.
+ """
+
+ # Copy original build_dict, and point copy to |build_num|.
+ temp_build_dict = dict(build_dict)
+ temp_build_dict['build_num'] = build_num
+
+ # Get build test status dictionary from builder for |build_num|.
+ build_test_status_dict = _GetBuildTestStatus(temp_build_dict)
+ steps_dict = build_test_status_dict[str(build_num)]['steps']
+ test_type_dict = steps_dict[build_dict['test_type']]
+
+ # Check if build failed or threw an exception:
+ if 'error' in test_type_dict:
+ return None
+
+ # Check isFinished status in test_type_dict.
+ if test_type_dict['isFinished']:
+ return build_test_status_dict
+
+ return None
+
+
+def _GetBuildTestStatus(build_dict):
+ """Get the build test status for the given build.
+
+ Fetch the build test status file from the builder for the build number,
+ specified in the build info dictionary given by |build_dict|.
+
+ Args:
+ build_dict: Build info dictionary.
+
+ Returns:
+ Build Test Status dictionary.
+ """
+ # Generate percent-encoded build test status url.
+ build_url = (('https://%s/p/%s/json/builders/%s/builds?'
+ 'select=%s/steps/%s/') %
+ (urllib2.quote(build_dict['builder_host']),
+ urllib2.quote(build_dict['builder_proj']),
+ urllib2.quote(build_dict['builder_name']),
+ build_dict['build_num'],
+ urllib2.quote(build_dict['test_type'])))
+ print('Fetching build test status file from: %s' % build_url)
+ return _FetchStatusFromUrl(build_dict, build_url)
+
+
+def _GetBuildStatus(build_dict):
+ """Get the build status for the given build.
+
+ Fetch the build status file from the builder for the build number, specified
+ in the build info dictionary given by |build_dict|.
+
+ Args:
+ build_dict: Build info dictionary.
+
+ Returns:
+ Build Status dictionary.
+ """
+ # Generate percent-encoded build status url.
+ build_url = (('https://%s/p/%s/json/builders/%s/builds/%s') %
+ (urllib2.quote(build_dict['builder_host']),
+ urllib2.quote(build_dict['builder_proj']),
+ urllib2.quote(build_dict['builder_name']),
+ build_dict['build_num']))
+ print('Fetching build status file from: %s' % build_url)
+ return _FetchStatusFromUrl(build_dict, build_url)
+
+
+def _FetchStatusFromUrl(build_dict, build_url):
+ """Get the status from the given URL.
+
+ Args:
+ build_dict: Build info dictionary.
+ build_url: URL to the status json file.
+
+ Returns:
+ Status dictionary.
+ """
+ # Fetch json status file from build url.
+ hosted_url = _SetUrlHost(build_url, build_dict['builder_host'])
+ url = urllib2.urlopen(hosted_url)
+ status_file = url.read()
+ url.close()
+
+ # Convert json status file to dictionary.
+ status_dict = json.loads(status_file)
+ return status_dict
+
+
+def _PrintChromiumVersion(build_properties):
+ """Get and print the version number of chromium used in the build.
+
+ Args:
+ build_properties: The properties dictionary for the build.
+
+ Returns:
+ A string containing the chromium version.
+ """
+ for property_list in build_properties:
+ if 'buildspec_version' in property_list:
+ chromium_version = property_list[1]
+ print(' Chromium version: %s' % chromium_version)
+ break
+ else:
+ chromium_version = None
+ print(' Warning: Build properties has no chromium version.')
+ return chromium_version
+
+
+def _GetTestsFailedList(build_dict, build_status_dict):
+ """Get list of failed tests for given build.
+
+ Extract test status summary, including number of tests disabled, flaky, and
+ failed, from the test step in |build_status_dict|. If the test step was
+ finished, then create a list of tests that failed from the failures string.
+
+ Args:
+ build_dict: Build info dictionary.
+ build_status_dict: Build status dictionary.
+
+ Returns:
+ List of tests that failed.
+ """
+ # Get test type from build_dict and tests steps from build_status_dict.
+ test_type = build_dict['test_type']
+ build_steps = build_status_dict['steps']
+
+ # Get count of disabled, flaky, failed, and test failures.
+ num_tests_disabled = 0
+ num_tests_flaky = 0
+ num_tests_failed = 0
+ test_failures = ''
+
+ for step_dict in build_steps:
+ text_list = step_dict['text']
+ if test_type in text_list:
+ for item in text_list:
+ m = re.match(r'(\d+) disabled', item)
+ if m:
+ num_tests_disabled = m.group(1)
+ continue
+ m = re.match(r'(\d+) flaky', item)
+ if m:
+ num_tests_flaky = m.group(1)
+ continue
+ m = re.match(r'failed (\d+)', item)
+ if m:
+ num_tests_failed = m.group(1)
+ continue
+ m = re.match(r'<br\/>failures:<br\/>(.*)<br\/>', item)
+ if m:
+ test_failures = m.group(1)
+ continue
+ break # Exit step_dict loop if test_type is in text_list.
+ is_finished = step_dict['isFinished']
+ else:
+ print('Error: build_steps has no \'%s\' step.' % test_type)
+ is_finished = 'Error'
+
+ # Split the test_failures into a tests_failed_list.
+ tests_failed_list = []
+ if num_tests_failed:
+ tests_failed_list = test_failures.split('<br/>')
+
+ print(' Test finished: %s' % is_finished)
+ print(' Disabled: %s' % num_tests_disabled)
+ print(' Flaky: %s' % num_tests_flaky)
+ print(' Failed: %s : %s' % (num_tests_failed, tests_failed_list))
+
+ return tests_failed_list
+
+
+def _GetStdioLogUrlFromBuildStatus(build_dict, build_test_status_dict):
+ """Get url to Stdio Log file from given build test status dictionary.
+
+ Args:
+ build_dict: Build info dictionary.
+ build_test_status_dict: Build Test Status dictionary.
+
+ Returns:
+ Url to the Stdio Log text file.
+ """
+
+ steps_dict = build_test_status_dict[str(build_dict['build_num'])]['steps']
+ test_type_dict = steps_dict[build_dict['test_type']]
+
+ if 'error' in test_type_dict:
+ return None
+
+ log_url = test_type_dict['logs'][0][1]
+ stdio_log_url = _SetUrlHost(log_url, build_dict['builder_host'])+'/text'
+ return stdio_log_url
+
+
+def _GetStdioLogTests(stdio_log_url, tests_failed_list):
+ """Get Stdio Log Tests from the given url.
+
+ Extracts tests from the stdio log file of the builder referenced by
+ |stdio_log_url|, and packs them into a stdio tests dictionary. This
+ dictionary uses the long test name as the key, and the value is a list
+ that contains the test result. We use a list for the test result to mirror
+ the format used by the test-result server.
+
+ If a test is in the |tests_failed_list|, then set the test result to the
+ the failure code: 'Q'. Otherwise, set result to the pass code: 'P'. The
+ result repetition count of '999' is a placeholder that indicates that the
+ value is not from the test-result server.
+
+ Here is the format of the dictionary:
+ {
+ MaybeSetMetadata/SafeBrowseService.MalwareImg/1: [[999, 'P']],
+ MaybeSetMetadata/SafeBrowseService.MalwareImg/2: [[999, 'Q']],
+ PlatformAppBrowserTest.ComponentBackgroundPage: [[999, 'P']],
+ NoSessionRestoreTest.LocalStorageClearedOnExit: [[999, 'P']]
+ }
+
+ Args:
+ stdio_log_url: url to stdio log tests text file.
+ tests_failed_list: list of failed tests, from the build status page.
+
+ Returns:
+ Dictionary of test instances from stdio log tests text file.
+ """
+ # Fetch builder stdio log text file from url.
+ print('\nFetching builder stdio log file from: %s' % stdio_log_url)
+ stdio_text_file = urllib2.urlopen(stdio_log_url)
+
+ # Extract test lines from stdio log text file to test_lines dictionary.
+ p_test = r'\[\d*/\d*\] (.*?) \(.*\)'
+ p_exit = r'exit code \(as seen by runtest.py\)\: (\d+)'
+ test_lines = []
+ exit_flag = False
+ exit_code = None
+
+ for line in stdio_text_file:
+ if not exit_flag:
+ m = re.match(p_test, line)
+ if m:
+ if line not in test_lines:
+ test_lines.append(line)
+ m = re.match(p_exit, line)
+ if m:
+ exit_flag = True
+ exit_code = m.group(1)
+ stdio_text_file.close()
+ print(' Total run tests extracted: %s' % len(test_lines))
+ if test_lines:
+ print(' Last run test line: "%s"' % test_lines[-1].strip())
+
+ # Extract test_lines data and pack into stdio tests dictionary.
+ stdio_tests_dict = {}
+ for i, line in enumerate(test_lines):
+ m = re.match(p_test, line)
+ if m:
+ long_test_name = m.group(1)
+ if long_test_name in tests_failed_list:
+ test_result = [[999, u'Q']] # Test result Failed code 'Q'.
+ else:
+ test_result = [[999, u'P']] # Test result Passed code 'P'.
+ stdio_tests_dict[long_test_name] = test_result
+ else:
+ print('Error: Invalid test line %s) %s' % (i, line.strip()))
+
+ print(' Test Exit Code: %s' % exit_code)
+ return stdio_tests_dict
def _RunAndNotrunTests(stdio_tests, user_tests):
- """Return lists of individual test instances of user-specified tests.
+ """Return lists of run and not-run instances of given user tests.
- The first list is of test instances present in the stdio tests list.
- Presence indicates that the test instance was run on the build. Second list
- is tests that are absent from the stdio tests list. Absence means that no
- instance of the test was run on the build.
+ The first list is of test instances present in the |stdio_tests| list.
+ Presence indicates that the test instance was run on the build. The second
+ list is tests that are absent from the |stdio_tests| list. Absence means
+ that no instance of the test was run on the build.
Note that there can be multiple instances of a user-specified test run on a
build if a) The test belongs to test group, and b) the test was run with
@@ -188,13 +514,13 @@
user_tests: List of test names specified by user.
Returns:
- 1) run_tests: list of test instances of user tests run on build.
- 2) notrun_tests: list of user tests not run on build.
+ 1) run_tests: list of test instances of user tests run on build.
+ 2) notrun_tests: list of user tests not run on build.
"""
run_user_tests = []
notrun_user_tests = []
for user_test in user_tests:
- pattern = r'(.*/)?%s(/\d*)?$' % user_test
+ pattern = r'(.*/)?%s(/\d*)?$' % user_test # pattern for test name.
found_run_test = False
for stdio_test in stdio_tests:
if re.search(pattern, stdio_test):
@@ -202,51 +528,55 @@
run_user_tests.append(stdio_test)
if not found_run_test:
notrun_user_tests.append(user_test)
- print ' Run instances of user tests: %s' % len(run_user_tests)
- print ' Not run user tests: %s\n' % len(notrun_user_tests)
+ print(' User tests: Instances run: %s' % len(run_user_tests))
+ print(' User tests: Not run: %s\n' % len(notrun_user_tests))
return run_user_tests, notrun_user_tests
-def _GetResultsDict(master, builder, test_type):
- """Get results dictionary from builder results.json file.
+def _GetTestResultsJson(master, builder_name, test_type):
+ """Get test results data from results.json file for the builder.
- The results dictionary contains information about recent tests run,
- test results, build numbers, chrome revision numbers, etc, for the
- last 500 builds on the specified builder.
+ The results.json file contains historical data about the tests run on the
+ given |builder_name| for the most recent (up to the last 500) builds. The
+ data includes test names, test result codes, build numbers, and chrome
+ revision numbers.
Args:
master: Master repo (e.g., 'ChromiumChromiumOS')
- builder: Builder name (e.g., 'Linux ChromiumOS Tests (2)')
- test_type: Type of browser test.
+ builder_name: Builder name (e.g., 'Linux ChromiumOS Tests (dbg)(1)')
+ test_type: Type of browsertests: browser_tests or interactive_ui_tests
Returns:
- Dictionary of builder results.
+ Contents of the results.json file from the test-result server for the
+ specified builder.
"""
- # Generate percent-encoded builder results url.
+ # Generate percent-encoded test results url for specified builder.
results_url = (('https://%s/testfile?master=%s&builder=%s'
'&testtype=%s&name=results.json') %
(urllib2.quote(_TR_HOST), urllib2.quote(master),
- urllib2.quote(builder), urllib2.quote(test_type)))
+ urllib2.quote(builder_name), urllib2.quote(test_type)))
- # Fetch results file from builder results url.
- print 'Fetching builder results file from %s' % results_url
- url = urllib2.urlopen(results_url)
- results_json = url.read()
- url.close()
-
- # Convert json results to native Python object.
- builder_results_dict = json.loads(results_json)
- return builder_results_dict[builder]
+ # Fetch results file from test results url.
+ print('Fetching test results file from %s' % results_url)
+ try:
+ url = urllib2.urlopen(results_url)
+ results_json = url.read()
+ url.close()
+ except urllib2.HTTPError:
+ results_json = None
+ print((' Warning: test-result history was not available '
+ 'for builder \'%s\'.\n' % builder_name))
+ return results_json
-def _CreateTestsResultsDictionary(builder_tests_dict):
- """Create dictionary of all tests+results from builder tests dictionary.
+def _CreateTestsResultsDictionary(tr_tests_dict):
+ """Create dictionary of all tests+results from the given tests dictionary.
- Parse individual tests and results from the builder tests dictionary,
- and place into a flattened tests results dictionary. Most tests are
- standalone, and keyed by thier test name. Some tests belong to a
- 'testinstance' group, and are keyed by their testinstance name, the
- test data value (e.g., '0', '1', '2'), and the test name.
+ Parse individual tests and results from the |tr_tests_dict|, and place them
+ into a flattened tests results dictionary. Most tests are standalone, and
+ keyed by their test name. Some tests belong to a testinstance group, and are
+ keyed by their testinstance group name, then the testinstance number
+ (e.g., '0', '1', '2'), and finally the test name.
For example, a standalone test:result is formatted thus:
"BookmarksTest.CommandOpensBookmarksTab": {
@@ -255,32 +585,39 @@
}
Tests grouped under a testinstance, are formatted thus:
- "MediaStreamDevicesControllerBrowserTestInstance": {
- "MediaStreamDevicesControllerBrowserTest.AudioCaptureAllowed": {
+ "KioskUpdateSuite": {
+ "KioskUpdateTest.PermissionChange": {
"1": {
"results": [...],
"times": [...]
}
- "MediaStreamDevicesControllerBrowserTest.VideoCaptureAllowed": {
+ "KioskUpdateTest.PermissionChange": {
"0": {
"results": [...],
"times": [...]
}
}
+ The flattened test results dictionary is formatted thus:
+ {
+ BookmarksTest.CommandOpensBookmarksTab: [[60, u'Q'], [440, u'P']],
+ KioskUpdateTest.PermissionChange/0: [[498, u'P'], [2, u'Q']]
+ KioskUpdateTest.PermissionChange/1: [[493, u'P'], [7, u'Q']]
+ }
+
Args:
- builder_tests_dict: Dictionary of test groups & tests on builder.
+ tr_tests_dict: Dictionary of test groups & tests.
Returns:
- Dictionary of flattened tests and their results on builder.
+ Dictionary of flattened tests and their results.
"""
tests_results_dict = {}
standalone = 0
group = 0
subtest = 0
- for group_name in builder_tests_dict:
- test_group = builder_tests_dict[group_name]
+ for group_name in tr_tests_dict:
+ test_group = tr_tests_dict[group_name]
if '.' in group_name:
standalone += 1
test_result = test_group['results']
@@ -295,39 +632,49 @@
long_test_name = '%s/%s/%s' % (group_name, test_name, value)
tests_results_dict[long_test_name] = test_result
- print ' Number of standalone tests: %s' % standalone
- print ' Number of instance tests (in %s groups): %s' % (group, subtest)
- print ' Total tests results: %s\n' % len(tests_results_dict)
+ print(' Number of standalone tests: %s' % standalone)
+ print(' Number of instance tests (in %s groups): %s' % (group, subtest))
+ print(' Total tests results: %s\n' % len(tests_results_dict))
return tests_results_dict
-def _CreateUserTestsResultsDict(test_results_dict, run_user_tests):
- """Create dictionary of tests:results for all user-specified tests.
+def _CreateUserTestsResults(run_user_tests, stdio_tests_dict,
+ tests_results_dict):
+ """Create dictionary of tests results for all user-specified tests.
- If a user specified test is missing from builder test+results, then default
- the test result to the code for missing test: 'O'.
+ If a user test is failed in the build status given by |stdio_tests_dict|,
+ then set the test result to failed code: 'Q'. If a user test is in the test
+ results given by |test_results_dict|, then use those results. Otherwise,
+ use the test result given by |stdio_tests_dict|. If a user test is missing
+ from both |stdio_tests_dict| and |test_results_dict|, then set the test
+ test result to missing code: 'O'.
Args:
- test_results_dict: Builder tests+results dictionary
run_user_tests: List of run instances of user specified tests.
+ stdio_tests_dict: builder's results.json test results.
+ tests_results_dict: test results from the tests-results server.
Returns:
- Dictionary of tests to results for all user specified tests.
+ Dictionary of tests and results for all user specified tests.
"""
user_tests_results_dict = {}
- # Iterate over run user-specified tests.
+ # Iterate over tests in the run user-specified tests list.
for test_name in run_user_tests:
- if test_name in test_results_dict: # Test has results.
- test_result = test_results_dict[test_name]
+ if (test_name in stdio_tests_dict and
+ stdio_tests_dict[test_name] == [[999, u'Q']]):
+ test_result = stdio_tests_dict[test_name]
+ elif test_name in tests_results_dict: # Use test-results server results.
+ test_result = tests_results_dict[test_name]
+ elif test_name in stdio_tests_dict: # Use builder results.json results.
+ test_result = stdio_tests_dict[test_name]
else:
test_result = [[999, u'O']] # Set result to missing.
user_tests_results_dict[test_name] = test_result
-
return user_tests_results_dict
-def _CreateResultOfTestsDict(user_tests_results_dict):
+def _CreateResultOfTests(user_tests_results_dict):
"""Create dictionary of user tests keyed by result.
Args:
@@ -357,13 +704,14 @@
_MISSING: missing_tests}
-def _ReportTestsByResult(result_of_tests_dict, tr_dict, report_dir):
+def _ReportTestsByResult(result_of_tests_dict, tr_dict, rout, rdir):
"""Print and write report of tests, grouped by result type.
Args:
result_of_tests_dict: Dictionary of results for tests.
tr_dict: Dictionary of tests to results for builder.
- report_dir: Directory where to save report.
+ rout: flag whether to write report file.
+ rdir: Directory where to write report.
"""
# Test report result types and section headers.
report_results_headers = {
@@ -374,122 +722,212 @@
}
report_section_order = [_NOTRUN, _FAILED, _PASSED, _MISSING]
- ofile = open(report_dir+'/report', 'w')
+ if rout:
+ ofile = open(rdir+'/report', 'w')
for result_type in report_section_order:
header = report_results_headers[result_type]
tests = result_of_tests_dict[result_type]
- print '%s (%s)' % (header, len(tests))
- ofile.write('%s (%s)\n' % (header, len(tests)))
- for num, test_name in enumerate(sorted(tests)):
- if test_name in tr_dict:
- print ' %s) %s: %s' % (num+1, test_name, tr_dict[test_name][0:2])
- ofile.write(' %s) %s: %s\n' % (num+1, test_name, tr_dict[test_name]))
+ print('%s (%s)' % (header, len(tests)))
+ if rout:
+ ofile.write('%s (%s)\n' % (header, len(tests)))
+ for num, test in enumerate(sorted(tests)):
+ if test in tr_dict:
+ print(' %s) %s: %s' % (num+1, test, tr_dict[test][0:2]))
+ if rout:
+ ofile.write(' %s) %s: %s\n' % (num+1, test, tr_dict[test]))
else:
- print ' %s) %s' % (num+1, test_name)
- ofile.write(' %s) %s\n' % (num+1, test_name))
- ofile.close()
+ print(' %s) %s' % (num+1, test))
+ if rout:
+ ofile.write(' %s) %s\n' % (num+1, test))
+ if rout:
+ ofile.close()
+
+
+def _SetUrlHost(url, host):
+ """Modify builder |url| to point to the the correct |host|.
+
+ Args:
+ url: Builder URL, which may not have the correct host.
+ host: Correct builder host.
+
+ Returns:
+ Builder URL with the correct host.
+ """
+ pattern = '(?:http.?://)?(?P<host>[^:/ ]+).*'
+ m = re.search(pattern, url)
+ original_host = m.group('host')
+ rehosted_url = re.sub(original_host, host, url)
+ return rehosted_url
def main():
"""Report test results of specified browsertests."""
parser = argparse.ArgumentParser(
description=('Report run status and test results for a user-specified '
- 'list of browser_test tests.'),
+ 'list of browsertest tests.'),
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--tests_file', dest='tests_file', default=_TESTS_FILE,
help=('Specify tests path/file '
'(default is %s).' % _TESTS_FILE))
+ parser.add_argument('--report_out', dest='report_out', default=None,
+ help=('Write report (default is None).'))
parser.add_argument('--report_dir', dest='report_dir', default=_REPORT_DIR,
help=('Specify path to report directory '
'(default is %s).' % _REPORT_DIR))
parser.add_argument('--master', dest='master', default=_TR_MASTER,
help=('Specify build master repository '
'(default is %s).' % _TR_MASTER))
- parser.add_argument('--builder', dest='builder', default=_TR_BUILDER,
- help=('Specify test builder '
- '(default is %s).' % _TR_BUILDER))
- parser.add_argument('--build_num', dest='build_num', default=_BUILD_NUMBER,
- help=('Specify builder build number '
+ parser.add_argument('--builder_host', dest='builder_host',
+ default=_BUILDER_HOST,
+ help=('Specify builder host name '
+ '(default is %s).' % _BUILDER_HOST))
+ parser.add_argument('--builder_project', dest='builder_project',
+ default=_BUILDER_PROJECT,
+ help=('Specify builder project name '
+ '(default is %s).' % _BUILDER_PROJECT))
+ parser.add_argument('--builder_name', dest='builder_name',
+ default=_BUILDER_NAME,
+ help=('Specify builder name '
+ '(default is %s).' % _BUILDER_NAME))
+ parser.add_argument('--build_num', dest='build_num', type=int,
+ default=_BUILD_NUMBER,
+ help=('Specify positive build number, or negative '
+ 'index from latest build '
'(default is %s).' % _BUILD_NUMBER))
parser.add_argument('--test_type', dest='test_type', default=_TEST_TYPE,
- help=('Specify test type '
+ help=('Specify test type: browser_tests or '
+ 'interactive_ui_tests '
'(default is %s).' % _TEST_TYPE))
+ parser.add_argument('--version', dest='cr_version', default=None,
+ help=('Specify chromium version number '
+ '(default is None).'))
parser.add_argument('--print_types', dest='print_types',
action='store_true', help='Print test result types.')
arguments = parser.parse_args()
+ ### Set parameters from CLI arguments, and check for valid values.
# Set parameters from command line arguments.
tests_file = arguments.tests_file
+ report_out = arguments.report_out
report_dir = arguments.report_dir
master = arguments.master
- builder = arguments.builder
+ builder_host = arguments.builder_host
+ builder_proj = arguments.builder_project
+ builder_name = arguments.builder_name
build_num = arguments.build_num
test_type = arguments.test_type
+ cr_version = arguments.cr_version
print_types = arguments.print_types
- # Print map of test result types.
+ # Print map of test result types and exit.
if print_types:
- print 'Test result types:'
- print json.dumps(_RESULT_TYPES, indent=4)
- sys.exit()
+ print('Test result types:')
+ print(json.dumps(_RESULT_TYPES, indent=4))
+ sys.exit(0)
- # Ensure default or user defined tests file points to a real file.
+ # Ensure default or user-defined |tests_file| points to a real file.
if not os.path.isfile(tests_file):
- print 'Error: Could not find tests file. Try passing in --tests_file.'
+ print('Error: Could not find tests file. Try passing in --tests_file.')
sys.exit(2)
- # Ensure default or user-defined report folder points to a real dir.
+ # Ensure default or user-defined |report_dir| points to a real directory.
if not os.path.exists(report_dir):
- print ('Error: Could not find report directory. '
- 'Try passing in --report_dir.')
+ print(('Error: Could not find report directory. '
+ 'Try passing in --report_dir.'))
sys.exit(2)
- # Get builder stdio log test info for build number.
- # Find latest completed build number.
- if build_num == '0' or build_num == '-1':
- print ('Error: Invalid build number: %s. '
- 'Using %s instead.' % (build_num, _BUILD_NUMBER))
- build_num = _BUILD_NUMBER
- completed_build_num = _FindLatestCompletedBuildNumber(build_num)
+ # Verify that user gave |build_num| or |cr_version|, but not both.
+ if build_num != _BUILD_NUMBER and cr_version:
+ print(('Error: You may specify the build_num or the cr_version, '
+ 'but not both.'))
+ sys.exit(2)
- # Get list of test instances run on builder for build number.
- stdio_log_url = _GetStdioLogUrl(builder, completed_build_num, test_type)
- stdio_tests_dict = _GetStdioLogTestsDict(stdio_log_url)
+ # Verify user gave valid |test_type|.
+ if test_type not in ['browser_tests', 'interactive_ui_tests']:
+ print(('Error: Invalid test_type: %s. Use \'browser_tests\' or '
+ '\'interactive_ui_tests\'') % test_type)
+ sys.exit(2)
- # Read list of user-specified tests from tests file.
+ # Pack valid build info into a portable dictionary.
+ build_dict = {
+ 'builder_host': builder_host,
+ 'builder_proj': builder_proj,
+ 'builder_name': builder_name,
+ 'build_num': build_num,
+ 'test_type': test_type,
+ 'cr_version': cr_version
+ }
+
+ ### Get list of user tests from |tests_file|.
user_tests = _GetUserSpecifiedTests(tests_file)
- if not user_tests:
- print 'Error: tests file is empty.'
- sys.exit(2)
- print ' Number of user tests: %s' % len(user_tests)
- # Get list of instances of run and not run user tests.
+ ### Determine build number from which to get status.
+ # Get list of available builds from builder.
+ builds_list = _GetBuildsList(build_dict)
+
+ # Find the latest completed build by chromium version.
+ if cr_version:
+ build_num = _FindBuildByChromiumVersion(build_dict)
+
+ # Set nominal build_num from builds available in |builds_list|.
+ build_dict['build_num'] = _GetNominalBuildNumber(build_num, builds_list)
+
+ # Get number of latest completed build, and update build_dict with it.
+ build_num, build_test_status_dict = (
+ _LatestCompletedBuild(build_dict, builds_list))
+ build_dict['build_num'] = build_num
+
+ ### Get build status from the builder for the build number.
+ # Get the build status of the latest completed build.
+ build_status_dict = _GetBuildStatus(build_dict)
+
+ # Extract the build properties, and print chromium version.
+ build_properties = build_status_dict['properties']
+ _PrintChromiumVersion(build_properties)
+
+ ### Get test status from the builder for the build number.
+ # Get list of failed tests from build status.
+ tests_failed_list = _GetTestsFailedList(build_dict, build_status_dict)
+
+ # Get stdio log URL from build test status for the latest build_num.
+ stdio_log_url = (
+ _GetStdioLogUrlFromBuildStatus(build_dict, build_test_status_dict))
+
+ # Get dictionary of test instances run on selected build.
+ stdio_tests_dict = _GetStdioLogTests(stdio_log_url, tests_failed_list)
+
+ # Get instances of run and not run user tests.
run_user_tests, notrun_user_tests = (
_RunAndNotrunTests(stdio_tests_dict, user_tests))
- # Get run user tests and results data from the specified builder.
- if run_user_tests:
- # Fetch builder results dictionary from test-results server.
- builder_results_dict = _GetResultsDict(master, builder, test_type)
- builder_tests_dict = builder_results_dict['tests']
+ ### Read test results from test-results server for the builder.
+ test_results_json = _GetTestResultsJson(master, builder_name, test_type)
+ if test_results_json:
+ # Extract tests results dictionary from results json for builder.
+ tr_tests_dict = json.loads(test_results_json)[builder_name]['tests']
+ tests_results_dict = _CreateTestsResultsDictionary(tr_tests_dict)
else:
- builder_tests_dict = {}
+ # Extract test results from stdio tests dictionary.
+ tests_results_dict = stdio_tests_dict
- # Extract tests to results dictionary from builder tests dictionary.
- tests_results_dict = _CreateTestsResultsDictionary(builder_tests_dict)
-
+ ### Combine run user tests, build test status, and test results into a
+ ### single dictionary of user tests and their results, and then into a
+ ### dictionary of results and their tests.
# Create dictionary of run user test instances and results.
user_tests_results_dict = (
- _CreateUserTestsResultsDict(tests_results_dict, run_user_tests))
+ _CreateUserTestsResults(run_user_tests, stdio_tests_dict,
+ tests_results_dict))
# Create dictionary of run tests that are failed, passed, and missing.
- result_of_tests_dict = _CreateResultOfTestsDict(user_tests_results_dict)
+ result_of_tests_dict = _CreateResultOfTests(user_tests_results_dict)
# Add list of not run tests to the result of tests dictionary.
result_of_tests_dict[_NOTRUN] = notrun_user_tests
- # Output tests by result type: notrun, failed, passed, and missing.
- _ReportTestsByResult(result_of_tests_dict, tests_results_dict, report_dir)
+ ### Output report of tests grouped by result. Result types are notrun,
+ ### failed, passed, and missing
+ _ReportTestsByResult(result_of_tests_dict, tests_results_dict,
+ report_out, report_dir)
if __name__ == '__main__':
main()
