graphics_dEQP: add/update scripts.
These scripts are used to maintain dEQP expectations.
Keep them in the tree.
BUG=None.
TEST=None.
Change-Id: I3f76b501a729215ce96e226507b0123ce5444b25
Reviewed-on: https://chromium-review.googlesource.com/334435
Trybot-Ready: Ilja Friedel <ihf@chromium.org>
Tested-by: Ilja Friedel <ihf@chromium.org>
Reviewed-by: Stéphane Marchesin <marcheu@chromium.org>
diff --git a/client/site_tests/graphics_dEQP/diff.sh b/client/site_tests/graphics_dEQP/diff.sh
deleted file mode 100755
index 51bb471..0000000
--- a/client/site_tests/graphics_dEQP/diff.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-gpus=(baytrail broadwell haswell ivybridge sandybridge)
-
-for gpu in ${gpus[*]}
-do
- rm expectations/${gpu}/*.json
- cat expectations/${gpu}/* | sort > /tmp/${gpu}.sorted
- cat expectations/${gpu}/* | sort | uniq > /tmp/${gpu}.sorted_uniq
- diff /tmp/${gpu}.sorted /tmp/${gpu}.sorted_uniq > ${gpu}.diff
-done
-
diff --git a/client/site_tests/graphics_dEQP/scripts/failure_matrix.py b/client/site_tests/graphics_dEQP/scripts/failure_matrix.py
new file mode 100755
index 0000000..bf84335
--- /dev/null
+++ b/client/site_tests/graphics_dEQP/scripts/failure_matrix.py
@@ -0,0 +1,134 @@
+#!/usr/bin/python
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# pylint: disable-msg=W0311
+
+import argparse
+import json
+import os
+
+
+gpu_list = [
+ #'pinetrail',
+ 'sandybridge',
+ 'ivybridge',
+ 'baytrail',
+ 'haswell',
+ 'broadwell',
+ 'braswell',
+ 'skylake',
+ 'mali-t604',
+ 'mali-t628',
+ 'mali-t760',
+ 'rogue',
+]
+
+_PROBLEM_STATUS = ['Fail', 'Flaky']
+
+status_dict = {
+ 'Fail': 'FAIL ',
+ 'Flaky': 'flaky',
+ 'Pass': ' + ',
+ 'NotSupported': ' --- ',
+ 'QualityWarning': 'qw ',
+ 'CompatibilityWarning': 'cw ',
+ 'Unknown': ' ??? ',
+ None: ' n/a ',
+}
+
+def load_expectation_dict(json_file):
+ data = {}
+ if os.path.isfile(json_file):
+ with open(json_file, 'r') as f:
+ text = f.read()
+ data = json.loads(text)
+ return data
+
+
+def load_expectations(json_file):
+ data = load_expectation_dict(json_file)
+ expectations = {}
+ # Convert from dictionary of lists to dictionary of sets.
+ for key in data:
+ expectations[key] = set(data[key])
+ return expectations
+
+
+def get_problem_count(dict, gpu):
+ count = 0
+ if gpu in dict:
+ for status in _PROBLEM_STATUS:
+ if status in dict[gpu]:
+ count = count + len((dict[gpu])[status])
+ else:
+ print 'Warning: %s not found in dict!' % gpu
+ return count
+
+
+def get_problem_tests(dict):
+ tests = set([])
+ for gpu in dict:
+ for status in _PROBLEM_STATUS:
+ if status in dict[gpu]:
+ tests = tests.union((dict[gpu])[status])
+ return sorted(list(tests))
+
+
+def get_test_result(dict, test):
+ for key in dict:
+ if test in dict[key]:
+ return key
+ return None
+
+
+argparser = argparse.ArgumentParser(
+ description='Create a matrix of failing tests per GPU.')
+argparser.add_argument('interface',
+ default='gles2',
+ help='Interface for matrix (gles2, gles3, gles31).')
+args = argparser.parse_args()
+status = '%s-master.json' % args.interface
+
+dict = {}
+for gpu in gpu_list:
+ filename = 'expectations/%s/%s' % (gpu, status)
+ dict[gpu] = load_expectations(filename)
+
+tests = get_problem_tests(dict)
+
+print 'Legend:'
+for key in status_dict:
+ print '%s --> %s' % (status_dict[key], key)
+print
+
+offset = ''
+for gpu in gpu_list:
+ print '%s%s' % (offset, gpu)
+ offset = '%s | ' % offset
+print offset
+
+text_count = ''
+text_del = ''
+for gpu in gpu_list:
+ text_count = '%s%5d ' % (text_count, get_problem_count(dict, gpu))
+ text_del = '%s=========' % text_del
+text_count = '%s Total failure count (Fail + Flaky)' % text_count
+print text_del
+print text_count
+print text_del
+
+for test in tests:
+ text = ''
+ for gpu in gpu_list:
+ result = get_test_result(dict[gpu], test)
+ status = status_dict[result]
+ text = '%s %s ' % (text, status)
+ text = '%s %s' % (text, test)
+ print text
+
+print text_del
+print '%s repeated' % text_count
+print text_del
+
diff --git a/client/site_tests/graphics_dEQP/process_logs.py b/client/site_tests/graphics_dEQP/scripts/process_logs.py
similarity index 68%
rename from client/site_tests/graphics_dEQP/process_logs.py
rename to client/site_tests/graphics_dEQP/scripts/process_logs.py
index e52b1d0..b4eab0a 100644
--- a/client/site_tests/graphics_dEQP/process_logs.py
+++ b/client/site_tests/graphics_dEQP/scripts/process_logs.py
@@ -15,7 +15,9 @@
import subprocess
_EXPECTATIONS_DIR = 'expectations'
-_AUTOTEST_RESULT_TEMPLATE = 'gs://chromeos-autotest-results/%s-chromeos-test/chromeos*/graphics_dEQP/debug/graphics_dEQP.DEBUG'
+_AUTOTEST_RESULT_ID_TEMPLATE = 'gs://chromeos-autotest-results/%s-chromeos-test/chromeos*/graphics_dEQP/debug/graphics_dEQP.DEBUG'
+#_AUTOTEST_RESULT_TAG_TEMPLATE = 'gs://chromeos-autotest-results/%s/graphics_dEQP/debug/graphics_dEQP.DEBUG'
+_AUTOTEST_RESULT_TAG_TEMPLATE = 'gs://chromeos-autotest-results/%s/debug/client.0.DEBUG'
# Use this template for tryjob results:
#_AUTOTEST_RESULT_TEMPLATE = 'gs://chromeos-autotest-results/%s-ihf/*/graphics_dEQP/debug/graphics_dEQP.DEBUG'
_BOARD_REGEX = re.compile(r'ChromeOS BOARD = (.+)')
@@ -30,7 +32,8 @@
re.MULTILINE)
_HASTY_TEST_RESULT_REGEX = re.compile(
r'\[stdout\] Test case \'(.+?)\'..$\n'
- r'.+?\[stdout\] (Pass|Fail|QualityWarning) \((.+)\)', re.MULTILINE)
+ r'.+?\[stdout\] (Pass|NotSupported|QualityWarning|CompatibilityWarning|'
+ r'Fail|ResourceError|Crash|Timeout|InternalError|Skipped) \((.+)\)', re.MULTILINE)
Logfile = namedtuple('Logfile', 'job_id name gs_path')
@@ -52,7 +55,7 @@
return board, gpu, filter, hasty
-def get_logs_from_gs(autotest_result_path):
+def copy_logs_from_gs_path(autotest_result_path):
logs = []
gs_paths = execute(['gsutil', 'ls', autotest_result_path]).splitlines()
for gs_path in gs_paths:
@@ -88,7 +91,8 @@
def get_not_passing_tests(text):
not_passing = []
for test, result in re.findall(_TEST_RESULT_REGEX, text):
- if not (result == 'Pass' or result == 'NotSupported'):
+ if not (result == 'Pass' or result == 'NotSupported' or result == 'Skipped' or
+ result == 'QualityWarning' or result == 'CompatibilityWarning'):
not_passing.append((test, result))
for test, result, details in re.findall(_HASTY_TEST_RESULT_REGEX, text):
if result != 'Pass':
@@ -99,7 +103,7 @@
def load_expectation_dict(json_file):
data = {}
if os.path.isfile(json_file):
- print('Loading file ' + json_file)
+ print 'Loading file ' + json_file
with open(json_file, 'r') as f:
text = f.read()
data = json.loads(text)
@@ -168,7 +172,7 @@
if key != 'Flaky':
not_flaky = list(status_dict[key] - flaky)
not_flaky.sort()
- print('Number of "%s" is %d.' % (key, len(not_flaky)))
+ print 'Number of "%s" is %d.' % (key, len(not_flaky))
clean_dict[key] = not_flaky
# And finally process flaky list/set.
@@ -209,23 +213,36 @@
'TestCase: ' in line or 'Result: ' in line or
'Test Options: ' in line or 'Running in hasty mode.' in line or
# For hasty logs we have:
- ' Pass (' in line or ' Fail (' in line or 'QualityWarning (' in line or
+ 'Pass (' in line or 'NotSupported (' in line or 'Skipped (' in line or
+ 'QualityWarning (' in line or 'CompatibilityWarning (' in line or
+ 'Fail (' in line or 'ResourceError (' in line or 'Crash (' in line or
+ 'Timeout (' in line or 'InternalError (' in line or
' Test case \'' in line):
text += line + '\n'
# TODO(ihf): Warn about or reject log files missing the end marker.
return text
+def all_passing(tests):
+ for _, result in tests:
+ if not (result == 'Pass'):
+ return False
+ return True
+
+
def process_logs(logs):
for log in logs:
text = load_log(log.name)
if text:
- print('================================================================')
- print('Loading %s...' % log.name)
+ print '================================================================'
+ print 'Loading %s...' % log.name
_, gpu, filter, hasty = get_metadata(text)
tests = get_all_tests(text)
- print('Found %d test results.' % len(tests))
- if tests:
+ print 'Found %d test results.' % len(tests)
+ if all_passing(tests):
+ # Delete logs that don't contain failures.
+ os.remove(log.name)
+ else:
# GPU family goes first in path to simplify adding/deleting families.
output_path = os.path.join(_EXPECTATIONS_DIR, gpu)
if not os.access(output_path, os.R_OK):
@@ -236,22 +253,77 @@
merge_expectation_list(expectation_path, tests)
+JOB_TAGS_ALL = (
+'select distinct job_tag from chromeos_autotest_db.tko_test_view_2 '
+'where not job_tag like "%%hostless" and '
+'test_name="graphics_dEQP" and '
+'build_version>="%s" and '
+'build_version<="%s" and '
+'((status = "FAIL" and not job_name like "%%.NotPass") or '
+'job_name like "%%.functional" or '
+'job_name like "%%-master")' )
+
+JOB_TAGS_MASTER = (
+'select distinct job_tag from chromeos_autotest_db.tko_test_view_2 '
+'where not job_tag like "%%hostless" and '
+'test_name="graphics_dEQP" and '
+'build_version>="%s" and '
+'build_version<="%s" and '
+'job_name like "%%-master"' )
+
+def get_result_paths_from_autotest_db(host, user, password, build_from,
+ build_to):
+ paths = []
+ # TODO(ihf): Introduce flag to toggle between JOB_TAGS_ALL and _MASTER.
+ sql = JOB_TAGS_ALL % (build_from, build_to)
+ cmd = ['mysql', '-u%s' % user, '-p%s' % password, '--host', host, '-e', sql]
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ for line in p.communicate()[0].splitlines():
+ # Skip over unrelated sql spew (really first line only):
+ if line and 'chromeos-test' in line:
+ paths.append(_AUTOTEST_RESULT_TAG_TEMPLATE % line.rstrip())
+ print 'Found %d potential results in the database.' % len(paths)
+ return paths
+
+
+def copy_logs_from_gs_paths(paths):
+ i = 1
+ for gs_path in paths:
+ print '[%d/%d] %s' % (i, len(paths), gs_path)
+ copy_logs_from_gs_path(gs_path)
+ i = i+1
+
+
argparser = argparse.ArgumentParser(
description='Download from GS and process dEQP logs into expectations.')
argparser.add_argument(
- 'result_ids',
- metavar='result_id',
- nargs='*', # Zero or more result_ids specified.
- help='List of result log IDs (wildcards for gsutil like 5678* are ok).')
+ '--host',
+ dest='host',
+ default='chromeos-server38.cbf',
+ help='Host containing autotest result DB.')
+argparser.add_argument('--user', dest='user', help='Database user account.')
+argparser.add_argument(
+ '--password',
+ dest='password',
+ help='Password for user account.')
+argparser.add_argument(
+ '--from',
+ dest='build_from',
+ help='Lowest build revision to include. Example: R51-8100.0.0')
+argparser.add_argument(
+ '--to',
+ dest='build_to',
+ help='Highest build revision to include. Example: R51-8101.0.0')
+
args = argparser.parse_args()
print pprint.pformat(args)
# This is somewhat optional. Remove existing expectations to start clean, but
# feel free to process them incrementally.
execute(['rm', '-rf', _EXPECTATIONS_DIR])
-for id in args.result_ids:
- gs_path = _AUTOTEST_RESULT_TEMPLATE % id
- logs = get_logs_from_gs(gs_path)
+
+copy_logs_from_gs_paths(get_result_paths_from_autotest_db(
+ args.host, args.user, args.password, args.build_from, args.build_to))
# This will include the just downloaded logs from GS as well.
logs = get_local_logs()
