client/site_tests/graphics_dEQP/scripts/failure_matrix.py

#!/usr/bin/python
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# pylint: disable-msg=W0311

import argparse
import json
import os


gpu_list = [
    #'pinetrail',
    'sandybridge',
    'ivybridge',
    'baytrail',
    'haswell',
    'broadwell',
    'braswell',
    'skylake',
    'mali-t604',
    'mali-t628',
    'mali-t760',
    'rogue',
]

_PROBLEM_STATUS = ['Fail', 'Flaky']

status_dict = {
    'Fail': 'FAIL ',
    'Flaky': 'flaky',
    'Pass': '  +  ',
    'NotSupported': ' --- ',
    'QualityWarning': 'qw   ',
    'CompatibilityWarning': 'cw   ',
    'Unknown': ' ??? ',
    None: ' n/a ',
}

def load_expectation_dict(json_file):
  data = {}
  if os.path.isfile(json_file):
    with open(json_file, 'r') as f:
      text = f.read()
      data = json.loads(text)
  return data


def load_expectations(json_file):
  data = load_expectation_dict(json_file)
  expectations = {}
  # Convert from dictionary of lists to dictionary of sets.
  for key in data:
    expectations[key] = set(data[key])
  return expectations


def get_problem_count(dict, gpu):
  count = 0
  if gpu in dict:
    for status in _PROBLEM_STATUS:
      if status in dict[gpu]:
        count = count + len((dict[gpu])[status])
  else:
    print 'Warning: %s not found in dict!' % gpu
  return count


def get_problem_tests(dict):
  tests = set([])
  for gpu in dict:
    for status in _PROBLEM_STATUS:
      if status in dict[gpu]:
        tests = tests.union((dict[gpu])[status])
  return sorted(list(tests))


def get_test_result(dict, test):
  for key in dict:
    if test in dict[key]:
      return key
  return None


argparser = argparse.ArgumentParser(
    description='Create a matrix of failing tests per GPU.')
argparser.add_argument('interface',
                       default='gles2',
                       help='Interface for matrix (gles2, gles3, gles31).')
args = argparser.parse_args()
status = '%s-master.json' % args.interface

dict = {}
for gpu in gpu_list:
  filename = 'expectations/%s/%s' % (gpu, status)
  dict[gpu] = load_expectations(filename)

tests = get_problem_tests(dict)

print 'Legend:'
for key in status_dict:
  print '%s  -->  %s' % (status_dict[key], key)
print

offset = ''
for gpu in gpu_list:
  print '%s%s' % (offset, gpu)
  offset = '%s    |    ' % offset
print offset

text_count = ''
text_del = ''
for gpu in gpu_list:
  text_count = '%s%5d    ' % (text_count, get_problem_count(dict, gpu))
  text_del = '%s=========' % text_del
text_count = '%s  Total failure count (Fail + Flaky)' % text_count
print text_del
print text_count
print text_del

for test in tests:
  text = ''
  for gpu in gpu_list:
    result = get_test_result(dict[gpu], test)
    status = status_dict[result]
    text = '%s  %s  ' % (text, status)
  text = '%s  %s' % (text, test)
  print text

print text_del
print '%s repeated' % text_count
print text_del