automation/clients/report/validate_failures.py

#!/usr/bin/python

# Script to compare testsuite failures against a list of known-to-fail
# tests.

# Contributed by Diego Novillo <dnovillo@google.com>
# Overhaul by Krystian Baclawski <kbaclawski@google.com>
#
# Copyright (C) 2011 Free Software Foundation, Inc.
#
# This file is part of GCC.
#
# GCC is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.
#
# GCC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GCC; see the file COPYING.  If not, write to
# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
"""This script provides a coarser XFAILing mechanism that requires no
detailed DejaGNU markings.  This is useful in a variety of scenarios:

- Development branches with many known failures waiting to be fixed.
- Release branches with known failures that are not considered
  important for the particular release criteria used in that branch.

The script must be executed from the toplevel build directory.  When
executed it will:

1) Determine the target built: TARGET
2) Determine the source directory: SRCDIR
3) Look for a failure manifest file in
   <SRCDIR>/contrib/testsuite-management/<TARGET>.xfail
4) Collect all the <tool>.sum files from the build tree.
5) Produce a report stating:
   a) Failures expected in the manifest but not present in the build.
   b) Failures in the build not expected in the manifest.
6) If all the build failures are expected in the manifest, it exits
   with exit code 0.  Otherwise, it exits with error code 1.
"""

import optparse
import logging
import os
import sys

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from dejagnu.manifest import Manifest
from dejagnu.summary import DejaGnuTestResult
from dejagnu.summary import DejaGnuTestRun

# Pattern for naming manifest files.  The first argument should be
# the toplevel GCC source directory.  The second argument is the
# target triple used during the build.
_MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail'


def GetMakefileVars(makefile_path):
  assert os.path.exists(makefile_path)

  with open(makefile_path) as lines:
    kvs = [line.split('=', 1) for line in lines if '=' in line]

    return dict((k.strip(), v.strip()) for k, v in kvs)


def GetSumFiles(build_dir):
  summaries = []

  for root, _, filenames in os.walk(build_dir):
    summaries.extend([os.path.join(root, filename)
                      for filename in filenames if filename.endswith('.sum')])

  return map(os.path.normpath, summaries)


def ValidBuildDirectory(build_dir, target):
  mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')]

  extra_paths = [os.path.join(build_dir, target),
                 os.path.join(build_dir, 'build-%s' % target)]

  return (all(map(os.path.exists, mandatory_paths)) and
          any(map(os.path.exists, extra_paths)))


def GetManifestPath(build_dir):
  makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile'))
  srcdir = makefile['srcdir']
  target = makefile['target']

  if not ValidBuildDirectory(build_dir, target):
    target = makefile['target_alias']

  if not ValidBuildDirectory(build_dir, target):
    logging.error('%s is not a valid GCC top level build directory.', build_dir)
    sys.exit(1)

  logging.info('Discovered source directory: "%s"', srcdir)
  logging.info('Discovered build target: "%s"', target)

  return _MANIFEST_PATH_PATTERN % (srcdir, target)


def CompareResults(manifest, actual):
  """Compare sets of results and return two lists:
     - List of results present in MANIFEST but missing from ACTUAL.
     - List of results present in ACTUAL but missing from MANIFEST.
  """
  # Report all the actual results not present in the manifest.
  actual_vs_manifest = actual - manifest

  # Filter out tests marked flaky.
  manifest_without_flaky_tests = set(filter(lambda result: not result.flaky,
                                            manifest))

  # Simlarly for all the tests in the manifest.
  manifest_vs_actual = manifest_without_flaky_tests - actual

  return actual_vs_manifest, manifest_vs_actual


def LogResults(level, results):
  log_fun = getattr(logging, level)

  for num, result in enumerate(sorted(results), start=1):
    log_fun('  %d) %s', num, result)


def CheckExpectedResults(manifest_path, build_dir):
  logging.info('Reading manifest file: "%s"', manifest_path)

  manifest = set(Manifest.FromFile(manifest_path))

  logging.info('Getting actual results from build directory: "%s"',
               os.path.realpath(build_dir))

  summaries = GetSumFiles(build_dir)

  actual = set()

  for summary in summaries:
    test_run = DejaGnuTestRun.FromFile(summary)
    failures = set(Manifest.FromDejaGnuTestRun(test_run))
    actual.update(failures)

  if manifest:
    logging.debug('Tests expected to fail:')
    LogResults('debug', manifest)

  if actual:
    logging.debug('Actual test failures:')
    LogResults('debug', actual)

  actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual)

  if actual_vs_manifest:
    logging.info('Build results not in the manifest:')
    LogResults('info', actual_vs_manifest)

  if manifest_vs_actual:
    logging.info('Manifest results not present in the build:')
    LogResults('info', manifest_vs_actual)
    logging.info('NOTE: This is not a failure!  ',
                 'It just means that the manifest expected these tests to '
                 'fail, but they worked in this configuration.')

  if actual_vs_manifest or manifest_vs_actual:
    sys.exit(1)

  logging.info('No unexpected failures.')


def ProduceManifest(manifest_path, build_dir, overwrite):
  if os.path.exists(manifest_path) and not overwrite:
    logging.error('Manifest file "%s" already exists.', manifest_path)
    logging.error('Use --force to overwrite.')
    sys.exit(1)

  testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir))
  manifests = map(Manifest.FromDejaGnuTestRun, testruns)

  with open(manifest_path, 'w') as manifest_file:
    manifest_strings = [manifest.Generate() for manifest in manifests]
    logging.info('Writing manifest to "%s".', manifest_path)
    manifest_file.write('\n'.join(manifest_strings))


def Main(argv):
  parser = optparse.OptionParser(usage=__doc__)
  parser.add_option(
      '-b',
      '--build_dir',
      dest='build_dir',
      action='store',
      metavar='PATH',
      default=os.getcwd(),
      help='Build directory to check. (default: current directory)')
  parser.add_option('-m',
                    '--manifest',
                    dest='manifest',
                    action='store_true',
                    help='Produce the manifest for the current build.')
  parser.add_option(
      '-f',
      '--force',
      dest='force',
      action='store_true',
      help=('Overwrite an existing manifest file, if user requested creating '
            'new one. (default: False)'))
  parser.add_option('-v',
                    '--verbose',
                    dest='verbose',
                    action='store_true',
                    help='Increase verbosity.')
  options, _ = parser.parse_args(argv[1:])

  if options.verbose:
    logging.root.setLevel(logging.DEBUG)

  manifest_path = GetManifestPath(options.build_dir)

  if options.manifest:
    ProduceManifest(manifest_path, options.build_dir, options.force)
  else:
    CheckExpectedResults(manifest_path, options.build_dir)


if __name__ == '__main__':
  logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
  Main(sys.argv)