metadata/utils/extract_configs.py

# Lint as: python3
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import ast
import collections
import glob
import logging
import os
import shutil
from typing import Generator, List, DefaultDict, NoReturn, Optional, Tuple

import control_files

_ROOT_DIR = os.path.realpath(
    os.path.join(os.path.realpath(__file__), '../../..'))
_METADATA_DIR = os.path.join(_ROOT_DIR, 'metadata')
_CATEGORY_DEFS_DIR = os.path.join(_METADATA_DIR, 'tests')
_CATEGORY_DEFS_PACKAGE = '//metadata/tests'
_ALL_TESTS_FILE = os.path.join(_METADATA_DIR, 'tests.star')


class ExtractionError(Exception):
  """Generic error from this package."""


def main():
  """Extract starlark metadata for all tests in this project.

  This script generates the starlark config files by parsing control files.
  The intent is to backfill majority of the required starlark configs this way,
  followed by hand-editing.
  """
  logging.basicConfig(level=logging.INFO)
  controls = control_files.load_all()
  logging.info('Loaded %d control files', len(controls))
  categorized = dict(_categorize_control_files(controls))
  logging.info('Categorized control files into %d categories', len(categorized))

  _delete_existing_defs()
  for category, control in categorized.items():
    with open(_category_def_file(category), 'w') as f:
      f.write(_generate_category_def(control))
  logging.info('Wrote starlark files to %s', _CATEGORY_DEFS_DIR)
  with open(_ALL_TESTS_FILE, 'w') as f:
    f.write(_generate_all_tests_def(categorized.keys()))
  logging.info('Included all categories in %s', _ALL_TESTS_FILE)


def _delete_existing_defs() -> NoReturn:
  if os.path.exists(_CATEGORY_DEFS_DIR):
    shutil.rmtree(_CATEGORY_DEFS_DIR)
    os.mkdir(_CATEGORY_DEFS_DIR)
  if os.path.exists(_ALL_TESTS_FILE):
    os.unlink(_ALL_TESTS_FILE)


def _categorize_control_files(controls: List[control_files.Control]
                             ) -> DefaultDict[str, control_files.Control]:
  categorized = collections.defaultdict(list)
  for c in controls:
    categorized[c.category].append(c)
  if 'uncategorized' in categorized:
    raise ExtractionError('"uncategorized" is reserved')
  if '' in categorized:
    categorized['uncategorized'] = categorized['']
    del categorized['']
  return categorized


def _category_def_file(category: str) -> str:
  return os.path.join(_CATEGORY_DEFS_DIR, '%s.star' % category)


_CATEGORY_TEMPLATE = """
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# !!! GENERATED FILE. DO NOT EDIT !!!

load('//metadata/test_common.star', 'test_common')

def define_tests():
    return [%(test_list)s
    ]
"""

_TEST_TEMPLATE = """
        test_common.define_test(
            %(name)s,
            suites = %(suites)s,
            main_package = %(main_package)s,
        )"""


def _generate_category_def(controls: List[control_files.Control]) -> str:
  controls = sorted(controls, key=lambda c: c.name)
  test_list = ','.join([
      _TEST_TEMPLATE % {
          'name': "'%s/%s'" % (c.category, c.name),
          'suites': sorted(c.suites),
          'main_package': "'%s'" % c.main_package,
      } for c in controls
  ])
  return _CATEGORY_TEMPLATE % {'test_list': test_list}


_ALL_CATEGORIES_TEMPLATE = """
# Copyright 2020 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# !!! GENERATED FILE. DO NOT EDIT !!!

%(load_inclusions)s

def define_tests():
    tests = []
    %(append_inclusions)s
    return tests
"""

_LOAD_CATEGORY_TEMPLATE = """
load('%(path)s', define_%(name)s = 'define_tests')"""

_APPEND_CATEGORY_TEMPLATE = """
    tests += define_%(name)s()"""


def _generate_all_tests_def(categories: List[str]) -> str:
  load_inclusions = [
      _LOAD_CATEGORY_TEMPLATE % {
          'name': c,
          'path': '%s/%s.star' % (_CATEGORY_DEFS_PACKAGE, c),
      } for c in categories
  ]
  append_inclusions = [
      _APPEND_CATEGORY_TEMPLATE % {
          'name': c
      } for c in categories
  ]
  return _ALL_CATEGORIES_TEMPLATE % {
      'load_inclusions': ''.join(load_inclusions),
      'append_inclusions': ''.join(append_inclusions),
  }


if __name__ == '__main__':
  main()