config/config_skew_unittest.py

# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unittests for checking skew between old and new config worlds."""

from __future__ import print_function

import os

from chromite.api.gen.chromiumos import builder_config_pb2
from chromite.config import chromeos_config
from chromite.lib import constants
from chromite.lib import cros_test_lib
from chromite.lib import osutils
from google.protobuf import json_format

BUILDER_CONFIG_FILENAME = os.path.join(
    constants.SOURCE_ROOT, 'infra/config/generated/builder_configs.cfg')

# Builders that we generally leave out of old versus new config comparisons
# as these builders existing in new config have no equivalent in the old
# config world.
# Postsubmit builders to exclude from build_targets.cfg
PS_EXCLUDE_BUILDERS = [
    "chromite-postsubmit",
    "grunt-unittest-only-postsubmit",
    "moblab-generic-vm-postsubmit",
]

# Parallel CQ builders to exclude in comparison.
PCQ_EXCLUDE_BUILDERS = [
    "chromite-cq",
]

# Legacy CQ builders to exclude in comparison.
LCQ_EXCLUDE_BUILDERS = [
    "chell-nowithdebug-paladin",
    "incremental-paladin",
    "falco-full-compile-paladin",
    "nyan_kitty-full-compile-paladin",
]

class ErrorWrapper(Exception):
  """Simple exception wrapper to provide more failure context."""

  def __init__(self, message, cause):
    super(ErrorWrapper, self).__init__(message + ", caused by " + repr(cause))
    self.cause = cause


class ConfigSkewTest(cros_test_lib.TestCase):
  """Tests checking if new config and legacy config are in sync."""

  def __init__(self, *args, **kwargs):
    super(ConfigSkewTest, self).__init__(*args, **kwargs)
    # This is loaded lazily, see self._get_new_configs().
    self._new_configs = None
    self.old_configs = chromeos_config.GetConfig()

  def _get_new_configs(self):
    """Lazily loads and returns the new configs.

    Lazily loads and returns the new approach configuration. This is
    done lazily because this configuration data is not available on
    builders sporting an external manifest. This test only executes
    the test methods if the parameter --config_skew is provided but
    it still constructs this class before the test methods become
    noops. Thus the lazy loading moving the load out of the constructor.
    """
    if self._new_configs is None:
      try:
        self._new_configs = json_format.Parse(
            osutils.ReadFile(BUILDER_CONFIG_FILENAME),
            builder_config_pb2.BuilderConfigs(),
            ignore_unknown_fields=True)
      except IOError as err:
        msg = "IOError, are you running this with an external manifest?"
        raise ErrorWrapper(msg, err)
    return self._new_configs.builder_configs

  def _get_new_config(self, name):
    for config in self._get_new_configs():
      if config.id.name == name:
        return config
    return None

  def _get_new_config_children(self, name, exclude=None):
    """Returns the children for a builder config in new config.

    Args:
      name: string, name of builder to return the children of.
      exclude: optional list, names of children to exclude.
    """
    exclude = exclude or []
    config = self._get_new_config(name)
    return [c for c in config.orchestrator.children if c not in exclude]

  def _get_old_config(self, name):
    return self.old_configs[name.replace("-cq", "-paladin")]

  def _get_old_config_slaves(self, name, exclude=None):
    exclude = exclude or []
    config = self.old_configs[name]
    return [c for c in config.slave_configs if c not in exclude]

  def _to_utf8(self, strings):
    return [string.decode("UTF-8") for string in strings]

  @cros_test_lib.ConfigSkewTest()
  def testPostsubmitBuildTargets(self):
    master_postsubmit_children = self._to_utf8(
        self._get_old_config_slaves("master-postsubmit"))
    # Exclude the special builders that old config is not expected to have.
    postsubmit_orchestrator_children = self._get_new_config_children(
        "postsubmit-orchestrator", PS_EXCLUDE_BUILDERS)

    missing_chromeos_config = (list(set(postsubmit_orchestrator_children) -
                                    set(master_postsubmit_children)))
    missing_build_targets = (list(set(master_postsubmit_children) -
                                  set(postsubmit_orchestrator_children)))
    if missing_chromeos_config:
      self.fail("Build targets need to be added to chromeos_config.py: %s" %
                missing_chromeos_config)
    if missing_build_targets:
      self.fail("Build targets need to be added to build_targets.star: %s" %
                missing_build_targets)

  @cros_test_lib.ConfigSkewTest()
  def testPostsubmitBuildTargetsCriticality(self):
    # Exclude the special builders that old config is not expected to have.
    importance_mismatch = {}
    for child_name in self._get_new_config_children(
        "postsubmit-orchestrator", PS_EXCLUDE_BUILDERS):
      new_config = self._get_new_config(child_name)
      old_config = self._get_old_config(child_name)
      # old_config doesn't exist is caught in another test, don't report here.
      new_critical = new_config.general.critical.value
      old_critical = old_config.important
      if old_config:
        if new_critical != old_critical:
          importance_mismatch.update({child_name:
                                      {'chromeos_config.py': old_critical,
                                       'build_targets.star': new_critical}})
    if len(importance_mismatch) > 0:
      self.fail("Criticality difference in configs: %s" % importance_mismatch)

  @cros_test_lib.ConfigSkewTest()
  def testCqBuildTargetsCriticality(self):
    # Exclude the special builders that old config is not expected to have.
    importance_mismatch = {}
    for child_name in self._get_new_config_children(
        "cq-orchestrator", PCQ_EXCLUDE_BUILDERS):
      new_config = self._get_new_config(child_name)
      old_config = self._get_old_config(child_name.replace("-cq", "-paladin"))
      # old_config doesn't exist is caught in another test, don't report here.
      new_critical = new_config.general.critical.value
      old_critical = old_config.important
      if old_config:
        if new_critical != old_critical:
          importance_mismatch.update({child_name: {
              'chromeos_config.py': old_critical,
              'build_targets.star': new_critical}})
    if len(importance_mismatch) > 0:
      self.fail("Criticality difference in configs: %s" % importance_mismatch)

  @cros_test_lib.ConfigSkewTest()
  def testCqBuildTargets(self):
    master_cq_children = self._to_utf8(
        self._get_old_config_slaves("master-paladin",
                                    LCQ_EXCLUDE_BUILDERS))
    # Exclude the special builders that old config is not expected to have.
    cq_orchestrator_children = self._get_new_config_children(
        "cq-orchestrator", PCQ_EXCLUDE_BUILDERS)
    sync_paladin_names = []
    sync_cq_names = []
    for child in master_cq_children:
      sync_paladin_names.append(child.replace("-paladin", "-cq"))
    for child in cq_orchestrator_children:
      sync_cq_names.append(child.replace("-cq", "-paladin"))
    missing_chromeos_config = (list(set(cq_orchestrator_children) -
                                    set(sync_paladin_names)))
    missing_build_targets = list(set(master_cq_children) - set(sync_cq_names))
    if missing_chromeos_config:
      self.fail("Build targets need to be added to chromeos_config.py: %s" %
                [m.replace("-cq", "-paladin") for m in
                 missing_chromeos_config])
    if missing_build_targets:
      self.fail("Build targets need to be added to build_targets.star: %s" %
                [m.replace("-paladin", "-cq") for m in missing_build_targets])