| # Copyright 2012 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Unittests for config.""" |
| |
| import copy |
| import json |
| import re |
| from unittest import mock |
| |
| from chromite.cbuildbot import builders |
| from chromite.cbuildbot.builders import generic_builders |
| from chromite.config import chromeos_config |
| from chromite.config import chromeos_test_config as chromeos_test |
| from chromite.format import formatters |
| from chromite.lib import config_lib |
| from chromite.lib import constants |
| from chromite.lib import cros_build_lib |
| from chromite.lib import cros_test_lib |
| from chromite.lib import git |
| from chromite.lib import osutils |
| |
| |
| # pylint: disable=protected-access |
| |
| |
| class ChromeosConfigTestBase(cros_test_lib.TestCase): |
| """Base class for tests of chromeos_config..""" |
| |
| def setUp(self): |
| self.site_config = chromeos_config.GetConfig() |
| |
| def isReleaseBranch(self): |
| ge_build_config = config_lib.LoadGEBuildConfigFromFile() |
| return ge_build_config["release_branch"] |
| |
| |
| class ConfigDumpTest(ChromeosConfigTestBase): |
| """Tests related to config_dump.json & chromeos_config.py""" |
| |
| def testDump(self): |
| """Ensure generated files are up to date.""" |
| # config_dump.json |
| new_dump = self.site_config.SaveConfigToString() |
| old_dump = osutils.ReadFile(constants.CHROMEOS_CONFIG_FILE) |
| |
| if new_dump != old_dump: |
| self.fail( |
| "config_dump.json does not match the defined configs. Run " |
| "config/refresh_generated_files" |
| ) |
| |
| # watefall_layout_dump.txt |
| # We run this as a sep program to avoid the config cache. |
| cmd = constants.CHROMITE_BIN_DIR / "cros_show_waterfall_layout" |
| result = cros_build_lib.run( |
| [cmd], capture_output=True, encoding="utf-8" |
| ) |
| |
| # Capturing cros_show_waterfall_layout gives 2 newlines at the end, but |
| # cros format wants 1, which refresh_generated_files uses to prevent |
| # presubmit hook errors, so format the data. |
| new_dump_raw = result.stdout |
| new_dump = formatters.whitespace.Data(new_dump_raw) |
| # Quick verification of above comment. |
| self.assertEqual(new_dump_raw.strip(), new_dump.strip()) |
| old_dump = osutils.ReadFile(constants.WATERFALL_CONFIG_FILE) |
| |
| if new_dump != old_dump: |
| self.fail( |
| "waterfall_layout_dump.txt does not match the defined configs. " |
| "Run config/refresh_generated_files" |
| ) |
| |
| # luci-scheduler.cfg |
| # We run this as a sep program to avoid the config cache. |
| cmd = constants.CHROMITE_DIR / "scripts" / "gen_luci_scheduler" |
| result = cros_build_lib.run( |
| [cmd], capture_output=True, encoding="utf-8" |
| ) |
| |
| new_dump = result.stdout |
| old_dump = osutils.ReadFile(constants.LUCI_SCHEDULER_CONFIG_FILE) |
| |
| if new_dump != old_dump: |
| self.fail( |
| "luci-scheduler.cfg does not match the defined configs. Run " |
| "config/refresh_generated_files" |
| ) |
| |
| def testSaveLoadReload(self): |
| """Make sure that loading and reloading the config is a no-op.""" |
| site_config_str = self.site_config.SaveConfigToString() |
| loaded = config_lib.LoadConfigFromString(site_config_str) |
| |
| self.longMessage = True |
| for name in self.site_config: |
| self.assertDictEqual(loaded[name], self.site_config[name], name) |
| |
| # This includes templates and the default build config. |
| self.assertEqual(self.site_config, loaded) |
| |
| loaded_str = loaded.SaveConfigToString() |
| |
| self.assertEqual(site_config_str, loaded_str) |
| |
| # Cycle through save load again, just for completeness. |
| loaded2 = config_lib.LoadConfigFromString(loaded_str) |
| loaded2_str = loaded2.SaveConfigToString() |
| self.assertEqual(loaded_str, loaded2_str) |
| |
| def testFullDump(self): |
| """Make sure we can dump long content without crashing.""" |
| # Note: This test takes ~ 1 second to run. |
| self.site_config.DumpExpandedConfigToString() |
| |
| |
| class FindConfigsForBoardTest(cros_test_lib.TestCase): |
| """Test locating of official build for a board. |
| |
| This test class used to live in config_lib_unittest, but was moved |
| here to help make lib/ hermetic and not depend on chromite/cbuildbot. |
| """ |
| |
| def setUp(self): |
| self.config = chromeos_config.GetConfig() |
| |
| def _CheckFullConfig( |
| self, board, external_expected=None, internal_expected=None |
| ): |
| """Check FindFullConfigsForBoard has expected results. |
| |
| Args: |
| board: Argument to pass to FindFullConfigsForBoard. |
| external_expected: Expected config name (singular) to be found. |
| internal_expected: Expected config name (singular) to be found. |
| """ |
| |
| def check_expected(l, expected): |
| if expected is not None: |
| self.assertTrue(expected in [v["name"] for v in l]) |
| |
| external, internal = self.config.FindFullConfigsForBoard(board) |
| self.assertFalse( |
| external_expected is None and internal_expected is None |
| ) |
| check_expected(external, external_expected) |
| check_expected(internal, internal_expected) |
| |
| def _CheckCanonicalConfig(self, board, ending): |
| self.assertEqual( |
| "-".join((board, ending)), |
| self.config.FindCanonicalConfigForBoard(board)["name"], |
| ) |
| |
| def testExternal(self): |
| """Test finding of a full builder.""" |
| self._CheckFullConfig( |
| "amd64-generic", external_expected="amd64-generic-full" |
| ) |
| |
| def testInternal(self): |
| """Test finding of a release builder.""" |
| self._CheckFullConfig("eve", internal_expected="eve-release") |
| |
| def testBoth(self): |
| """Both an external and internal config exist for board.""" |
| self._CheckFullConfig( |
| "nocturne", |
| external_expected="nocturne-full", |
| internal_expected="nocturne-release", |
| ) |
| |
| def testExternalCanonicalResolution(self): |
| """Test an external canonical config.""" |
| self._CheckCanonicalConfig("amd64-generic", "full") |
| |
| def testInternalCanonicalResolution(self): |
| """Test prefer internal over external when both exist.""" |
| self._CheckCanonicalConfig("nocturne", "release") |
| |
| def testAFDOCanonicalResolution(self): |
| """Test prefer non-AFDO over AFDO builder.""" |
| self._CheckCanonicalConfig("eve", "release") |
| |
| def testOneFullConfigPerBoard(self): |
| """There is at most one 'full' config for a board.""" |
| |
| # Verifies the number of external 'full' and internal 'release' build |
| # per board. This is to ensure that we fail any new configs that |
| # wrongly have names like *-bla-release or *-bla-full. This case can |
| # also be caught if the new suffix was added to |
| # config_lib.CONFIG_TYPE_DUMP_ORDER (see testNonOverlappingConfigTypes), |
| # but that's not guaranteed to happen. |
| def AtMostNumConfigs(board, label, configs, number): |
| if len(configs) > number: |
| self.fail( |
| "Found more than one %s config for %s: %r" |
| % (label, board, [c["name"] for c in configs]) |
| ) |
| |
| boards = set() |
| for build_config in self.config.values(): |
| boards.update(build_config["boards"]) |
| |
| # Sanity check of the boards. |
| self.assertTrue(boards) |
| |
| for b in boards: |
| numExternal = 2 if b == "amd64-generic" else 1 |
| external, internal = self.config.FindFullConfigsForBoard(b) |
| AtMostNumConfigs(b, "external", external, numExternal) |
| AtMostNumConfigs(b, "internal", internal, 1) |
| |
| |
| class UnifiedBuildConfigTestCase(object): |
| """Base test class that builds a fake unibuild config model.""" |
| |
| def setUp(self): |
| # Code assumes at least one non-unified build exists, so we're |
| # accommodating that by keeping the non-unified reef board. |
| self._fake_ge_build_config_json = """ |
| { |
| "metadata_version": "1.0", |
| "release_branch": true, |
| "reference_board_unified_builds": [ |
| { |
| "name": "coral", |
| "reference_board_name": "coral", |
| "builder": "RELEASE", |
| "experimental": true, |
| "arch": "X86_INTERNAL", |
| "models" : [ |
| { |
| "name": "coral", |
| "board_name": "coral" |
| }, |
| { |
| "name": "robo", |
| "board_name": "robo", |
| "test_suites": ["sanity"], |
| "cq_test_enabled": true |
| } |
| ] |
| } |
| ], |
| "boards": [ |
| { |
| "name": "reef", |
| "configs": [ |
| { |
| "builder": "RELEASE", |
| "experimental": false, |
| "leader_board": true, |
| "board_group": "reef", |
| "arch": "X86_INTERNAL" |
| } |
| ] |
| } |
| ] |
| } |
| """ |
| self._fake_ge_build_config = json.loads(self._fake_ge_build_config_json) |
| |
| defaults = chromeos_config.DefaultSettings() |
| self._site_config = config_lib.SiteConfig(defaults=defaults) |
| self._ge_build_config = config_lib.LoadGEBuildConfigFromFile() |
| self._boards_dict = chromeos_config.GetBoardTypeToBoardsDict( |
| self._ge_build_config |
| ) |
| |
| chromeos_config.GeneralTemplates(self._site_config) |
| chromeos_test.GeneralTemplates( |
| self._site_config, self._fake_ge_build_config |
| ) |
| chromeos_config.ReleaseBuilders( |
| self._site_config, self._boards_dict, self._fake_ge_build_config |
| ) |
| |
| |
| class UnifiedBuildReleaseBuilders( |
| cros_test_lib.OutputTestCase, UnifiedBuildConfigTestCase |
| ): |
| """Tests that verify how unified builder configs are generated""" |
| |
| def setUp(self): |
| UnifiedBuildConfigTestCase.setUp(self) |
| |
| def testUnifiedReleaseBuilders(self): |
| coral_release = self._site_config["coral-release"] |
| self.assertIsNotNone(coral_release) |
| models = coral_release["models"] |
| self.assertIn( |
| config_lib.ModelTestConfig("coral", "coral", [], False), models |
| ) |
| self.assertIn( |
| config_lib.ModelTestConfig("robo", "robo", ["sanity"]), models |
| ) |
| |
| master_release = self._site_config["master-release"] |
| self.assertIn("coral-release", master_release["slave_configs"]) |
| |
| |
| class ConfigClassTest(ChromeosConfigTestBase): |
| """Tests of the config class itself.""" |
| |
| def testAppendUseflags(self): |
| base_config = config_lib.BuildConfig(useflags=[]) |
| inherited_config_1 = base_config.derive( |
| useflags=config_lib.append_useflags(["foo", "bar", "-baz"]) |
| ) |
| inherited_config_2 = inherited_config_1.derive( |
| useflags=config_lib.append_useflags(["-bar", "baz"]) |
| ) |
| self.assertEqual(base_config.useflags, []) |
| self.assertEqual(inherited_config_1.useflags, ["-baz", "bar", "foo"]) |
| self.assertEqual(inherited_config_2.useflags, ["-bar", "baz", "foo"]) |
| |
| |
| class CBuildBotTest(ChromeosConfigTestBase): |
| """General tests of chromeos_config.""" |
| |
| def findAllSlaveBuilds(self): |
| """Test helper for finding all slave builds. |
| |
| Returns: |
| Set of slave build config names. |
| """ |
| all_slaves = set() |
| for config in self.site_config.values(): |
| if config.master: |
| all_slaves.update(config.slave_configs) |
| |
| return all_slaves |
| |
| def _GetBoardTypeToBoardsDict(self): |
| """Get boards dict. |
| |
| Returns: |
| A dict mapping a board type to a collections of board names. |
| """ |
| ge_build_config = config_lib.LoadGEBuildConfigFromFile() |
| return chromeos_config.GetBoardTypeToBoardsDict(ge_build_config) |
| |
| def testConfigsKeysMismatch(self): |
| """Verify that all configs contain exactly the default keys. |
| |
| This checks for mispelled keys, or keys that are somehow removed. |
| """ |
| expected_keys = set(self.site_config.GetDefault()) |
| for build_name, config in self.site_config.items(): |
| config_keys = set(config) |
| |
| extra_keys = config_keys.difference(expected_keys) |
| self.assertFalse( |
| extra_keys, |
| ( |
| "Config %s has extra values %s" |
| % (build_name, list(extra_keys)) |
| ), |
| ) |
| |
| missing_keys = expected_keys.difference(config_keys) |
| self.assertFalse( |
| missing_keys, |
| ( |
| "Config %s is missing values %s" |
| % (build_name, list(missing_keys)) |
| ), |
| ) |
| |
| def testConfigsHaveName(self): |
| """Configs must have names set.""" |
| for build_name, config in self.site_config.items(): |
| self.assertTrue(build_name == config["name"]) |
| |
| def testConfigsHaveValidDisplayLabel(self): |
| """Configs must have names set.""" |
| for build_name, config in self.site_config.items(): |
| self.assertIn( |
| config.display_label, |
| config_lib.ALL_DISPLAY_LABEL, |
| 'Invalid display_label "%s" on "%s"' |
| % (config.display_label, build_name), |
| ) |
| |
| def testConfigsHaveValidLuciBuilder(self): |
| """Configs must have names set.""" |
| for build_name, config in self.site_config.items(): |
| self.assertIn( |
| config.luci_builder, |
| config_lib.ALL_LUCI_BUILDER, |
| 'Invalid luci_builder "%s" on "%s"' |
| % (config.luci_builder, build_name), |
| ) |
| |
| def testMasterSlaveConfigsExist(self): |
| """Configs listing slave configs, must list valid configs.""" |
| for config in self.site_config.values(): |
| if config.master: |
| # Any builder with slaves must set both of these. |
| self.assertTrue(config.master) |
| self.assertIsNotNone(config.slave_configs) |
| |
| # If a builder lists slave config names, ensure they are all |
| # valid, and have an assigned waterfall. |
| for slave_name in config.slave_configs: |
| self.assertIn(slave_name, self.site_config) |
| else: |
| self.assertIsNone(config.slave_configs) |
| |
| def testMasterSlaveConfigsSorted(self): |
| """Configs listing slave configs, must list valid configs.""" |
| for config in self.site_config.values(): |
| if config.slave_configs is not None: |
| expected = sorted(config.slave_configs) |
| |
| self.assertEqual(config.slave_configs, expected) |
| |
| def testOnlySlaveConfigsNotImportant(self): |
| """Configs listing slave configs, must list valid configs.""" |
| all_slaves = self.findAllSlaveBuilds() |
| |
| for config in self.site_config.values(): |
| self.assertTrue( |
| config.important or config.name in all_slaves, |
| "%s is not marked important, but is not a slave." % config.name, |
| ) |
| |
| def testConfigUseflags(self): |
| """Useflags must be lists. |
| |
| Strings are interpreted as arrays of characters for this, which is not |
| useful. |
| """ |
| for build_name, config in self.site_config.items(): |
| useflags = config.get("useflags") |
| if not useflags is None: |
| self.assertIsInstance( |
| useflags, |
| list, |
| "Config %s: useflags should be a list." % build_name, |
| ) |
| |
| def testBoards(self): |
| """Verify 'boards' is explicitly set for every config.""" |
| for build_name, config in self.site_config.items(): |
| self.assertIsInstance( |
| config["boards"], |
| (tuple, list), |
| "Config %s doesn't have a list of boards." % build_name, |
| ) |
| self.assertEqual( |
| len(set(config["boards"])), |
| len(config["boards"]), |
| "Config %s has duplicate boards." % build_name, |
| ) |
| if config["builder_class_name"] in ( |
| "sdk_builders.ChrootSdkBuilder", |
| "misc_builders.RefreshPackagesBuilder", |
| ): |
| self.assertTrue( |
| len(config["boards"]) >= 1, |
| "Config %s requires 1 or more boards." % build_name, |
| ) |
| |
| def testOverlaySettings(self): |
| """Verify overlays and push_overlays have legal values.""" |
| for build_name, config in self.site_config.items(): |
| overlays = config["overlays"] |
| push_overlays = config["push_overlays"] |
| |
| self.assertTrue( |
| overlays in [None, "public", "private", "both"], |
| "Config %s: has unexpected overlays value." % build_name, |
| ) |
| self.assertTrue( |
| push_overlays in [None, "public", "private", "both"], |
| "Config %s: has unexpected push_overlays value." % build_name, |
| ) |
| |
| if overlays is None: |
| subset = [None] |
| elif overlays == "public": |
| subset = [None, "public"] |
| elif overlays == "private": |
| subset = [None, "private"] |
| elif overlays == "both": |
| subset = [None, "public", "private", "both"] |
| |
| self.assertTrue( |
| push_overlays in subset, |
| ( |
| "Config %s: push_overlays should be a subset of overlays." |
| % build_name |
| ), |
| ) |
| |
| def testOverlayMaster(self): |
| """Verify that only one master is pushing uprevs for each overlay.""" |
| masters = {} |
| for build_name, config in self.site_config.items(): |
| overlays = config["overlays"] |
| push_overlays = config["push_overlays"] |
| if ( |
| overlays |
| and push_overlays |
| and config["uprev"] |
| and config["master"] |
| and not config["branch"] |
| and not config["workspace_branch"] |
| and not config["debug"] |
| ): |
| other_master = masters.get(push_overlays) |
| err_msg = "Found two masters for push_overlays=%s: %s and %s" |
| self.assertFalse( |
| other_master, |
| err_msg % (push_overlays, build_name, other_master), |
| ) |
| masters[push_overlays] = build_name |
| |
| if "both" in masters: |
| self.assertEqual(len(masters), 1, "Found too many masters.") |
| |
| def testChromeRev(self): |
| """Verify chrome_rev has an expected value""" |
| for build_name, config in self.site_config.items(): |
| self.assertTrue( |
| config["chrome_rev"] |
| in constants.VALID_CHROME_REVISIONS + [None], |
| "Config %s: has unexpected chrome_rev value." % build_name, |
| ) |
| self.assertFalse( |
| config["chrome_rev"] == constants.CHROME_REV_LOCAL, |
| "Config %s: has unexpected chrome_rev_local value." |
| % build_name, |
| ) |
| if config["chrome_rev"]: |
| self.assertTrue( |
| config_lib.IsPFQType(config["build_type"]), |
| "Config %s: has chrome_rev but is not a PFQ." % build_name, |
| ) |
| |
| def testValidVMTestType(self): |
| """Verify vm_tests has an expected value""" |
| for build_name, config in self.site_config.items(): |
| if config["vm_tests"] is None: |
| continue |
| for vm_test in config["vm_tests"]: |
| self.assertTrue( |
| vm_test.test_type in constants.VALID_VM_TEST_TYPES, |
| "Config %s: has unexpected vm test type value." |
| % build_name, |
| ) |
| if vm_test.test_type == constants.VM_SUITE_TEST_TYPE: |
| self.assertTrue( |
| vm_test.test_suite is not None, |
| "Config %s: has unexpected vm test suite value." |
| % build_name, |
| ) |
| |
| def testImageTestMustHaveBaseImage(self): |
| """Verify image_test build is only enabled with 'base' in images.""" |
| for build_name, config in self.site_config.items(): |
| if config.get("image_test", False): |
| self.assertTrue( |
| "base" in config["images"], |
| "Build %s runs image_test but does not have base image" |
| % build_name, |
| ) |
| |
| def testDisableHWQualWithoutTestImage(self): |
| """Don't run steps that need a test image, without a test image.""" |
| for build_name, config in self.site_config.items(): |
| if config.hwqual and config.upload_hw_test_artifacts: |
| self.assertIn( |
| "test", |
| config.images, |
| "Build %s must create a test image " |
| "to enable hwqual" % build_name, |
| ) |
| |
| def testBuildType(self): |
| """Verifies that all configs use valid build types.""" |
| for build_name, config in self.site_config.items(): |
| # For builders that have explicit classes, this check doesn't make |
| # sense. |
| if config["builder_class_name"]: |
| continue |
| self.assertIn( |
| config["build_type"], |
| constants.VALID_BUILD_TYPES, |
| "Config %s: has unexpected build_type value." % build_name, |
| ) |
| |
| def testBuildToRun(self): |
| """Verify we don't try to run tests without building them.""" |
| for build_name, config in self.site_config.items(): |
| self.assertFalse( |
| isinstance(config["useflags"], list) |
| and "-build_tests" in config["useflags"] |
| and config["vm_tests"], |
| "Config %s: has vm_tests and use -build_tests." % build_name, |
| ) |
| |
| def testSyncToChromeSdk(self): |
| """Verify none of the configs build chrome sdk but don't sync chrome.""" |
| for build_name, config in self.site_config.items(): |
| if config["sync_chrome"] is not None and not config["sync_chrome"]: |
| self.assertFalse( |
| config["chrome_sdk"], |
| "Config %s: has chrome_sdk but not sync_chrome." |
| % build_name, |
| ) |
| |
| def testOverrideVmTestsOnly(self): |
| """VM/unit tests listed should also be supported.""" |
| for build_name, config in self.site_config.items(): |
| if config.vm_tests_override is not None: |
| for test in config.vm_tests: |
| self.assertIn( |
| test, |
| config.vm_tests_override, |
| "Config %s: has %s VM test, not in override (%s, %s)." |
| % ( |
| build_name, |
| test, |
| config.vm_tests, |
| config.vm_tests_override, |
| ), |
| ) |
| |
| def testVmTestsOnlyOnVmTestBoards(self): |
| """Verify that only VM capable boards run VM tests.""" |
| for _, config in self.site_config.items(): |
| if config["vm_tests"] or config["vm_tests_override"]: |
| for board in config["boards"]: |
| self.assertIn( |
| board, |
| chromeos_test.vmtest_boards, |
| "Board %s not able to run VM tests." % board, |
| ) |
| for child_config in config.child_configs: |
| if ( |
| child_config["vm_tests"] |
| or child_config["vm_tests_override"] |
| ): |
| for board in config["boards"]: |
| self.assertIn( |
| board, |
| chromeos_test.vmtest_boards, |
| "Board %s not able to run VM tests." % board, |
| ) |
| |
| def testHWTestsArchivingHWTestArtifacts(self): |
| """Verify all configs upload artifacts that need them for hw testing.""" |
| for build_name, config in self.site_config.items(): |
| if config.hw_tests or config.hw_tests_override: |
| self.assertTrue( |
| config.upload_hw_test_artifacts, |
| "%s is trying to run hw tests without uploading payloads." |
| % build_name, |
| ) |
| |
| def testTryjobConfigsDontDefineOverrides(self): |
| """Make sure that no tryjob safe configs define test overrides.""" |
| for build_name, config in self.site_config.items(): |
| if not config_lib.isTryjobConfig(config): |
| continue |
| |
| self.assertIsNone( |
| config.vm_tests_override, |
| "Config %s: is tryjob safe, but defines vm_tests_override." |
| % build_name, |
| ) |
| |
| self.assertIsNone( |
| config.hw_tests_override, |
| "Config %s: is tryjob safe, but defines hw_tests_override." |
| % build_name, |
| ) |
| |
| def testHWTestsReleaseBuilderRequirement(self): |
| """Make sure all release configs run hw tests.""" |
| expected_exceptions = set( |
| ( |
| build_name |
| for build_name, config in self.site_config.items() |
| if config.hw_tests_disabled_bug |
| ) |
| ) |
| missing_tests = set() |
| running_tests = set() |
| for build_name, config in self.site_config.items(): |
| if ( |
| config.build_type == "canary" |
| and "test" in config.images |
| and config.upload_hw_test_artifacts |
| and config.hwqual |
| ): |
| check_name = build_name |
| # Release tryjobs match their release job. |
| if "-release-tryjob" in check_name: |
| check_name = check_name.replace("-tryjob", "") |
| if ( |
| check_name.startswith("betty-") |
| or check_name.startswith("novato-") |
| or check_name.startswith("amd64-generic-") |
| ): |
| # Betty is vm-only, so never does hardware tests. See |
| # crbug/998427. |
| continue |
| elif check_name not in expected_exceptions: |
| # If it's not listed as an exception, it needs to run |
| # hardware tests. |
| if not config.hw_tests and not config.hw_tests_disabled_bug: |
| missing_tests.add(build_name) |
| elif config.hw_tests: |
| # It is listed as an exception, and it is running hardware |
| # tests. It must be removed from the exceptions list. |
| running_tests.add(build_name) |
| # Assert at the end, so that we can print the entire list. |
| self.assertEqual( |
| set(), |
| running_tests, |
| "Expected no hw_tests, but found them: %s" % running_tests, |
| ) |
| self.assertEqual( |
| set(), |
| missing_tests, |
| "Builds must run hardware tests: %s" % missing_tests, |
| ) |
| |
| def testHWTestsReleaseBuilderWeakRequirement(self): |
| """Make sure most release configs run hw tests.""" |
| for build_name, config in self.site_config.items(): |
| if config.hw_tests_disabled_bug: |
| continue |
| if ( |
| config.build_type == "canary" |
| and "test" in config.images |
| and config.upload_hw_test_artifacts |
| and config.hwqual |
| ): |
| self.assertTrue( |
| config.hw_tests, |
| "Release builder %s must run hw tests." % build_name, |
| ) |
| |
| def testValidUnifiedMasterConfig(self): |
| """Make sure any unified master configurations are valid.""" |
| for build_name, config in self.site_config.items(): |
| error = "Unified config for %s has invalid values" % build_name |
| # Unified masters must be internal and must rev both overlays. |
| if config["master"] and config["manifest_version"]: |
| self.assertTrue(config["internal"], error) |
| elif not config["master"] and config["manifest_version"]: |
| # Unified slaves can rev either public or both depending on |
| # whether they are internal or not. |
| if not config["internal"]: |
| self.assertEqual( |
| config["overlays"], constants.PUBLIC_OVERLAYS, error |
| ) |
| |
| def testGetSlaves(self): |
| """Make sure every master has a valid list of slaves""" |
| for build_name, config in self.site_config.items(): |
| if config.master: |
| configs = self.site_config.GetSlavesForMaster(config) |
| self.assertEqual( |
| len(configs), |
| len(set(repr(x) for x in configs)), |
| "Duplicate board in slaves of %s will cause upload " |
| "prebuilts failures" % build_name, |
| ) |
| |
| def _getSlaveConfigsForMaster(self, master_config_name): |
| """Helper to fetch the configs for all slaves of a given master.""" |
| master_config = self.site_config[master_config_name] |
| |
| # Get a list of all active Paladins. |
| return [self.site_config[n] for n in master_config.slave_configs] |
| |
| def testGetSlavesOnTrybot(self): |
| """Make sure every master has a valid list of slaves""" |
| mock_options = mock.Mock() |
| mock_options.remote_trybot = True |
| for _, config in self.site_config.items(): |
| if config["master"]: |
| configs = self.site_config.GetSlavesForMaster( |
| config, mock_options |
| ) |
| self.assertEqual([], configs) |
| |
| def testFactoryFirmwareValidity(self): |
| """Ensures that firmware/factory branches have at least 1 valid name.""" |
| tracking_branch = git.GetChromiteTrackingBranch() |
| for branch in ["firmware", "factory"]: |
| if tracking_branch.startswith(branch): |
| saw_config_for_branch = False |
| for build_name in self.site_config: |
| if build_name.endswith("-%s" % branch): |
| self.assertFalse( |
| "release" in build_name, |
| "Factory|Firmware release builders should not " |
| "contain release in their name.", |
| ) |
| saw_config_for_branch = True |
| |
| self.assertTrue( |
| saw_config_for_branch, |
| "No config found for %s branch. As this is the %s branch, " |
| "all release configs that are being used must end in %s." |
| % (branch, tracking_branch, branch), |
| ) |
| |
| def testNoNewBuildersOnlyGroups(self): |
| """Grouped builders are deprecated. |
| |
| Ensure now new users are created. See crbug.com/691810. |
| """ |
| for build_name, config in self.site_config.items(): |
| # These group builders are allowlisted, for now. |
| if not ( |
| build_name |
| in ( |
| "test-ap-group", |
| "test-ap-group-tryjob", |
| "mixed-wificell-pre-cq", |
| ) |
| or build_name.endswith("release-afdo") |
| or build_name.endswith("release-afdo-tryjob") |
| ): |
| self.assertFalse( |
| config.child_configs, |
| "Unexpected group builder found: %s" % build_name, |
| ) |
| |
| def testAFDOSameInChildConfigs(self): |
| """Verify that 'afdo_use' is the same for all children in a group.""" |
| msg = ( |
| "Child config %s for %s should have same value for afdo_use " |
| "as other children" |
| ) |
| for build_name, config in self.site_config.items(): |
| if build_name.endswith("-group"): |
| prev_value = None |
| self.assertTrue( |
| config.child_configs, |
| "Config %s should have child configs" % build_name, |
| ) |
| for child_config in config.child_configs: |
| if prev_value is None: |
| prev_value = child_config.afdo_use |
| else: |
| self.assertEqual( |
| child_config.afdo_use, |
| prev_value, |
| msg % (child_config.name, build_name), |
| ) |
| |
| def testNoGrandChildConfigs(self): |
| """Verify that no child configs have a child config.""" |
| for build_name, config in self.site_config.items(): |
| for child_config in config.child_configs: |
| for grandchild_config in child_config.child_configs: |
| self.fail( |
| "Config %s has grandchild %s" |
| % (build_name, grandchild_config.name) |
| ) |
| |
| def testUseChromeLKGMImpliesInternal(self): |
| """Currently use_chrome_lkgm refers only to internal manifests.""" |
| for build_name, config in self.site_config.items(): |
| if config["use_chrome_lkgm"]: |
| self.assertTrue( |
| config["internal"], |
| "Chrome lkgm currently only works with an internal " |
| "manifest: %s" % (build_name,), |
| ) |
| |
| def _HasValidSuffix(self, config_name, config_types): |
| """Given a config_name, see if it has a suffix in config_types. |
| |
| Args: |
| config_name: Name of config to compare. |
| config_types: A tuple/list of config suffixes. |
| |
| Returns: |
| True, if the config has a suffix matching one of the types. |
| """ |
| for config_type in config_types: |
| if ( |
| config_name.endswith("-" + config_type) |
| or config_name == config_type |
| ): |
| return True |
| |
| return False |
| |
| def testCantBeBothTypesOfAFDO(self): |
| """Using afdo_generate and afdo_use together doesn't work.""" |
| for config in self.site_config.values(): |
| self.assertFalse(config["afdo_use"] and config["afdo_generate"]) |
| self.assertFalse(config["afdo_use"] and config["afdo_generate_min"]) |
| self.assertFalse( |
| config["afdo_generate"] and config["afdo_generate_min"] |
| ) |
| |
| def testValidPrebuilts(self): |
| """Verify all builders have valid prebuilt values.""" |
| for build_name, config in self.site_config.items(): |
| msg = "Config %s: has unexpected prebuilts value." % build_name |
| valid_values = (False, constants.PRIVATE, constants.PUBLIC) |
| self.assertTrue(config["prebuilts"] in valid_values, msg) |
| |
| def testValidHWTestPriority(self): |
| """Verify that hw test priority is valid.""" |
| for build_name, config in self.site_config.items(): |
| for test_config in config["hw_tests"]: |
| self.assertTrue( |
| test_config.priority in constants.HWTEST_VALID_PRIORITIES, |
| "%s has an invalid hwtest priority." % build_name, |
| ) |
| |
| def testPushImagePaygenDependancies(self): |
| """Paygen requires PushImage.""" |
| for build_name, config in self.site_config.items(): |
| # paygen can't complete without push_image, except for payloads |
| # where --channel arguments meet the requirements. |
| if config["paygen"]: |
| self.assertTrue( |
| config["push_image"] |
| or config["build_type"] == constants.PAYLOADS_TYPE, |
| "%s has paygen without push_image" % build_name, |
| ) |
| |
| def testPaygenTestDependancies(self): |
| """paygen testing requires upload_hw_test_artifacts.""" |
| for build_name, config in self.site_config.items(): |
| # This requirement doesn't apply to payloads(-tryjob) builds. |
| # Payloads(-tryjob) are using artifacts from a previous build. |
| if build_name.endswith("-payloads") or build_name.endswith( |
| "-payloads-tryjob" |
| ): |
| continue |
| |
| if config["paygen"] and not config["paygen_skip_testing"]: |
| self.assertTrue( |
| config["upload_hw_test_artifacts"], |
| "%s is not upload_hw_test_artifacts, but also not" |
| " paygen_skip_testing" % build_name, |
| ) |
| |
| def testPayloadImageIsBuilt(self): |
| for build_name, config in self.site_config.items(): |
| if config.payload_image is not None: |
| self.assertNotEqual( |
| "recovery", |
| config.payload_image, |
| "%s wants to generate payloads from recovery " |
| "images, which is not allowed." % build_name, |
| ) |
| self.assertIn( |
| config.payload_image, |
| config.images, |
| "%s builds payloads from %s, which is not in images " |
| "list %s" |
| % (build_name, config.payload_image, config.images), |
| ) |
| |
| def testBuildPackagesForRecoveryImage(self): |
| """Tests that we build the packages required for recovery image.""" |
| for build_name, config in self.site_config.items(): |
| if "recovery" in config.images: |
| if not config.packages: |
| # No packages are specified. Defaults to build all packages. |
| continue |
| |
| self.assertIn( |
| "chromeos-base/chromeos-initramfs", |
| config.packages, |
| "%s does not build chromeos-initramfs, which is required " |
| "for creating the recovery image" % build_name, |
| ) |
| |
| def testBuildRecoveryImageFlags(self): |
| """Ensure the right flags are disabled building a recovery image.""" |
| incompatible_flags = ["paygen", "signer_tests"] |
| for build_name, config in self.site_config.items(): |
| for flag in incompatible_flags: |
| if ( |
| config[flag] |
| and config.build_type != constants.PAYLOADS_TYPE |
| ): |
| self.assertIn( |
| "recovery", |
| config.images, |
| "%s does not build the recovery image, which is " |
| "incompatible with %s=True" % (build_name, flag), |
| ) |
| |
| def testBuildBaseImageForRecoveryImage(self): |
| """Tests that we build the packages required for recovery image.""" |
| for build_name, config in self.site_config.items(): |
| if "recovery" in config.images: |
| self.assertIn( |
| "base", |
| config.images, |
| "%s does not build the base image, which is required for " |
| "building the recovery image" % build_name, |
| ) |
| |
| def testExternalConfigsDoNotUseInternalFeatures(self): |
| """External configs should not use chrome_internal, or official.xml.""" |
| msg = ( |
| "%s is not internal, so should not use chrome_internal, or an " |
| "internal manifest" |
| ) |
| for build_name, config in self.site_config.items(): |
| if not config["internal"]: |
| self.assertFalse( |
| "chrome_internal" in config["useflags"], msg % build_name |
| ) |
| self.assertNotEqual( |
| config.get("manifest"), |
| constants.OFFICIAL_MANIFEST, |
| msg % build_name, |
| ) |
| |
| def testNoShadowedUseflags(self): |
| """Configs should not have both useflags x and -x.""" |
| msg = "%s contains useflag %s and -%s." |
| for build_name, config in self.site_config.items(): |
| useflag_set = set(config["useflags"]) |
| for flag in useflag_set: |
| if not flag.startswith("-"): |
| self.assertFalse( |
| "-" + flag in useflag_set, |
| msg % (build_name, flag, flag), |
| ) |
| |
| def testHealthCheckEmails(self): |
| """Configs should only have valid email addresses or aliases""" |
| msg = "%s contains an invalid tree alias or email address: %s" |
| for build_name, config in self.site_config.items(): |
| health_alert_recipients = config["health_alert_recipients"] |
| for recipient in health_alert_recipients: |
| self.assertTrue( |
| re.match(r"[^@]+@[^@]+\.[^@]+", recipient) |
| or recipient == constants.CHROME_GARDENER, |
| msg % (build_name, recipient), |
| ) |
| |
| def testCheckBuilderClass(self): |
| """Verify builder_class_name is a valid value.""" |
| for build_name, config in self.site_config.items(): |
| builder_class_name = config["builder_class_name"] |
| if builder_class_name is None: |
| continue |
| |
| cls = builders.GetBuilderClass(builder_class_name) |
| self.assertTrue( |
| issubclass(cls, generic_builders.Builder), |
| msg=("config %s has a broken builder_class_name" % build_name), |
| ) |
| |
| def testDistinctBoardSets(self): |
| """Verify that distinct board sets are distinct.""" |
| boards_dict = self._GetBoardTypeToBoardsDict() |
| # Every board should be in exactly one of the distinct board sets. |
| for board in boards_dict["all_boards"]: |
| found = False |
| for s in boards_dict["distinct_board_sets"]: |
| if board in s: |
| if found: |
| assert False, "%s in multiple board sets." % board |
| else: |
| found = True |
| if not found: |
| assert False, "%s in no board sets" % board |
| for s in boards_dict["distinct_board_sets"]: |
| for board in s - boards_dict["all_boards"]: |
| assert False, ( |
| "%s in distinct_board_sets but not in all_boards" % board |
| ) |
| |
| def testCanaryBuildTimeouts(self): |
| """Verify we get the expected timeout values.""" |
| msg = "%s doesn't have expected timout: (%s != %s)" |
| for build_name, config in self.site_config.items(): |
| if config.build_type != constants.CANARY_TYPE: |
| continue |
| expected = 18 * 60 * 60 |
| |
| self.assertEqual( |
| config.build_timeout, |
| expected, |
| msg % (build_name, config.build_timeout, expected), |
| ) |
| |
| def testBuildTimeouts(self): |
| """Verify that timeout values are sensible.""" |
| for build_name, config in self.site_config.items(): |
| # Chrome infra has a hard limit of 24h. |
| self.assertLessEqual( |
| config.build_timeout, |
| 24 * 60 * 60, |
| "%s timeout %s is greater than 24h" |
| % (build_name, config.build_timeout), |
| ) |
| |
| def testLuciScheduler(self): |
| """LUCI Scheduler entries only work for swarming builds.""" |
| for config in self.site_config.values(): |
| if config.schedule is not None: |
| # TODO: validate the scheduler syntax. |
| self.assertIsInstance(config.schedule, str) |
| |
| if config.triggered_gitiles is not None: |
| self.assertEqual( |
| config.schedule, |
| "triggered", |
| "triggered_gitiles requires triggered schedule on config %s" |
| % config.name, |
| ) |
| |
| try: |
| for trigger in config.triggered_gitiles: |
| gitiles_url = trigger[0] |
| ref_list = trigger[1] |
| self.assertIsInstance(gitiles_url, str) |
| for ref in ref_list: |
| self.assertIsInstance(ref, str) |
| if len(trigger) > 2: |
| for path_regexp in trigger[2]: |
| self.assertIsInstance(path_regexp, str) |
| except (TypeError, ValueError): |
| self.fail( |
| ( |
| "%s has a triggered_gitiles that is malformed: %r\n" |
| "Simple example: [['url', ['refs/heads/main']]]" |
| ) |
| % (config.name, config.triggered_gitiles) |
| ) |
| |
| def testNotificationConfigsType(self): |
| """Verify notification_configs has an expected value""" |
| for config in self.site_config.values(): |
| if config["notification_configs"] is None: |
| continue |
| for notification_config in config["notification_configs"]: |
| self.assertIsInstance( |
| notification_config, config_lib.NotificationConfig |
| ) |
| |
| |
| class TemplateTest(ChromeosConfigTestBase): |
| """Tests for templates.""" |
| |
| def testConfigNamesMatchTemplate(self): |
| """Test that all configs have names that match their templates.""" |
| for name, config in self.site_config.items(): |
| # Rapid builders are special snowflakes that are release-tryjobs but |
| # scheduled as a priority builder. |
| if name.endswith("-rapid"): |
| return |
| # Tryjob configs should be tested based on what they are mirrored |
| # from. |
| if name.endswith("-tryjob"): |
| name = name[: -len("-tryjob")] |
| |
| template = config._template |
| if template: |
| # We mix '-' and '_' in various name spaces. |
| name = name.replace("_", "-") |
| template = template.replace("_", "-") |
| child_configs = config.child_configs |
| if not child_configs: |
| msg = "%s should end with %s to match its template" |
| self.assertTrue( |
| name.endswith(template), msg % (name, template) |
| ) |
| else: |
| msg = ( |
| "Child config of %s has name that does not match its " |
| "template" |
| ) |
| self.assertTrue( |
| child_configs[0].name.endswith(template), msg % name |
| ) |
| |
| for other in self.site_config.GetTemplates(): |
| if name.endswith(other) and other != template: |
| if template: |
| msg = "%s has more specific template: %s" % ( |
| name, |
| other, |
| ) |
| self.assertGreater(len(template), len(other), msg) |
| else: |
| msg = "%s should have %s as template" % (name, other) |
| self.assertFalse(name, msg) |
| |
| |
| class BoardConfigsTest(ChromeosConfigTestBase): |
| """Tests for the per-board templates.""" |
| |
| def setUp(self): |
| ge_build_config = config_lib.LoadGEBuildConfigFromFile() |
| boards_dict = chromeos_config.GetBoardTypeToBoardsDict(ge_build_config) |
| |
| self.external_board_configs = chromeos_config.CreateBoardConfigs( |
| self.site_config, boards_dict, ge_build_config |
| ) |
| |
| self.internal_board_configs = ( |
| chromeos_config.CreateInternalBoardConfigs( |
| self.site_config, boards_dict, ge_build_config |
| ) |
| ) |
| |
| def testBoardConfigsSuperset(self): |
| """Ensure all external boards are listed as internal, also.""" |
| for board in self.external_board_configs: |
| self.assertIn(board, self.internal_board_configs) |
| |
| def _verifyNoTests(self, board_configs): |
| """Defining tests in board specific templates doesn't work.""" |
| for board, template in board_configs.items(): |
| self.assertFalse( |
| "vm_tests" in template and template.vm_tests, |
| "Per-board template for %s defining vm_tests" % board, |
| ) |
| self.assertFalse( |
| "vm_tests_override" in template and template.vm_tests_override, |
| "Per-board template for %s defining vm_tests_override" % board, |
| ) |
| self.assertFalse( |
| "hw_tests" in template and template.hw_tests, |
| "Per-board template for %s defining hw_tests" % board, |
| ) |
| self.assertFalse( |
| "hw_tests_override" in template and template.hw_tests_override, |
| "Per-board template for %s defining hw_tests_override" % board, |
| ) |
| |
| def testExternalsDontDefineTests(self): |
| """Verify no external boards define tests at the board level.""" |
| self._verifyNoTests(self.external_board_configs) |
| |
| def testInternalsDontDefineTests(self): |
| """Verify no internal boards define tests at the board level.""" |
| self._verifyNoTests(self.internal_board_configs) |
| |
| def testUpdateBoardConfigs(self): |
| """Test UpdateBoardConfigs.""" |
| pre_test = copy.deepcopy(self.internal_board_configs) |
| update_boards = list(pre_test)[2:5] |
| |
| result = chromeos_config.UpdateBoardConfigs( |
| self.internal_board_configs, |
| update_boards, |
| test_specific_flag=True, |
| ) |
| |
| # The source wasn't modified. |
| self.assertEqual(self.internal_board_configs, pre_test) |
| |
| # The result as the same list of boards. |
| self.assertCountEqual(list(result), list(pre_test)) |
| |
| # And only appropriate values were updated. |
| for b in pre_test: |
| if b in update_boards: |
| # Has new key. |
| self.assertTrue( |
| result[b].test_specific_flag, "Failed in %s" % b |
| ) |
| else: |
| # Was not updated. |
| self.assertEqual(result[b], pre_test[b], "Failed in %s" % b) |