| # Copyright 2012 The ChromiumOS Authors | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """Configuration options for various cbuildbot builders.""" | |
| import copy | |
| import logging | |
| from chromite.config import chromeos_config_boards as chromeos_boards | |
| from chromite.lib import config_lib | |
| from chromite.lib import constants | |
| from chromite.utils import memoize | |
| def _frozen_ge_set(ge_build_config, values, extras=None): | |
| """Return a frozenset of things in GE.""" | |
| separate_board_names = set( | |
| config_lib.GeBuildConfigAllBoards(ge_build_config) | |
| ) | |
| unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config) | |
| unified_board_names = { | |
| b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME] | |
| for b in unified_builds | |
| } | |
| board_names = separate_board_names | unified_board_names | |
| return frozenset(x for x in values if x in board_names).union( | |
| extras or frozenset() | |
| ) | |
| def add_images(required_images): | |
| """Add required images when applying changes to a BuildConfig. | |
| Used similarly to append_useflags. | |
| Args: | |
| required_images: A list of image names that need to be present in the | |
| final build config. | |
| Returns: | |
| A callable suitable for use with BuildConfig.apply. | |
| """ | |
| required_images = set(required_images) | |
| def handler(old_images): | |
| if not old_images: | |
| old_images = [] | |
| new_images = old_images | |
| for image_name in required_images: | |
| if set(required_images).issubset(new_images): | |
| break | |
| new_images.append(image_name) | |
| return new_images | |
| return handler | |
| def remove_images(unsupported_images): | |
| """Remove unsupported images when applying changes to a BuildConfig. | |
| Used similarly to append_useflags. | |
| Args: | |
| unsupported_images: A list of image names that should not be present in | |
| the final build config. | |
| Returns: | |
| A callable suitable for use with BuildConfig.apply. | |
| """ | |
| unsupported = set(unsupported_images) | |
| def handler(old_images): | |
| if not old_images: | |
| old_images = [] | |
| return [i for i in old_images if i not in unsupported] | |
| return handler | |
| def GetBoardTypeToBoardsDict(ge_build_config): | |
| """Get board type to board names dict. | |
| Args: | |
| ge_build_config: Dictionary containing the decoded GE configuration | |
| file. | |
| Returns: | |
| A dict mapping board types to board name collections. | |
| The dict contains board types including distinct_board_sets, | |
| all_release_boards, all_boards, and internal_boards. | |
| """ | |
| ge_arch_board_dict = config_lib.GetArchBoardDict(ge_build_config) | |
| boards_dict = {} | |
| arm_internal_release_boards = ( | |
| chromeos_boards.arm_internal_release_boards | |
| | ge_arch_board_dict.get(config_lib.CONFIG_ARM_INTERNAL, set()) | |
| ) | |
| arm_external_boards = ( | |
| chromeos_boards.arm_external_boards | |
| | ge_arch_board_dict.get(config_lib.CONFIG_ARM_EXTERNAL, set()) | |
| ) | |
| x86_internal_release_boards = ( | |
| chromeos_boards.x86_internal_release_boards | |
| | ge_arch_board_dict.get(config_lib.CONFIG_X86_INTERNAL, set()) | |
| ) | |
| x86_external_boards = ( | |
| chromeos_boards.x86_external_boards | |
| | ge_arch_board_dict.get(config_lib.CONFIG_X86_EXTERNAL, set()) | |
| ) | |
| # Every board should be in only 1 of the above sets. | |
| boards_dict["distinct_board_sets"] = [ | |
| arm_internal_release_boards, | |
| arm_external_boards, | |
| x86_internal_release_boards, | |
| x86_external_boards, | |
| ] | |
| arm_full_boards = arm_internal_release_boards | arm_external_boards | |
| x86_full_boards = x86_internal_release_boards | x86_external_boards | |
| arm_boards = arm_full_boards | |
| x86_boards = x86_full_boards | |
| boards_dict["all_release_boards"] = ( | |
| arm_internal_release_boards | x86_internal_release_boards | |
| ) | |
| all_boards = x86_boards | arm_boards | |
| boards_dict["all_boards"] = all_boards | |
| boards_dict["internal_boards"] = boards_dict["all_release_boards"] | |
| boards_dict["generic_kernel_boards"] = frozenset( | |
| ["amd64-generic"], | |
| ) | |
| all_ge_boards = set() | |
| for val in ge_arch_board_dict.values(): | |
| all_ge_boards |= val | |
| boards_dict["unknown_boards"] = frozenset(all_ge_boards - all_boards) | |
| return boards_dict | |
| def DefaultSettings(): | |
| """Create the default build config values for this site. | |
| Returns: | |
| dict: of default config_lib.BuildConfig values to use for this site. | |
| """ | |
| # Site specific adjustments for default BuildConfig values. | |
| defaults = config_lib.DefaultSettings() | |
| # Git repository URL for our manifests. | |
| # https://chromium.googlesource.com/chromiumos/manifest | |
| # https://chrome-internal.googlesource.com/chromeos/manifest-internal | |
| defaults["manifest_repo_url"] = config_lib.GetSiteParams().MANIFEST_URL | |
| return defaults | |
| def GeneralTemplates(site_config): | |
| """Defines templates that are shared between categories of builders. | |
| Args: | |
| site_config: A SiteConfig object to add the templates too. | |
| ge_build_config: Dictionary containing the decoded GE configuration | |
| file. | |
| """ | |
| # Builder type templates. | |
| site_config.AddTemplate( | |
| "full", | |
| # Full builds are test builds to show that we can build from scratch, | |
| # so use settings to build from scratch, and archive the results. | |
| usepkg_build_packages=False, | |
| chrome_sdk=True, | |
| build_timeout=12 * 60 * 60, | |
| display_label=config_lib.DISPLAY_LABEL_FULL, | |
| build_type=constants.FULL_TYPE, | |
| luci_builder=config_lib.LUCI_BUILDER_FULL, | |
| archive_build_debug=True, | |
| images=["base", "recovery", "test", "factory_install"], | |
| git_sync=True, | |
| description="Full Builds", | |
| doc=( | |
| "https://dev.chromium.org/chromium-os/build/builder-overview#" | |
| "TOC-Continuous" | |
| ), | |
| ) | |
| site_config.AddTemplate( | |
| "external", | |
| internal=False, | |
| overlays=constants.PUBLIC_OVERLAYS, | |
| manifest_repo_url=config_lib.GetSiteParams().MANIFEST_URL, | |
| manifest=constants.DEFAULT_MANIFEST, | |
| ) | |
| # This builds with more source available. | |
| site_config.AddTemplate( | |
| "internal", | |
| internal=True, | |
| overlays=constants.BOTH_OVERLAYS, | |
| manifest_repo_url=config_lib.GetSiteParams().MANIFEST_INT_URL, | |
| ) | |
| site_config.AddTemplate( | |
| "infra_builder", | |
| luci_builder=config_lib.LUCI_BUILDER_INFRA, | |
| ) | |
| # This adds Chrome branding. | |
| site_config.AddTemplate( | |
| "official_chrome", | |
| useflags=config_lib.append_useflags([constants.USE_CHROME_INTERNAL]), | |
| ) | |
| # This sets chromeos_official. | |
| site_config.AddTemplate( | |
| "official", | |
| site_config.templates.official_chrome, | |
| chromeos_official=True, | |
| ) | |
| site_config.AddTemplate( | |
| "release_common", | |
| site_config.templates.full, | |
| site_config.templates.official, | |
| site_config.templates.internal, | |
| display_label=config_lib.DISPLAY_LABEL_RELEASE, | |
| build_type=constants.CANARY_TYPE, | |
| luci_builder=config_lib.LUCI_BUILDER_LEGACY_RELEASE, | |
| suite_scheduling=True, | |
| # Because release builders never use prebuilts, they need the | |
| # longer timeout. See crbug.com/938958. | |
| build_timeout=18 * 60 * 60, | |
| useflags=config_lib.append_useflags(["-cros-debug", "thinlto"]), | |
| manifest=constants.OFFICIAL_MANIFEST, | |
| manifest_version=True, | |
| images=["base", "recovery", "test", "factory_install"], | |
| sign_types=["recovery", "accessory_rwsig"], | |
| push_image=True, | |
| upload_symbols=True, | |
| run_build_configs_export=True, | |
| binhost_bucket="gs://chromeos-dev-installer", | |
| binhost_key="RELEASE_BINHOST", | |
| binhost_base_url=( | |
| "https://commondatastorage.googleapis.com/chromeos-dev-installer" | |
| ), | |
| git_sync=False, | |
| description="Release Builds (canary) (internal)", | |
| chrome_sdk=True, | |
| doc=( | |
| "https://dev.chromium.org/chromium-os/build/builder-overview#" | |
| "TOC-Canaries" | |
| ), | |
| ) | |
| site_config.AddTemplate( | |
| "release", | |
| site_config.templates.release_common, | |
| luci_builder=config_lib.LUCI_BUILDER_LEGACY_RELEASE, | |
| ) | |
| # Factory releases much inherit from these classes. | |
| # Modifications for these release builders should go here. | |
| # Naming conventions also must be followed. Factory branches must end | |
| # in "-factory". | |
| site_config.AddTemplate( | |
| "factory", | |
| site_config.templates.release_common, | |
| display_label=config_lib.DISPLAY_LABEL_FACTORY, | |
| chrome_sdk=False, | |
| chrome_sdk_build_chrome=False, | |
| description="Factory Builds", | |
| factory_toolkit=True, | |
| images=["test", "factory_install"], | |
| luci_builder=config_lib.LUCI_BUILDER_FACTORY, | |
| sign_types=["factory"], | |
| upload_hw_test_artifacts=True, | |
| upload_symbols=False, | |
| ) | |
| # This should be used by any "workspace_builders." | |
| site_config.AddTemplate( | |
| "workspace", | |
| postsync_patch=False, | |
| ) | |
| site_config.AddTemplate( | |
| "build_external_chrome", | |
| useflags=config_lib.append_useflags( | |
| ["-%s" % constants.USE_CHROME_INTERNAL] | |
| ), | |
| ) | |
| site_config.AddTemplate( | |
| "buildspec", | |
| site_config.templates.workspace, | |
| site_config.templates.internal, | |
| luci_builder=config_lib.LUCI_BUILDER_FACTORY, | |
| master=True, | |
| boards=[], | |
| build_type=constants.GENERIC_TYPE, | |
| uprev=True, | |
| overlays=constants.BOTH_OVERLAYS, | |
| push_overlays=constants.BOTH_OVERLAYS, | |
| builder_class_name="workspace_builders.BuildSpecBuilder", | |
| build_timeout=4 * 60 * 60, | |
| description="Buildspec creator.", | |
| ) | |
| def CreateBoardConfigs(boards_dict, ge_build_config): | |
| """Create mixin templates for each board.""" | |
| # Extract the full list of board names from GE data. | |
| separate_board_names = set( | |
| config_lib.GeBuildConfigAllBoards(ge_build_config) | |
| ) | |
| unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config) | |
| unified_board_names = { | |
| b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME] | |
| for b in unified_builds | |
| } | |
| board_names = separate_board_names | unified_board_names | |
| # TODO(crbug.com/648473): Remove these after GE adds them to their data set. | |
| board_names = board_names.union(boards_dict["all_boards"]) | |
| result = {} | |
| for board in board_names: | |
| board_config = config_lib.BuildConfig(boards=[board]) | |
| if board in chromeos_boards.nofactory_boards: | |
| board_config.apply( | |
| factory=False, | |
| factory_toolkit=False, | |
| factory_install_netboot=False, | |
| images=remove_images(["factory_install"]), | |
| ) | |
| if board in chromeos_boards.builder_incompatible_binaries_boards: | |
| board_config.apply(unittests=False) | |
| result[board] = board_config | |
| return result | |
| def CreateInternalBoardConfigs(site_config, boards_dict, ge_build_config): | |
| """Create mixin templates for each board.""" | |
| result = CreateBoardConfigs(boards_dict, ge_build_config) | |
| for board in boards_dict["internal_boards"]: | |
| if board in result: | |
| result[board].apply( | |
| site_config.templates.internal, | |
| site_config.templates.official_chrome, | |
| manifest=constants.OFFICIAL_MANIFEST, | |
| ) | |
| return result | |
| def UpdateBoardConfigs(board_configs, boards, *args, **kwargs): | |
| """Update "board_configs" for selected chromeos_boards. | |
| Args: | |
| board_configs: Dict in CreateBoardConfigs format to filter from. | |
| boards: Iterable of boards to update in the dict. | |
| *args: List of templates to apply. | |
| **kwargs: Individual keys to update. | |
| Returns: | |
| Copy of board_configs dict with boards boards update with templates | |
| and values applied. | |
| """ | |
| result = board_configs.copy() | |
| for b in boards: | |
| result[b] = result[b].derive(*args, **kwargs) | |
| return result | |
| def FullBuilders(site_config, boards_dict, ge_build_config): | |
| """Create all full builders. | |
| Args: | |
| site_config: config_lib.SiteConfig to be modified by adding templates | |
| and configs. | |
| boards_dict: A dict mapping board types to board name collections. | |
| ge_build_config: Dictionary containing the decoded GE configuration | |
| file. | |
| """ | |
| active_builders = _frozen_ge_set( | |
| ge_build_config, | |
| [], | |
| ("amd64-generic",), | |
| ) | |
| # Move the following builders to active_builders once they are consistently | |
| # green. | |
| unstable_builders = _frozen_ge_set(ge_build_config, []) | |
| external_board_configs = CreateBoardConfigs(boards_dict, ge_build_config) | |
| site_config.AddForBoards( | |
| config_lib.CONFIG_TYPE_FULL, | |
| ["amd64-generic"], | |
| external_board_configs, | |
| site_config.templates.full, | |
| site_config.templates.build_external_chrome, | |
| internal=False, | |
| manifest_repo_url=config_lib.GetSiteParams().MANIFEST_URL, | |
| overlays=constants.PUBLIC_OVERLAYS, | |
| prebuilts=constants.PUBLIC, | |
| ) | |
| master_config = site_config.Add( | |
| "master-full", | |
| site_config.templates.full, | |
| site_config.templates.internal, | |
| site_config.templates.build_external_chrome, | |
| boards=[], | |
| master=True, | |
| manifest_version=True, | |
| overlays=constants.PUBLIC_OVERLAYS, | |
| slave_configs=[], | |
| schedule=None, | |
| ) | |
| master_config.AddSlaves( | |
| site_config.ApplyForBoards( | |
| config_lib.CONFIG_TYPE_FULL, | |
| active_builders, | |
| manifest_version=True, | |
| ) | |
| ) | |
| master_config.AddSlaves( | |
| site_config.ApplyForBoards( | |
| config_lib.CONFIG_TYPE_FULL, | |
| unstable_builders, | |
| manifest_version=True, | |
| important=False, | |
| ) | |
| ) | |
| def FactoryBuilders(site_config, _boards_dict, _ge_build_config): | |
| """Create all factory build configs. | |
| Args: | |
| site_config: config_lib.SiteConfig to be modified by adding templates | |
| and configs. | |
| boards_dict: A dict mapping board types to board name collections. | |
| ge_build_config: Dictionary containing the decoded GE configuration | |
| file. | |
| """ | |
| # pylint: disable=unused-variable | |
| # Intervals: | |
| # None: Do not schedule automatically. | |
| DAILY = "with 24h interval" # 1 day interval | |
| WEEKLY = "with 168h interval" # 1 week interval | |
| MONTHLY = "with 720h interval" # 30 day interval | |
| TRIGGERED = "triggered" # Only when triggered | |
| branch_builders = [ | |
| (MONTHLY, "factory-oak-8182.B", ["elm", "hana"]), | |
| (MONTHLY, "factory-gru-8652.B", ["kevin"]), | |
| (MONTHLY, "factory-gale-8743.19.B", ["gale"]), | |
| (MONTHLY, "factory-reef-8811.B", ["reef", "pyro", "sand", "snappy"]), | |
| (MONTHLY, "factory-gru-9017.B", ["gru", "bob"]), | |
| (MONTHLY, "factory-eve-9667.B", ["eve"]), | |
| (MONTHLY, "factory-coral-10122.B", ["coral"]), | |
| (MONTHLY, "factory-fizz-10167.B", ["fizz"]), | |
| (MONTHLY, "factory-scarlet-10211.B", ["scarlet"]), | |
| (MONTHLY, "factory-soraka-10323.39.B", ["soraka"]), | |
| (MONTHLY, "factory-poppy-10504.B", ["nautilus"]), | |
| (MONTHLY, "factory-nami-10715.B", ["nami", "kalista"]), | |
| (MONTHLY, "factory-nocturne-11066.B", ["nocturne"]), | |
| (MONTHLY, "factory-grunt-11164.B", ["grunt"]), | |
| (MONTHLY, "factory-grunt-11164.135.B", ["grunt"]), | |
| (MONTHLY, "factory-rammus-11289.B", ["rammus"]), | |
| (WEEKLY, "factory-octopus-11512.B", ["octopus"]), | |
| (WEEKLY, "factory-atlas-11907.B", ["atlas"]), | |
| (WEEKLY, "factory-sarien-12033.B", ["sarien"]), | |
| (WEEKLY, "factory-mistral-12361.B", ["mistral"]), | |
| (WEEKLY, "factory-kukui-12587.B", ["kukui", "jacuzzi"]), | |
| (WEEKLY, "factory-hatch-12692.B", ["hatch"]), | |
| (WEEKLY, "factory-excelsior-12812.B", ["excelsior"]), | |
| (WEEKLY, "factory-drallion-13080.B", ["drallion"]), | |
| (MONTHLY, "factory-endeavour-13295.B", ["endeavour"]), | |
| (WEEKLY, "factory-puff-13329.B", ["puff", "puff-moblab"]), | |
| (WEEKLY, "factory-zork-13427.B", ["zork"]), | |
| (WEEKLY, "factory-trogdor-13443.B", ["trogdor", "strongbad"]), | |
| (DAILY, "factory-trogdor-15210.B", ["trogdor", "strongbad"]), | |
| (DAILY, "factory-strongbad-13963.B", ["trogdor", "strongbad"]), | |
| (WEEKLY, "factory-volteer-13600.B", ["volteer"]), | |
| (WEEKLY, "factory-dedede-13683.B", ["dedede"]), | |
| (TRIGGERED, "factory-keeby-14162.B", ["keeby"]), | |
| (WEEKLY, "factory-zork-13700.B", ["zork"]), | |
| (WEEKLY, "factory-puff-13813.B", ["puff"]), | |
| (WEEKLY, "factory-asurada-13929.B", ["asurada"]), | |
| (WEEKLY, "factory-ambassador-14265.B", ["ambassador"]), | |
| (DAILY, "factory-kukui-14374.B", ["kukui", "jacuzzi"]), | |
| (WEEKLY, "factory-cherry-14455.B", ["cherry"]), | |
| (WEEKLY, "factory-brya-14517.B", ["brya", "brask"]), | |
| (WEEKLY, "factory-guybrush-14908.B", ["guybrush"]), | |
| (TRIGGERED, "factory-brya-14909.124.B", ["brya"]), | |
| (WEEKLY, "factory-corsola-15196.B", ["corsola", "staryu"]), | |
| (DAILY, "factory-nissa-15199.B", ["nissa"]), | |
| (DAILY, "factory-brya-15231.B", ["brya", "brask", "constitution"]), | |
| (DAILY, "factory-skyrim-15384.B", ["skyrim"]), | |
| # This is intended to create master branch tryjobs, NOT for production | |
| # builds. Update the associated list of boards as needed. | |
| ( | |
| None, | |
| "master", | |
| [ | |
| "atlas", | |
| "octopus", | |
| "rammus", | |
| "coral", | |
| "eve", | |
| "sarien", | |
| "mistral", | |
| "drallion", | |
| ], | |
| ), | |
| ] | |
| _FACTORYBRANCH_TIMEOUT = 12 * 60 * 60 | |
| # Requires that you set boards, and workspace_branch. | |
| site_config.AddTemplate( | |
| "factorybranch", | |
| site_config.templates.factory, | |
| site_config.templates.workspace, | |
| sign_types=["factory"], | |
| build_type=constants.GENERIC_TYPE, | |
| uprev=True, | |
| overlays=constants.BOTH_OVERLAYS, | |
| push_overlays=constants.BOTH_OVERLAYS, | |
| useflags=config_lib.append_useflags( | |
| ["-cros-debug", "thinlto", "chrome_internal"] | |
| ), | |
| builder_class_name="workspace_builders.FactoryBranchBuilder", | |
| build_timeout=_FACTORYBRANCH_TIMEOUT, | |
| description="TOT builder to build a factory branch.", | |
| doc="https://goto.google.com/tot-for-firmware-branches", | |
| ) | |
| for active, branch, boards in branch_builders: | |
| schedule = {} | |
| if active: | |
| schedule = { | |
| "schedule": active, | |
| } | |
| # Define the buildspec builder for the branch. | |
| branch_master = site_config.Add( | |
| "%s-buildspec" % branch, | |
| site_config.templates.buildspec, | |
| display_label=config_lib.DISPLAY_LABEL_FACTORY, | |
| workspace_branch=branch, | |
| build_timeout=_FACTORYBRANCH_TIMEOUT, | |
| **schedule, | |
| ) | |
| # Define the per-board builders for the branch. | |
| for board in boards: | |
| child = site_config.Add( | |
| "%s-%s-factorybranch" % (board, branch), | |
| site_config.templates.factorybranch, | |
| boards=[board], | |
| workspace_branch=branch, | |
| ) | |
| branch_master.AddSlave(child) | |
| def ReleaseBuilders(site_config, boards_dict, ge_build_config): | |
| """Create all release builders. | |
| Args: | |
| site_config: config_lib.SiteConfig to be modified by adding templates | |
| and configs. | |
| boards_dict: A dict mapping board types to board name collections. | |
| ge_build_config: Dictionary containing the decoded GE configuration | |
| file. | |
| """ | |
| board_configs = CreateInternalBoardConfigs( | |
| site_config, boards_dict, ge_build_config | |
| ) | |
| def _CreateMasterConfig( | |
| name, template=site_config.templates.release, schedule=" 0 12 * * *" | |
| ): | |
| return site_config.Add( | |
| name, | |
| template, | |
| boards=[], | |
| master=True, | |
| slave_configs=[], | |
| sync_chrome=True, | |
| chrome_sdk=False, | |
| # Because PST is 8 hours from UTC, these times are the same in both. | |
| # But daylight savings time is NOT adjusted for | |
| schedule=schedule, | |
| ) | |
| ### Master release configs. | |
| master_config = _CreateMasterConfig("master-release") | |
| def _AssignToMaster(config): | |
| """Add |config| as a slave config to the appropriate master config.""" | |
| # Default to chromeos master release builder. | |
| master_config.AddSlave(config) | |
| ### Release configs. | |
| # Used for future bvt migration. | |
| _release_experimental_boards = _frozen_ge_set( | |
| ge_build_config, | |
| [ | |
| "elm-kernelnext", | |
| "grunt-kernelnext", | |
| "hana-kernelnext", | |
| "hatch-kernelnext", | |
| "volteer-kernelnext", | |
| "zork-kernelnext", | |
| ], | |
| ) | |
| _release_enable_skylab_hwtest = _frozen_ge_set( | |
| ge_build_config, | |
| [ | |
| "asuka", | |
| "coral", | |
| "nyan_blaze", | |
| "reef", | |
| ], | |
| ) | |
| _release_enable_skylab_partial_boards = { | |
| "coral": ["astronaut", "nasher", "lava"], | |
| } | |
| _release_enable_skylab_cts_hwtest = _frozen_ge_set( | |
| ge_build_config, | |
| [ | |
| "terra", | |
| ], | |
| ) | |
| _no_unittest_configs = [ | |
| "grunt-kernelnext-release", | |
| "guybrush-kernelnext-release", | |
| "zork-connectivitynext-release", | |
| "zork-minios-release", | |
| ] | |
| def _get_skylab_settings(board_name): | |
| """Get skylab settings for release builder. | |
| Args: | |
| board_name: A string board name. | |
| Returns: | |
| A dict mapping suite types to booleans indicating whether this suite | |
| on this board is to be run on Skylab. Current suite types: | |
| - cts: all suites using pool:cts, | |
| - default: the rest of the suites. | |
| """ | |
| return { | |
| "cts": board_name in _release_enable_skylab_cts_hwtest, | |
| "default": board_name in _release_enable_skylab_hwtest, | |
| } | |
| builder_to_boards_dict = config_lib.GroupBoardsByBuilder( | |
| ge_build_config[config_lib.CONFIG_TEMPLATE_BOARDS] | |
| ) | |
| _all_release_builder_boards = builder_to_boards_dict[ | |
| config_lib.CONFIG_TEMPLATE_RELEASE | |
| ] | |
| for unibuild in config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config): | |
| reference_board_name = unibuild[ | |
| config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME | |
| ] | |
| if reference_board_name != "eve": | |
| continue | |
| config_name = "%s-release" % reference_board_name | |
| # Move unibuild to skylab. | |
| important = not unibuild[config_lib.CONFIG_TEMPLATE_EXPERIMENTAL] | |
| if reference_board_name in _release_experimental_boards: | |
| important = False | |
| props = { | |
| "important": important, | |
| } | |
| if config_name in _no_unittest_configs: | |
| props["unittests"] = False | |
| site_config.AddForBoards( | |
| config_lib.CONFIG_TYPE_RELEASE, | |
| [reference_board_name], | |
| board_configs, | |
| site_config.templates.release, | |
| **props, | |
| ) | |
| _AssignToMaster(site_config[config_name]) | |
| def GetReleaseConfigName(board): | |
| """Convert a board name into a release config name.""" | |
| return "%s-release" % board | |
| def GetConfigName(builder, board): | |
| """Convert a board name into a config name.""" | |
| if builder == config_lib.CONFIG_TEMPLATE_RELEASE: | |
| return GetReleaseConfigName(board) | |
| else: | |
| # Currently just support RELEASE builders | |
| raise Exception("Do not support other builders.") | |
| def _GetConfigValues(board): | |
| """Get and return config values from template and user definitions.""" | |
| important = not board[config_lib.CONFIG_TEMPLATE_EXPERIMENTAL] | |
| if board["name"] in _release_experimental_boards: | |
| important = False | |
| # Move non-unibuild to skylab. | |
| config_values = { | |
| "important": important, | |
| } | |
| return config_values | |
| def _AdjustUngroupedReleaseConfigs(builder_ungrouped_dict): | |
| """Adjust for ungrouped release boards""" | |
| for builder in builder_ungrouped_dict: | |
| for board in builder_ungrouped_dict[builder]: | |
| config_name = GetConfigName( | |
| builder, board[config_lib.CONFIG_TEMPLATE_NAME] | |
| ) | |
| site_config[config_name].apply( | |
| _GetConfigValues(board), | |
| ) | |
| _AssignToMaster(site_config[config_name]) | |
| def _AdjustGroupedReleaseConfigs(builder_group_dict): | |
| """Adjust leader and follower configs for grouped boards""" | |
| for builder in builder_group_dict: | |
| for group in builder_group_dict[builder]: | |
| board_group = builder_group_dict[builder][group] | |
| # Leaders are built on baremetal builders and run all tests | |
| # needed by the related boards. | |
| for board in board_group.leader_boards: | |
| config_name = GetConfigName( | |
| builder, board[config_lib.CONFIG_TEMPLATE_NAME] | |
| ) | |
| site_config[config_name].apply( | |
| _GetConfigValues(board), | |
| ) | |
| _AssignToMaster(site_config[config_name]) | |
| # Followers are built on GCE instances, and turn off testing | |
| # that breaks on GCE. The missing tests run on the leader board. | |
| for board in board_group.follower_boards: | |
| config_name = GetConfigName( | |
| builder, board[config_lib.CONFIG_TEMPLATE_NAME] | |
| ) | |
| site_config[config_name].apply( | |
| _GetConfigValues(board), | |
| chrome_sdk_build_chrome=False, | |
| ) | |
| _AssignToMaster(site_config[config_name]) | |
| def SpecialtyBuilders(site_config): | |
| """Add a variety of specialized builders or tryjobs. | |
| Args: | |
| site_config: config_lib.SiteConfig to be modified by adding templates | |
| and configs. | |
| """ | |
| site_config.AddWithoutTemplate( | |
| "success-build", | |
| site_config.templates.external, | |
| boards=[], | |
| display_label=config_lib.DISPLAY_LABEL_TRYJOB, | |
| luci_builder=config_lib.LUCI_BUILDER_TRY, | |
| builder_class_name="test_builders.SucessBuilder", | |
| description="Builder always passes as quickly as possible.", | |
| ) | |
| # Used by cbuildbot/stages/sync_stages_unittest | |
| site_config.AddWithoutTemplate( | |
| "sync-test-cbuildbot", | |
| boards=[], | |
| display_label=config_lib.DISPLAY_LABEL_TRYJOB, | |
| luci_builder=config_lib.LUCI_BUILDER_INFRA, | |
| builder_class_name="test_builders.SucessBuilder", | |
| description="Used by sync_stages_unittest.", | |
| ) | |
| site_config.AddWithoutTemplate( | |
| "fail-build", | |
| site_config.templates.external, | |
| boards=[], | |
| display_label=config_lib.DISPLAY_LABEL_TRYJOB, | |
| luci_builder=config_lib.LUCI_BUILDER_TRY, | |
| builder_class_name="test_builders.FailBuilder", | |
| description="Builder always fails as quickly as possible.", | |
| ) | |
| site_config.AddWithoutTemplate( | |
| "config-updater", | |
| site_config.templates.internal, | |
| site_config.templates.infra_builder, | |
| display_label=config_lib.DISPLAY_LABEL_UTILITY, | |
| description=( | |
| "Build Config Updater reads updated GE config files from" | |
| " GS, and commits them to chromite after running tests." | |
| ), | |
| build_type=constants.GENERIC_TYPE, | |
| build_timeout=2 * 60 * 60, | |
| boards=[], | |
| builder_class_name="config_builders.UpdateConfigBuilder", | |
| schedule="@hourly", | |
| ) | |
| site_config.AddWithoutTemplate( | |
| "luci-scheduler-updater", | |
| site_config.templates.internal, | |
| site_config.templates.infra_builder, | |
| display_label=config_lib.DISPLAY_LABEL_UTILITY, | |
| description="Deploy changes to luci_scheduler.cfg.", | |
| build_type=constants.GENERIC_TYPE, | |
| boards=[], | |
| builder_class_name="config_builders.LuciSchedulerBuilder", | |
| schedule="triggered", | |
| triggered_gitiles=[ | |
| [ | |
| "https://chromium.googlesource.com/chromiumos/chromite", | |
| ["refs/heads/main"], | |
| ["config/luci-scheduler.cfg"], | |
| ], | |
| [ | |
| ( | |
| "https://chrome-internal.googlesource.com/chromeos/infra/" | |
| "config" | |
| ), | |
| ["refs/heads/main"], | |
| ["generated/luci-scheduler.cfg"], | |
| ], | |
| ], | |
| ) | |
| def TryjobMirrors(site_config): | |
| """Create tryjob specialized variants of every build config. | |
| This creates a new 'tryjob' config for every existing config, unless the | |
| existing config is already a tryjob config. | |
| Args: | |
| site_config: config_lib.SiteConfig to be modified by adding templates | |
| and configs. | |
| """ | |
| tryjob_configs = {} | |
| for build_name, config in site_config.items(): | |
| # Don't mirror builds that are already tryjob safe. | |
| if config_lib.isTryjobConfig(config): | |
| continue | |
| tryjob_name = build_name + "-tryjob" | |
| # Don't overwrite mirrored versions that were explicitly created | |
| # earlier. | |
| if tryjob_name in site_config: | |
| continue | |
| tryjob_config = copy.deepcopy(config) | |
| tryjob_config.apply( | |
| name=tryjob_name, | |
| display_label=config_lib.DISPLAY_LABEL_TRYJOB, | |
| luci_builder=config_lib.LUCI_BUILDER_TRY, | |
| # Generally make tryjobs safer. | |
| chroot_replace=True, | |
| debug=True, | |
| push_image=False, | |
| # Force uprev. This is so patched in changes are always built. | |
| uprev=True, | |
| gs_path=config_lib.GS_PATH_DEFAULT, | |
| schedule=None, | |
| suite_scheduling=False, | |
| triggered_gitiles=None, | |
| important=True, | |
| ) | |
| # Force uprev. This is so patched in changes are always built. | |
| if tryjob_config.internal: | |
| tryjob_config.apply(overlays=constants.BOTH_OVERLAYS) | |
| if tryjob_config.master: | |
| tryjob_config.apply(debug_cidb=True) | |
| if tryjob_config.slave_configs: | |
| new_children = [] | |
| for c in tryjob_config.slave_configs: | |
| if not config_lib.isTryjobConfig(site_config[c]): | |
| c = "%s-tryjob" % c | |
| new_children.append(c) | |
| tryjob_config.apply(slave_configs=new_children) | |
| # Save off the new config so we can insert into site_config. | |
| tryjob_configs[tryjob_name] = tryjob_config | |
| for tryjob_name, tryjob_config in tryjob_configs.items(): | |
| site_config[tryjob_name] = tryjob_config | |
| @memoize.Memoize | |
| def GetConfig(): | |
| """Create the Site configuration for all ChromeOS builds. | |
| Returns: | |
| A config_lib.SiteConfig. | |
| """ | |
| defaults = DefaultSettings() | |
| ge_build_config = config_lib.LoadGEBuildConfigFromFile() | |
| boards_dict = GetBoardTypeToBoardsDict(ge_build_config) | |
| # If there are unknown boards in the GE config, issue a warning and ignore | |
| # them. | |
| unknown = boards_dict["unknown_boards"] | |
| if unknown: | |
| logging.warning( | |
| "dropping unknown boards from GE config: %s", | |
| " ".join(x for x in unknown), | |
| ) | |
| ge_build_config["boards"] = [ | |
| x for x in ge_build_config["boards"] if x["name"] not in unknown | |
| ] | |
| boards_dict = GetBoardTypeToBoardsDict(ge_build_config) | |
| # site_config with no templates or build configurations. | |
| site_config = config_lib.SiteConfig(defaults=defaults) | |
| GeneralTemplates(site_config) | |
| ReleaseBuilders(site_config, boards_dict, ge_build_config) | |
| SpecialtyBuilders(site_config) | |
| FactoryBuilders(site_config, boards_dict, ge_build_config) | |
| FullBuilders(site_config, boards_dict, ge_build_config) | |
| TryjobMirrors(site_config) | |
| return site_config |