Google Git
Sign in
cos / mirrors / cros / chromiumos / chromite / refs/heads/stabilize-atlas-11177.B / . / config / chromeos_config.py
blob: 8fffbf8c1b0328410602404525bc584507f347bb [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Configuration options for various cbuildbot builders."""
from __future__ import print_function
import copy
import os
from chromite.lib.const import waterfall
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import factory
def getInfoVMTest():
suites = [
'vmtest-informational4'
]
ret = []
for suite in suites:
ret.append(config_lib.VMTestConfig(
constants.VM_SUITE_TEST_TYPE,
test_suite=suite,
warn_only=True,
timeout=12 * 60 * 60))
return ret
class HWTestList(object):
"""Container for methods to generate HWTest lists."""
def __init__(self, ge_build_config):
"""Helper class for creating hwtests.
Args:
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
self.is_release_branch = ge_build_config[
config_lib.CONFIG_TEMPLATE_RELEASE_BRANCH]
def DefaultList(self, **kwargs):
"""Returns a default list of HWTestConfigs for a build.
Args:
*kwargs: overrides for the configs
"""
installer_kwargs = kwargs.copy()
# Force au suite to run first.
installer_kwargs['priority'] = constants.HWTEST_CQ_PRIORITY
async_kwargs = kwargs.copy()
async_kwargs['priority'] = constants.HWTEST_POST_BUILD_PRIORITY
async_kwargs['async'] = True
async_kwargs['suite_min_duts'] = 1
async_kwargs['timeout'] = config_lib.HWTestConfig.ASYNC_HW_TEST_TIMEOUT
if self.is_release_branch:
bvt_inline_kwargs = async_kwargs
else:
bvt_inline_kwargs = kwargs.copy()
bvt_inline_kwargs['timeout'] = (
config_lib.HWTestConfig.SHARED_HW_TEST_TIMEOUT)
# BVT + INSTALLER suite.
return [
config_lib.HWTestConfig(constants.HWTEST_BVT_SUITE,
**bvt_inline_kwargs),
config_lib.HWTestConfig(constants.HWTEST_ARC_COMMIT_SUITE,
**bvt_inline_kwargs),
self.TastConfig(constants.HWTEST_TAST_CQ_SUITE, **bvt_inline_kwargs),
config_lib.HWTestConfig(constants.HWTEST_INSTALLER_SUITE,
blocking=True, **installer_kwargs),
config_lib.HWTestConfig(constants.HWTEST_COMMIT_SUITE,
**async_kwargs),
config_lib.HWTestConfig(constants.HWTEST_CANARY_SUITE,
**async_kwargs),
]
def DefaultListCanary(self, **kwargs):
"""Returns a default list of config_lib.HWTestConfig's for a canary build.
Args:
*kwargs: overrides for the configs
"""
# Set minimum_duts default to 4, which means that lab will check the
# number of available duts to meet the minimum requirement before creating
# the suite job for canary builds.
kwargs.setdefault('minimum_duts', 4)
kwargs.setdefault('file_bugs', True)
return self.DefaultList(**kwargs)
def AFDOList(self, **kwargs):
"""Returns a default list of HWTestConfigs for a AFDO build.
For more details on AFDO:
- https://gcc.gnu.org/wiki/AutoFDO
- https://sites.google.com/a/google.com/chromeos-toolchain-team-home2
-> 11. AFDO PROFILE-GUIDED OPTIMIZATIONS
Args:
*kwargs: overrides for the configs
"""
afdo_dict = dict(pool=constants.HWTEST_SUITES_POOL,
timeout=120 * 60, async=True, retry=False,
max_retries=None)
afdo_dict.update(kwargs)
return [config_lib.HWTestConfig('perf_v2', **afdo_dict)]
def DefaultListNonCanary(self, **kwargs):
"""Return a default list of HWTestConfigs for a non-canary build.
Optional arguments may be overridden in `kwargs`, except that
the `blocking` setting cannot be provided.
"""
# N.B. The ordering here is coupled with the column order of
# entries in _paladin_hwtest_assignments, below. If you change the
# ordering here you must also change the ordering there.
return [
config_lib.HWTestConfig(
constants.HWTEST_PROVISION_SUITE,
blocking=True,
suite_args={'num_required': 1},
**kwargs),
config_lib.HWTestConfig(constants.HWTEST_BVT_SUITE,
**kwargs),
config_lib.HWTestConfig(constants.HWTEST_COMMIT_SUITE,
**kwargs),
config_lib.HWTestConfig(constants.HWTEST_ARC_COMMIT_SUITE,
**kwargs)]
def DefaultListCQ(self, **kwargs):
"""Return a default list of HWTestConfigs for a CQ build.
Optional arguments may be overridden in `kwargs`, except that
the `blocking` setting cannot be provided.
"""
default_dict = dict(pool=constants.HWTEST_PALADIN_POOL,
timeout=config_lib.HWTestConfig.PALADIN_HW_TEST_TIMEOUT,
file_bugs=False, priority=constants.HWTEST_CQ_PRIORITY,
minimum_duts=4)
# Allows kwargs overrides to default_dict for cq.
default_dict.update(kwargs)
suite_list = self.DefaultListNonCanary(**default_dict)
suite_list.append(self.TastConfig(constants.HWTEST_TAST_CQ_SUITE,
**default_dict))
return suite_list
def DefaultListPFQ(self, **kwargs):
"""Return a default list of HWTestConfigs for a Chrome PFQ build.
Optional arguments may be overridden in `kwargs`, except that
the `blocking` setting cannot be provided.
"""
default_dict = dict(pool=constants.HWTEST_PFQ_POOL, file_bugs=True,
priority=constants.HWTEST_PFQ_PRIORITY,
minimum_duts=4)
# Allows kwargs overrides to default_dict for pfq.
default_dict.update(kwargs)
suite_list = self.DefaultListNonCanary(**default_dict)
suite_list.append(self.TastConfig(constants.HWTEST_TAST_CHROME_PFQ_SUITE,
**default_dict))
return suite_list
def DefaultListChromePFQInformational(self, **kwargs):
"""Return a default list of HWTestConfigs for an inform. Chrome PFQ build.
Optional arguments may be overridden in `kwargs`, except that
the `blocking` setting cannot be provided.
"""
# The informational PFQ does not honor to retry jobs. This reduces
# hwtest times and shows flakes clearer.
default_dict = dict(pool=constants.HWTEST_PFQ_POOL, file_bugs=True,
priority=constants.HWTEST_PFQ_PRIORITY,
retry=False, max_retries=None, minimum_duts=1)
# Allows kwargs overrides to default_dict for pfq.
default_dict.update(kwargs)
suite_list = self.DefaultListNonCanary(**default_dict)
suite_list.append(config_lib.HWTestConfig(
constants.HWTEST_CHROME_INFORMATIONAL, warn_only=True, **default_dict))
suite_list.append(self.TastConfig(constants.HWTEST_TAST_CHROME_PFQ_SUITE,
**default_dict))
return suite_list
def SharedPoolPFQ(self, **kwargs):
"""Return a list of HWTestConfigs for Chrome PFQ which uses a shared pool.
The returned suites will run in pool:critical by default, which is
shared with other types of builders (canaries, cq). The first suite in the
list is a blocking sanity suite that verifies the build will not break dut.
"""
sanity_dict = dict(pool=constants.HWTEST_MACH_POOL,
file_bugs=True, priority=constants.HWTEST_PFQ_PRIORITY)
sanity_dict.update(kwargs)
sanity_dict.update(dict(minimum_duts=1, suite_min_duts=1,
blocking=True))
default_dict = dict(pool=constants.HWTEST_MACH_POOL,
suite_min_duts=3)
default_dict.update(kwargs)
suite_list = [config_lib.HWTestConfig(constants.HWTEST_SANITY_SUITE,
**sanity_dict)]
suite_list.extend(self.DefaultListPFQ(**default_dict))
return suite_list
def DefaultListAndroidPFQ(self, **kwargs):
"""Return a default list of HWTestConfig's for an ARC PFQ build.
Optional arguments may be overridden in `kwargs`, except that
the `blocking` setting cannot be provided.
"""
default_dict = dict(file_bugs=True,
timeout=config_lib.HWTestConfig.ASYNC_HW_TEST_TIMEOUT,
priority=constants.HWTEST_PFQ_PRIORITY, minimum_duts=3)
# Allows kwargs overrides to default_dict for pfq.
default_dict.update(kwargs)
return [
config_lib.HWTestConfig(constants.HWTEST_ARC_COMMIT_SUITE,
pool=constants.HWTEST_MACH_POOL,
**default_dict),
self.TastConfig(constants.HWTEST_TAST_ANDROID_PFQ_SUITE,
pool=constants.HWTEST_MACH_POOL, **default_dict),
]
def SharedPoolAndroidPFQ(self, **kwargs):
"""Return a list of HWTestConfigs for ARC PFQ which uses a shared pool.
The returned suites will run in pool:critical by default, which is
shared with other types of builders (canaries, cq). The first suite in the
list is a blocking sanity suite that verifies the build will not break dut.
"""
sanity_dict = dict(pool=constants.HWTEST_MACH_POOL,
file_bugs=True, priority=constants.HWTEST_PFQ_PRIORITY)
sanity_dict.update(kwargs)
sanity_dict.update(dict(minimum_duts=1, suite_min_duts=1,
blocking=True))
default_dict = dict(suite_min_duts=3)
default_dict.update(kwargs)
suite_list = [config_lib.HWTestConfig(constants.HWTEST_SANITY_SUITE,
**sanity_dict)]
suite_list.extend(self.DefaultListAndroidPFQ(**default_dict))
return suite_list
def SharedPoolCQ(self, **kwargs):
"""Return a list of HWTestConfigs for CQ which uses a shared pool.
The returned suites will run in pool:critical by default, which is
shared with other types of builder (canaries, pfq). The first suite in the
list is a blocking sanity suite that verifies the build will not break dut.
"""
sanity_dict = dict(pool=constants.HWTEST_MACH_POOL,
timeout=config_lib.HWTestConfig.SHARED_HW_TEST_TIMEOUT,
file_bugs=False, priority=constants.HWTEST_CQ_PRIORITY)
sanity_dict.update(kwargs)
sanity_dict.update(dict(minimum_duts=1, suite_min_duts=1,
blocking=True))
default_dict = dict(pool=constants.HWTEST_MACH_POOL,
suite_min_duts=10)
default_dict.update(kwargs)
suite_list = [config_lib.HWTestConfig(constants.HWTEST_SANITY_SUITE,
**sanity_dict)]
suite_list.extend(self.DefaultListCQ(**default_dict))
return suite_list
def SharedPoolCanary(self, **kwargs):
"""Return a list of HWTestConfigs for Canary which uses a shared pool.
The returned suites will run in pool:critical by default, which is
shared with CQs. The first suite in the list is a blocking sanity suite
that verifies the build will not break dut.
"""
sanity_dict = dict(pool=constants.HWTEST_MACH_POOL, file_bugs=True)
sanity_dict.update(kwargs)
sanity_dict.update(dict(minimum_duts=1, suite_min_duts=1,
blocking=True))
default_dict = dict(pool=constants.HWTEST_MACH_POOL,
suite_min_duts=6)
default_dict.update(kwargs)
suite_list = [config_lib.HWTestConfig(constants.HWTEST_SANITY_SUITE,
**sanity_dict)]
suite_list.extend(self.DefaultListCanary(**default_dict))
return suite_list
def AFDORecordTest(self, **kwargs):
default_dict = dict(pool=constants.HWTEST_MACH_POOL,
warn_only=True, file_bugs=True,
timeout=constants.AFDO_GENERATE_TIMEOUT,
priority=constants.HWTEST_PFQ_PRIORITY)
# Allows kwargs overrides to default_dict for cq.
default_dict.update(kwargs)
return config_lib.HWTestConfig(constants.HWTEST_AFDO_SUITE, **default_dict)
def WiFiCellPoolPreCQ(self, **kwargs):
"""Return a list of HWTestConfigs which run wifi tests.
This should be used by the ChromeOS WiFi team to ensure changes pass the
wifi tests as a pre-cq sanity check.
"""
default_dict = dict(pool=constants.HWTEST_WIFICELL_PRE_CQ_POOL,
file_bugs=False,
priority=constants.HWTEST_DEFAULT_PRIORITY,
retry=False, max_retries=None, minimum_duts=1)
default_dict.update(kwargs)
suite_list = [config_lib.HWTestConfig(constants.WIFICELL_PRE_CQ,
**default_dict)]
return suite_list
def BluestreakPoolPreCQ(self, **kwargs):
"""Return a list of HWTestConfigs which run bluestreak tests.
This should be used by the ChromeOS MRHW team to ensure changes pass the
CFM tests as a pre-cq sanity check.
"""
default_dict = dict(pool=constants.HWTEST_BLUESTREAK_PRE_CQ_POOL,
file_bugs=False,
priority=constants.HWTEST_DEFAULT_PRIORITY,
retry=False, max_retries=None, minimum_duts=1)
default_dict.update(kwargs)
suite_list = [config_lib.HWTestConfig(constants.BLUESTREAK_PRE_CQ,
**default_dict)]
return suite_list
def ToolchainTestFull(self, machine_pool, **kwargs):
"""Return full set of HWTESTConfigs to run toolchain correctness tests."""
default_dict = dict(pool=machine_pool, async=False,
file_bugs=False,
priority=constants.HWTEST_DEFAULT_PRIORITY)
default_dict.update(kwargs)
return [config_lib.HWTestConfig(constants.HWTEST_BVT_SUITE,
**default_dict),
config_lib.HWTestConfig(constants.HWTEST_COMMIT_SUITE,
**default_dict),
self.TastConfig(constants.HWTEST_TAST_CQ_SUITE,
**default_dict),
config_lib.HWTestConfig('security',
**default_dict),
config_lib.HWTestConfig('kernel_daily_regression',
**default_dict),
config_lib.HWTestConfig('kernel_daily_benchmarks',
**default_dict)]
def ToolchainTestMedium(self, machine_pool, **kwargs):
"""Return list of HWTESTConfigs to run toolchain LLVM correctness tests.
Since the kernel is not built with LLVM, it makes no sense for the
toolchain to run kernel tests on LLVM builds.
"""
default_dict = dict(pool=machine_pool, async=False,
file_bugs=False,
priority=constants.HWTEST_DEFAULT_PRIORITY)
default_dict.update(kwargs)
return [config_lib.HWTestConfig(constants.HWTEST_BVT_SUITE,
**default_dict),
config_lib.HWTestConfig(constants.HWTEST_COMMIT_SUITE,
**default_dict),
self.TastConfig(constants.HWTEST_TAST_CQ_SUITE,
**default_dict),
config_lib.HWTestConfig('security',
**default_dict)]
# pylint: disable=unused-argument
def CtsGtsQualTests(self, **kwargs):
"""Return a list of HWTestConfigs for CTS, GTS tests."""
cts_config = dict(
pool=constants.HWTEST_CTS_POOL,
timeout=config_lib.HWTestConfig.CTS_QUAL_HW_TEST_TIMEOUT,
priority=constants.HWTEST_CTS_PRIORITY,
async=True,
enable_skylab=False)
cts_config.update(kwargs)
gts_config = dict(
pool=constants.HWTEST_GTS_POOL,
timeout=config_lib.HWTestConfig.GTS_QUAL_HW_TEST_TIMEOUT,
priority=constants.HWTEST_GTS_PRIORITY,
async=True,
enable_skylab=False)
gts_config.update(kwargs)
return [config_lib.HWTestConfig(constants.HWTEST_CTS_QUAL_SUITE,
**cts_config),
config_lib.HWTestConfig(constants.HWTEST_GTS_QUAL_SUITE,
**gts_config)]
def TastConfig(self, suite_name, **kwargs):
"""Return an HWTestConfig that runs the provided Tast test suite.
Args:
suite_name: String suite name, e.g. constants.HWTEST_TAST_CQ_SUITE.
kwargs: Dict containing additional keyword args to use when constructing
the HWTestConfig.
Returns:
HWTestConfig object for running the suite.
"""
kwargs = kwargs.copy()
# Tast test suites run at most three jobs (for system, Chrome, and Android
# tests) and have short timeouts, so request at most 1 DUT (while retaining
# passed-in requests for 0 DUTs).
if kwargs.get('minimum_duts', 0):
kwargs['minimum_duts'] = 1
if kwargs.get('suite_min_duts', 0):
kwargs['suite_min_duts'] = 1
return config_lib.HWTestConfig(suite_name, **kwargs)
def append_useflags(useflags):
"""Used to append a set of useflags to existing useflags.
Useflags that shadow prior use flags will cause the prior flag to be removed.
(e.g. appending '-foo' to 'foo' will cause 'foo' to be removed)
Examples:
new_config = base_config.derive(useflags=append_useflags(['foo', '-bar'])
Args:
useflags: List of string useflags to append.
"""
assert isinstance(useflags, (list, set))
shadowed_useflags = {'-' + flag for flag in useflags
if not flag.startswith('-')}
shadowed_useflags.update({flag[1:] for flag in useflags
if flag.startswith('-')})
def handler(old_useflags):
new_useflags = set(old_useflags or [])
new_useflags.update(useflags)
new_useflags.difference_update(shadowed_useflags)
return sorted(list(new_useflags))
return handler
def remove_images(unsupported_images):
"""Remove unsupported images when applying changes to a BuildConfig.
Used similarly to append_useflags.
Args:
unsupported_images: A list of image names that should not be present
in the final build config.
Returns:
A callable suitable for use with BuildConfig.apply.
"""
unsupported = set(unsupported_images)
def handler(old_images):
if not old_images:
old_images = []
return [i for i in old_images if i not in unsupported]
return handler
TRADITIONAL_VM_TESTS_SUPPORTED = [
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False),
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)]
#
# Define assorted constants describing various sets of boards.
#
# Base per-board configuration.
# Every board must appear in exactly 1 of the following sets.
_arm_internal_release_boards = frozenset([
'arkham',
'beaglebone',
'beaglebone_servo',
'bob',
'capri',
'capri-zfpga',
'cheza',
'cobblepot',
'daisy',
'daisy_skate',
'daisy_spring',
'elm',
'gale',
'gonzo',
'gru',
'hana',
'kevin',
'kevin-arcnext',
'kukui',
'lasilla-ground',
'lasilla-sky',
'macchiato-ground',
'nyan_big',
'nyan_blaze',
'nyan_kitty',
'oak',
'octavius',
'peach_pi',
'peach_pit',
'romer',
'scarlet',
'veyron_fievel',
'veyron_jaq',
'veyron_jerry',
'veyron_mickey',
'veyron_mighty',
'veyron_minnie',
'veyron_rialto',
'veyron_speedy',
'veyron_tiger',
'whirlwind',
'wooten',
])
_arm_external_boards = frozenset([
'arm-generic',
'arm64-generic',
'arm64-llvmpipe',
'tael',
])
_x86_internal_release_boards = frozenset([
'amd64-generic-cheets',
'amd64-generic-goofy',
'asuka',
'atlas',
'auron_paine',
'auron_yuna',
'banjo',
'banon',
'betty',
'betty-arc64',
'betty-arcnext',
'buddy',
'candy',
'caroline',
'caroline-arcnext',
'cave',
'celes',
'chell',
'clapper',
'coral',
'cyan',
'dragonegg',
'edgar',
'enguarde',
'eve',
'eve-arcnext',
'eve-arcvm',
'eve-campfire',
'expresso',
'falco',
'falco_li',
'fizz',
'fizz-accelerator',
'fizz-moblab',
'gandof',
'glados',
'glimmer',
'gnawty',
'grunt',
'guado',
'guado-accelerator',
'guado_labstation',
'guado_moblab',
'heli',
'jecht',
'kalista',
'kefka',
'kip',
'lakitu',
'lakitu-gpu',
'lakitu-nc',
'lakitu-st',
'lakitu_next',
'lars',
'leon',
'link',
'lulu',
'mccloud',
'monroe',
'nami',
'nautilus',
'ninja',
'nocturne',
'novato',
'novato-arc64',
'octopus',
'orco',
'panther',
'parrot_ivb',
'peppy',
'poppy',
'pyro',
'quawks',
'rammus',
'reef',
'reks',
'relm',
'rikku',
'samus',
'sand',
'sentry',
'setzer',
'slippy',
'snappy',
'soraka',
'squawks',
'stout',
'sumo',
'swanky',
'terra',
'tidus',
'tricky',
'ultima',
'winky',
'wizpig',
'wolf',
'zako',
])
_x86_external_boards = frozenset([
'amd64-generic',
'moblab-generic-vm',
'tatl',
'x32-generic',
])
# Board can appear in 1 or more of the following sets.
_brillo_boards = frozenset([
'arkham',
'gale',
'whirlwind',
])
_accelerator_boards = frozenset([
'fizz-accelerator',
'guado-accelerator',
])
_beaglebone_boards = frozenset([
'beaglebone',
'beaglebone_servo',
])
_lakitu_boards = frozenset([
'lakitu',
'lakitu-gpu',
'lakitu-nc',
'lakitu-st',
'lakitu_next',
])
_lassen_boards = frozenset([
'lassen',
])
_loonix_boards = frozenset([
'capri',
'capri-zfpga',
'cobblepot',
'gonzo',
'lasilla-ground',
'lasilla-sky',
'macchiato-ground',
'octavius',
'romer',
'wooten',
])
_moblab_boards = frozenset([
'fizz-moblab',
'guado_moblab',
'moblab-generic-vm',
])
_scribe_boards = frozenset([
'guado-macrophage',
])
_termina_boards = frozenset([
'tatl',
'tael',
])
_nofactory_boards = (
_lakitu_boards | _termina_boards | _lassen_boards | frozenset([
'x30evb',
])
)
_toolchains_from_source = frozenset([
'x32-generic',
])
_noimagetest_boards = (_lakitu_boards | _loonix_boards | _termina_boards
| _scribe_boards)
_nohwqual_boards = (_lakitu_boards | _lassen_boards | _loonix_boards
| _termina_boards | _beaglebone_boards)
_norootfs_verification_boards = frozenset([
])
_base_layout_boards = _lakitu_boards | _termina_boards
_vmtest_boards = frozenset([
# Full VMTest support on ChromeOS is currently limited to devices derived
# from betty & co.
'amd64-generic', # Has kernel 4.4, used with public Chromium.
'betty', # amd64 Chrome OS VM board with 32 bit arm/x86 ARC++ ABI.
'betty-arc64', # Chrome OS VM board with 64 bit x86_64 ARC++ ABI.
'betty-arcnext', # Like betty but with the next version of ARC++.
'novato', # Like betty but with GMSCore but not the Play Store
'novato-arc64', # 64 bit x86_64 ARC++ ABI
]) | _lakitu_boards # All lakitu boards have VM support.
def GetBoardTypeToBoardsDict(ge_build_config):
"""Get board type to board names dict.
Args:
ge_build_config: Dictionary containing the decoded GE configuration file.
Returns:
A dict mapping board types to board name collections.
The dict contains board types including distinct_board_sets,
all_release_boards, all_full_boards, all_boards, internal_boards,
and no_vmtest_boards.
"""
ge_arch_board_dict = config_lib.GetArchBoardDict(ge_build_config)
boards_dict = {}
arm_internal_release_boards = (
_arm_internal_release_boards |
ge_arch_board_dict.get(config_lib.CONFIG_ARM_INTERNAL, set())
)
arm_external_boards = (
_arm_external_boards |
ge_arch_board_dict.get(config_lib.CONFIG_ARM_EXTERNAL, set())
)
x86_internal_release_boards = (
_x86_internal_release_boards |
ge_arch_board_dict.get(config_lib.CONFIG_X86_INTERNAL, set())
)
x86_external_boards = (
_x86_external_boards |
ge_arch_board_dict.get(config_lib.CONFIG_X86_EXTERNAL, set())
)
# Every board should be in only 1 of the above sets.
boards_dict['distinct_board_sets'] = [
arm_internal_release_boards,
arm_external_boards,
x86_internal_release_boards,
x86_external_boards,
]
arm_full_boards = (
arm_internal_release_boards |
arm_external_boards)
x86_full_boards = (
x86_internal_release_boards |
x86_external_boards)
arm_boards = arm_full_boards
x86_boards = x86_full_boards
boards_dict['all_release_boards'] = (
_arm_internal_release_boards |
_x86_internal_release_boards
)
boards_dict['all_full_boards'] = (
arm_full_boards |
x86_full_boards
)
all_boards = x86_boards | arm_boards
boards_dict['all_boards'] = (
all_boards
)
boards_dict['internal_boards'] = boards_dict['all_release_boards']
# This set controls the final vmtest override. It allows us to specify
# vm_tests for each class of builders, but only execute on _vmtest_boards.
boards_dict['no_vmtest_boards'] = (
all_boards - _vmtest_boards
)
return boards_dict
def DefaultSettings(site_params):
"""Create the default build config values for this site.
Args:
site_params: A populated config_lib.SiteParameters instance.
Returns:
dict: of default config_lib.BuildConfig values to use for this site.
"""
# Site specific adjustments for default BuildConfig values.
defaults = config_lib.DefaultSettings()
# Git repository URL for our manifests.
# https://chromium.googlesource.com/chromiumos/manifest
# https://chrome-internal.googlesource.com/chromeos/manifest-internal
defaults['manifest_repo_url'] = site_params['MANIFEST_URL']
return defaults
def GeneralTemplates(site_config, ge_build_config):
"""Defines templates that are shared between categories of builders.
Args:
site_config: A SiteConfig object to add the templates too.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
is_release_branch = ge_build_config[config_lib.CONFIG_TEMPLATE_RELEASE_BRANCH]
hw_test_list = HWTestList(ge_build_config)
# TryjobMirrors uses hw_tests_override to ensure that tryjobs run all suites
# rather than just the ones that are assigned to the board being used. Add
# bvt-tast-cq here since it includes system, Chrome, and Android tests.
site_config.AddTemplate(
'default_hw_tests_override',
hw_tests_override=hw_test_list.DefaultList(
pool=constants.HWTEST_TRYBOT_POOL,
file_bugs=False),
)
# Config parameters for builders that do not run tests on the builder.
site_config.AddTemplate(
'no_unittest_builder',
unittests=False,
)
# Notice all builders except for _vmtest_boards should not run vmtest.
site_config.AddTemplate(
'no_vmtest_builder',
vm_tests=[],
vm_tests_override=None,
gce_tests=[],
)
site_config.AddTemplate(
'no_hwtest_builder',
hw_tests=[],
hw_tests_override=[],
)
# Builder type templates.
site_config.AddTemplate(
'full',
site_config.templates.default_hw_tests_override,
# Full builds are test builds to show that we can build from scratch,
# so use settings to build from scratch, and archive the results.
usepkg_build_packages=False,
chrome_sdk=True,
display_label=config_lib.DISPLAY_LABEL_FULL,
build_type=constants.FULL_TYPE,
archive_build_debug=True,
images=['base', 'recovery', 'test', 'factory_install'],
git_sync=True,
description='Full Builds',
image_test=True,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Continuous',
)
site_config.AddTemplate(
'paladin',
hw_tests_override=(
hw_test_list.DefaultListNonCanary(pool=constants.HWTEST_TRYBOT_POOL,
file_bugs=False) +
[hw_test_list.TastConfig(constants.HWTEST_TAST_CQ_SUITE,
pool=constants.HWTEST_TRYBOT_POOL,
file_bugs=False)]),
chroot_replace=False,
display_label=config_lib.DISPLAY_LABEL_CQ,
build_type=constants.PALADIN_TYPE,
overlays=constants.PUBLIC_OVERLAYS,
prebuilts=constants.PUBLIC,
manifest_version=True,
description='Commit Queue',
upload_standalone_images=False,
images=['base', 'test'],
image_test=True,
chrome_sdk=True,
chrome_sdk_build_chrome=False,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#TOC-CQ',
# This only applies to vmtest enabled boards like betty and novato.
# TODO(derat): Add TastVMTestConfig here.
vm_tests=[config_lib.VMTestConfig(
constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)],
vm_tests_override=TRADITIONAL_VM_TESTS_SUPPORTED,
)
# Incremental builders are intended to test the developer workflow.
# For that reason, they don't uprev.
site_config.AddTemplate(
'incremental',
site_config.templates.default_hw_tests_override,
display_label=config_lib.DISPLAY_LABEL_INCREMENATAL,
build_type=constants.INCREMENTAL_TYPE,
chroot_replace=False,
uprev=False,
overlays=constants.PUBLIC_OVERLAYS,
description='Incremental Builds',
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Continuous',
)
site_config.AddTemplate(
'external',
internal=False,
overlays=constants.PUBLIC_OVERLAYS,
manifest_repo_url=site_config.params['MANIFEST_URL'],
manifest=constants.DEFAULT_MANIFEST,
)
# This builds with more source available.
site_config.AddTemplate(
'internal',
internal=True,
overlays=constants.BOTH_OVERLAYS,
manifest_repo_url=site_config.params['MANIFEST_INT_URL'],
)
site_config.AddTemplate(
'accelerator',
sync_chrome=False,
chrome_sdk=False,
)
site_config.AddTemplate(
'brillo',
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
)
site_config.AddTemplate(
'lakitu',
site_config.templates.no_hwtest_builder,
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
)
site_config.AddTemplate(
'lassen',
sync_chrome=False,
chrome_sdk=False,
image_test=False,
)
site_config.AddTemplate(
'x30evb',
site_config.templates.no_hwtest_builder,
sync_chrome=False,
chrome_sdk=False,
signer_tests=False,
paygen=False,
upload_hw_test_artifacts=False,
image_test=False,
images=['base', 'test'],
packages=['virtual/target-os',
'virtual/target-os-dev',
'virtual/target-os-test'],
)
site_config.AddTemplate(
'termina',
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
signer_tests=False,
sign_types=None,
paygen=False,
upload_hw_test_artifacts=False,
upload_stripped_packages=['sys-kernel/*kernel*'],
image_test=False,
images=['base', 'test'],
packages=['virtual/target-os',
'virtual/target-os-dev',
'virtual/target-os-test'],
)
site_config.AddTemplate(
'loonix',
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
# TODO(harshmodi): Re-enable this when we start using vboot
signer_tests=False,
paygen=False,
upload_hw_test_artifacts=False,
image_test=False,
images=remove_images(['recovery', 'factory_install'])
)
# An anchor of Laktiu' test customizations.
# TODO: renable SIMPLE_AU_TEST_TYPE once b/67510964 is fixed.
site_config.AddTemplate(
'lakitu_test_customizations',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
gce_tests=[config_lib.GCETestConfig(constants.GCE_SUITE_TEST_TYPE,
test_suite='gce-sanity'),
config_lib.GCETestConfig(constants.GCE_SUITE_TEST_TYPE,
test_suite='gce-smoke')],
)
# No GCE tests for lakitu-nc; Enable 'hsm' profile by default.
site_config.AddTemplate(
'lakitu_nc_customizations',
profile='hsm',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
)
# Test customizations for lakitu boards' paladin builders.
site_config.AddTemplate(
'lakitu_paladin_test_customizations',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
gce_tests=[config_lib.GCETestConfig(constants.GCE_SUITE_TEST_TYPE,
test_suite='gce-sanity')],
)
# An anchor of Laktiu' notification email settings.
site_config.AddTemplate(
'lakitu_notification_emails',
# Send an email on build failures.
health_threshold=1,
health_alert_recipients=['gci-alerts+buildbots@google.com'],
)
site_config.AddTemplate(
'moblab',
site_config.templates.no_vmtest_builder,
image_test=False,
)
site_config.AddTemplate(
'beaglebone',
site_config.templates.brillo,
image_test=False,
rootfs_verification=False,
paygen=False,
signer_tests=False,
images=remove_images(['dev', 'test', 'recovery', 'factory_install']),
)
# This adds Chrome branding.
site_config.AddTemplate(
'official_chrome',
useflags=append_useflags([constants.USE_CHROME_INTERNAL]),
)
# This sets chromeos_official.
site_config.AddTemplate(
'official',
site_config.templates.official_chrome,
chromeos_official=True,
)
site_config.AddTemplate(
'asan',
site_config.templates.default_hw_tests_override,
profile='asan',
# THESE IMAGES CAN DAMAGE THE LAB and cannot be used for hardware testing.
disk_layout='16gb-rootfs',
# TODO(deymo): ASan builders generate bigger files, in particular a bigger
# Chrome binary, that update_engine can't handle in delta payloads due to
# memory limits. Remove the following lines once crbug.com/329248 is
# fixed.
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-ASAN',
)
site_config.AddTemplate(
'ubsan',
site_config.templates.default_hw_tests_override,
profile='ubsan',
# Need larger rootfs for ubsan builds.
disk_layout='16gb-rootfs',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-ASAN',
)
site_config.AddTemplate(
'fuzzer',
site_config.templates.default_hw_tests_override,
site_config.templates.full,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_INFORMATIONAL,
profile='fuzzer',
chrome_sdk=False,
sync_chrome=False,
# Run fuzzer builder specific stages.
builder_class_name='fuzzer_builders.FuzzerBuilder',
# Need larger rootfs since fuzzing also enables asan.
disk_layout='2gb-rootfs',
gs_path='gs://chromeos-fuzzing-artifacts/libfuzzer-asan',
images=['base'],
packages=['virtual/target-fuzzers'],
)
site_config.AddTemplate(
'telemetry',
site_config.templates.default_hw_tests_override,
display_label=config_lib.DISPLAY_LABEL_CHROME_INFORMATIONAL,
build_type=constants.INCREMENTAL_TYPE,
uprev=False,
overlays=constants.PUBLIC_OVERLAYS,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='telemetry_unit_server',
# Add an extra 60 minutes.
timeout=120 * 60)],
description='Telemetry Builds',
)
site_config.AddTemplate(
'external_chromium_pfq',
site_config.templates.default_hw_tests_override,
build_type=constants.CHROME_PFQ_TYPE,
uprev=False,
# Increase the master timeout to 6 hours crbug.com/611139.
build_timeout=6 * 60 * 60,
overlays=constants.PUBLIC_OVERLAYS,
manifest_version=True,
chrome_rev=constants.CHROME_REV_LATEST,
chrome_sdk=True,
unittests=False,
description='Preflight Chromium Uprev & Build (public)',
# Add betty smoke VMTest crbug.com/710629.
vm_tests=[config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE)],
vm_tests_override=None,
)
# TODO(davidjames): Convert this to an external config once the unified master
# logic is ready.
site_config.AddTemplate(
'chromium_pfq',
site_config.templates.internal,
site_config.templates.external_chromium_pfq,
display_label=config_lib.DISPLAY_LABEL_CHROME_PFQ,
description='Preflight Chromium Uprev & Build (internal)',
overlays=constants.BOTH_OVERLAYS,
prebuilts=constants.PUBLIC,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Chrome-PFQ',
)
site_config.AddTemplate(
'chrome_pfq',
site_config.templates.chromium_pfq,
site_config.templates.official,
overlays=constants.BOTH_OVERLAYS,
description='Preflight Chrome Uprev & Build (internal)',
prebuilts=constants.PRIVATE,
)
site_config.AddTemplate(
'chrome_try',
build_type=constants.CHROME_PFQ_TYPE,
chrome_rev=constants.CHROME_REV_TOT,
manifest_version=False,
)
site_config.AddTemplate(
'chromium_pfq_informational',
site_config.templates.external_chromium_pfq,
site_config.templates.chrome_try,
display_label=config_lib.DISPLAY_LABEL_CHROME_INFORMATIONAL,
chrome_sdk=False,
unittests=False,
description='Informational Chromium Uprev & Build (public)',
)
site_config.AddTemplate(
'chrome_pfq_informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.internal,
site_config.templates.official,
unittests=False,
description='Informational Chrome Uprev & Build (internal)',
)
site_config.AddTemplate(
'chrome_pfq_cheets_informational',
site_config.templates.chrome_pfq_informational,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
hw_tests_override=hw_test_list.SharedPoolAndroidPFQ()
)
site_config.AddTemplate(
'tot_asan_informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.asan,
display_label=config_lib.DISPLAY_LABEL_INFORMATIONAL,
unittests=True,
description='Build TOT Chrome with Address Sanitizer (Clang)',
)
site_config.AddTemplate(
'tot_ubsan_informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.ubsan,
display_label=config_lib.DISPLAY_LABEL_INFORMATIONAL,
unittests=True,
description='Build TOT Chrome with Undefined Behavior Sanitizer (Clang)',
)
site_config.AddTemplate(
'chrome_perf',
site_config.templates.chrome_pfq_informational,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
description='Chrome Performance test bot',
hw_tests=[config_lib.HWTestConfig(
'perf_v2', pool=constants.HWTEST_CHROME_PERF_POOL,
timeout=90 * 60, critical=True)],
use_chrome_lkgm=True,
useflags=append_useflags(['-cros-debug']),
)
site_config.AddTemplate(
'pre_flight_branch',
site_config.templates.internal,
site_config.templates.official_chrome,
build_type=constants.PFQ_TYPE,
build_timeout=20 * 60,
manifest_version=True,
branch=True,
master=True,
slave_configs=[],
vm_tests=[],
vm_tests_override=TRADITIONAL_VM_TESTS_SUPPORTED,
hw_tests=[],
hw_tests_override=[],
uprev=True,
overlays=constants.BOTH_OVERLAYS,
push_overlays=constants.BOTH_OVERLAYS,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Chrome-PFQ',
active_waterfall=waterfall.WATERFALL_RELEASE)
site_config.AddTemplate(
'internal_paladin',
site_config.templates.paladin,
site_config.templates.internal,
site_config.templates.official_chrome,
manifest=constants.OFFICIAL_MANIFEST,
overlays=constants.BOTH_OVERLAYS,
prebuilts=constants.PRIVATE,
vm_tests=[],
description=site_config.templates.paladin.description + ' (internal)',
)
# Used for paladin builders with nowithdebug flag (a.k.a -cros-debug)
site_config.AddTemplate(
'internal_nowithdebug_paladin',
site_config.templates.internal_paladin,
useflags=append_useflags(['-cros-debug']),
description=(site_config.templates.paladin.description +
' (internal, nowithdebug)'),
prebuilts=False,
)
# Internal incremental builders don't use official chrome because we want
# to test the developer workflow.
site_config.AddTemplate(
'internal_incremental',
site_config.templates.internal,
site_config.templates.incremental,
overlays=constants.BOTH_OVERLAYS,
description='Incremental Builds (internal)',
)
# A test-ap image is just a test image with a special profile enabled.
# Note that each board enabled for test-ap use has to have the testbed-ap
# profile linked to from its private overlay.
site_config.AddTemplate(
'test_ap',
site_config.templates.internal,
site_config.templates.no_vmtest_builder,
site_config.templates.default_hw_tests_override,
display_label=config_lib.DISPLAY_LABEL_UTILITY,
build_type=constants.INCREMENTAL_TYPE,
description='WiFi AP images used in testing',
profile='testbed-ap',
)
# Create tryjob build configs to help with stress testing.
site_config.AddTemplate(
'unittest_stress',
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
build_type=constants.TRYJOB_TYPE,
description='Run Unittests repeatedly to look for flake.',
builder_class_name='test_builders.UnittestStressBuilder',
active_waterfall=waterfall.WATERFALL_SWARMING,
# Make this available, so we can stress a previous build.
manifest_version=True,
)
site_config.AddTemplate(
'release',
site_config.templates.full,
site_config.templates.official,
site_config.templates.internal,
site_config.templates.default_hw_tests_override,
display_label=config_lib.DISPLAY_LABEL_RELEASE,
build_type=constants.CANARY_TYPE,
chroot_use_image=False,
suite_scheduling=True,
build_timeout=12 * 60 * 60 if is_release_branch else (7 * 60 + 50) * 60,
useflags=append_useflags(['-cros-debug']),
afdo_use=True,
manifest=constants.OFFICIAL_MANIFEST,
manifest_version=True,
images=['base', 'recovery', 'test', 'factory_install'],
sign_types=['recovery'],
push_image=True,
upload_symbols=True,
binhost_bucket='gs://chromeos-dev-installer',
binhost_key='RELEASE_BINHOST',
binhost_base_url='https://commondatastorage.googleapis.com/'
'chromeos-dev-installer',
dev_installer_prebuilts=True,
git_sync=False,
vm_tests=[
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'),
config_lib.VMTestConfig(constants.DEV_MODE_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)],
# Some release builders disable VMTests to be able to build on GCE, but
# still want VMTests enabled on trybot builders.
vm_tests_override=[
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'),
config_lib.VMTestConfig(constants.DEV_MODE_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)],
hw_tests=(hw_test_list.CtsGtsQualTests() +
hw_test_list.SharedPoolCanary()),
paygen=True,
signer_tests=True,
hwqual=True,
description="Release Builds (canary) (internal)",
chrome_sdk=True,
image_test=True,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Canaries',
)
### Release AFDO configs.
site_config.AddTemplate(
'release_afdo',
site_config.templates.release,
suite_scheduling=False,
hw_tests=(
hw_test_list.DefaultList(pool=constants.HWTEST_SUITES_POOL) +
hw_test_list.AFDOList()
),
push_image=False,
paygen=False,
dev_installer_prebuilts=False,
)
site_config.AddTemplate(
'release_afdo_generate',
site_config.templates.release_afdo,
afdo_generate_min=True,
afdo_use=False,
afdo_update_ebuild=True,
hw_tests=[hw_test_list.AFDORecordTest()],
hw_tests_override=[hw_test_list.AFDORecordTest(
pool=constants.HWTEST_TRYBOT_POOL,
file_bugs=False,
priority=constants.HWTEST_DEFAULT_PRIORITY,
)],
)
site_config.AddTemplate(
'release_afdo_use',
site_config.templates.release_afdo,
afdo_use=True,
)
site_config.AddTemplate(
'moblab_release',
site_config.templates.release,
description='Moblab release builders',
images=['base', 'recovery', 'test'],
afdo_use=False,
signer_tests=False,
hw_tests=[
config_lib.HWTestConfig(constants.HWTEST_MOBLAB_SUITE,
timeout=120*60),
config_lib.HWTestConfig(constants.HWTEST_BVT_SUITE,
warn_only=True),
hw_test_list.TastConfig(constants.HWTEST_TAST_CQ_SUITE,
warn_only=True),
config_lib.HWTestConfig(constants.HWTEST_INSTALLER_SUITE,
warn_only=True)],
)
# Factory and Firmware releases much inherit from these classes.
# Modifications for these release builders should go here.
# Naming conventions also must be followed. Factory and firmware branches
# must end in -factory or -firmware suffixes.
site_config.AddTemplate(
'factory',
site_config.templates.release,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
display_label=config_lib.DISPLAY_LABEL_FACTORY,
afdo_use=False,
chrome_sdk=False,
chrome_sdk_build_chrome=False,
description='Factory Builds',
dev_installer_prebuilts=False,
factory_toolkit=True,
hwqual=False,
images=['test', 'factory_install'],
image_test=False,
paygen=False,
signer_tests=False,
sign_types=['factory'],
upload_hw_test_artifacts=False,
upload_symbols=False,
)
site_config.AddTemplate(
'firmware_base',
site_config.templates.no_vmtest_builder,
display_label=config_lib.DISPLAY_LABEL_FIRMWARE,
images=[],
hwqual=False,
factory_toolkit=False,
packages=['virtual/chromeos-firmware'],
usepkg_build_packages=True,
sync_chrome=False,
chrome_sdk=False,
unittests=False,
hw_tests=[],
hw_tests_override=None,
dev_installer_prebuilts=False,
upload_hw_test_artifacts=False,
upload_symbols=False,
useflags=append_useflags(['chromeless_tty']),
signer_tests=False,
paygen=False,
image_test=False,
sign_types=['firmware'],
)
site_config.AddTemplate(
'firmware',
site_config.templates.release,
site_config.templates.firmware_base,
description='Firmware Canary',
manifest=constants.DEFAULT_MANIFEST,
afdo_use=False,
)
site_config.AddTemplate(
'depthcharge_firmware',
site_config.templates.firmware,
useflags=append_useflags(['depthcharge']))
site_config.AddTemplate(
'depthcharge_full_firmware',
site_config.templates.full,
site_config.templates.internal,
site_config.templates.firmware_base,
useflags=append_useflags(['depthcharge']),
description='Firmware Informational',
)
site_config.AddTemplate(
'payloads',
site_config.templates.internal,
site_config.templates.no_vmtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
build_type=constants.PAYLOADS_TYPE,
builder_class_name='release_builders.GeneratePayloadsBuilder',
description='Regenerate release payloads.',
# Sync to the code used to do the build the first time.
manifest_version=True,
# This is the actual work we want to do.
paygen=True,
upload_hw_test_artifacts=False,
active_waterfall=waterfall.WATERFALL_SWARMING,
)
site_config.AddTemplate(
'build_external_chrome',
useflags=append_useflags(['-%s' % constants.USE_CHROME_INTERNAL]),
)
# Tast is an alternate system for running integration tests.
# The expression specified here matches the union of the tast.mustpass-*
# Autotest server tests, which are used to run "important" Tast tests on
# real hardware in the lab.
site_config.AddTemplate(
'tast_vm_paladin_tests',
tast_vm_tests=[
config_lib.TastVMTestConfig('tast_vm_paladin',
['(!disabled && !informational)'])],
)
# The expression specified here matches the union of tast.mustpass-* and
# tast.informational-*.
site_config.AddTemplate(
'tast_vm_canary_tests',
tast_vm_tests=[
config_lib.TastVMTestConfig('tast_vm_canary', ['(!disabled)'])],
)
site_config.AddTemplate(
'moblab_vm_tests',
moblab_vm_tests=[
config_lib.MoblabVMTestConfig(constants.MOBLAB_VM_SMOKE_TEST_TYPE)],
)
def CreateBoardConfigs(site_config, boards_dict, ge_build_config):
"""Create mixin templates for each board."""
# Extract the full list of board names from GE data.
separate_board_names = set(config_lib.GeBuildConfigAllBoards(ge_build_config))
unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config)
unified_board_names = set([b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]
for b in unified_builds])
board_names = separate_board_names | unified_board_names
# TODO(crbug.com/648473): Remove these, after GE adds them to their data set.
board_names = board_names.union(boards_dict['all_boards'])
result = dict()
for board in board_names:
board_config = config_lib.BuildConfig(boards=[board])
if board in _brillo_boards:
board_config.apply(site_config.templates.brillo)
if board in _lakitu_boards:
board_config.apply(site_config.templates.lakitu)
if board in _lassen_boards:
board_config.apply(site_config.templates.lassen)
if board in ['x30evb']:
board_config.apply(site_config.templates.x30evb)
if board in _loonix_boards:
board_config.apply(site_config.templates.loonix)
if board in _moblab_boards:
board_config.apply(site_config.templates.moblab)
if board in _accelerator_boards:
board_config.apply(site_config.templates.accelerator)
if board in _termina_boards:
board_config.apply(site_config.templates.termina)
if board in _nofactory_boards:
board_config.apply(factory=False,
factory_toolkit=False,
factory_install_netboot=False,
images=remove_images(['factory_install']))
if board in _toolchains_from_source:
board_config.apply(usepkg_toolchain=False)
if board in _noimagetest_boards:
board_config.apply(image_test=False)
if board in _nohwqual_boards:
board_config.apply(hwqual=False)
if board in _norootfs_verification_boards:
board_config.apply(rootfs_verification=False)
if board in _base_layout_boards:
board_config.apply(disk_layout='base')
if board in _beaglebone_boards:
board_config.apply(site_config.templates.beaglebone)
if board == 'moblab-generic-vm':
board_config.apply(site_config.templates.moblab_vm_tests)
result[board] = board_config
return result
def CreateInternalBoardConfigs(site_config, boards_dict, ge_build_config):
"""Create mixin templates for each board."""
result = CreateBoardConfigs(site_config, boards_dict, ge_build_config)
for board in boards_dict['internal_boards']:
if board in result:
result[board].apply(site_config.templates.internal,
site_config.templates.official_chrome,
manifest=constants.OFFICIAL_MANIFEST)
return result
def UpdateBoardConfigs(board_configs, boards, *args, **kwargs):
"""Update "board_configs" for selected boards.
Args:
board_configs: Dict in CreateBoardConfigs format to filter from.
boards: Iterable of boards to update in the dict.
args: List of templates to apply.
kwargs: Individual keys to update.
Returns:
Copy of board_configs dict with boards boards update with templates
and values applied.
"""
result = board_configs.copy()
for b in boards:
result[b] = result[b].derive(*args, **kwargs)
return result
def ToolchainBuilders(site_config, boards_dict, ge_build_config):
"""Define templates used for toolchain builders.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
site_config.AddTemplate(
'toolchain',
# Full build, AFDO, latest-toolchain, -cros-debug, and simple-chrome.
site_config.templates.full,
site_config.templates.internal,
site_config.templates.official_chrome,
site_config.templates.no_vmtest_builder,
display_label=config_lib.DISPLAY_LABEL_TOOLCHAIN,
build_type=constants.TOOLCHAIN_TYPE,
buildslave_type=constants.GCE_BEEFY_BUILD_SLAVE_TYPE,
images=['base', 'test', 'recovery'],
build_timeout=(15 * 60 + 50) * 60,
# Need to re-enable platform_SyncCrash after issue crosbug/658864 is
# fixed. Need to re-enable network_VPNConnect.* tests after issue
# crosbug/585936 is fixed. Need to re-enable
# power_DarkResumeShutdownServer after issue crosbug/689598 is fixed.
# According to crosbug/653496 security_OpenFDs will not work for
# non-official builds, so we need to leave it permanently disabled.
# Need to reenable power_DarkResumeDisplay after crosbug/703250 is fixed.
# Need to reenable cheets_SELinuxTest after crosbug/693308 is fixed.
# Need to reenable security_SMMLocked when crosbug/654610 is fixed.
useflags=append_useflags(['-cros-debug',
'-tests_security_OpenFDs',
'-tests_platform_SyncCrash',
'-tests_network_VPNConnect.l2tpipsec_xauth',
'-tests_network_VPNConnect.l2tpipsec_psk',
'-tests_power_DarkResumeShutdownServer',
'-tests_power_DarkResumeDisplay',
'-tests_security_SMMLocked',
'-tests_cheets_SELinuxTest',
'thinlto']),
afdo_use=True,
latest_toolchain=True,
manifest=constants.OFFICIAL_MANIFEST,
manifest_version=True,
git_sync=False,
description="Toolchain Builds (internal)",
)
site_config.AddTemplate(
'gcc_toolchain',
site_config.templates.toolchain,
description='Full release build with next minor GCC toolchain revision',
useflags=append_useflags(['next_gcc']),
hw_tests=hw_test_list.ToolchainTestFull(constants.HWTEST_SUITES_POOL),
hw_tests_override=hw_test_list.ToolchainTestFull(
constants.HWTEST_SUITES_POOL),
)
site_config.AddTemplate(
'llvm_toolchain',
site_config.templates.toolchain,
description='Full release build with LLVM toolchain',
hw_tests=hw_test_list.ToolchainTestMedium(constants.HWTEST_SUITES_POOL),
hw_tests_override=hw_test_list.ToolchainTestMedium(
constants.HWTEST_SUITES_POOL),
)
site_config.AddTemplate(
'llvm_next_toolchain',
site_config.templates.llvm_toolchain,
description='Full release build with LLVM (next) toolchain',
useflags=append_useflags(['llvm-next']),
)
### Toolchain waterfall entries.
### Toolchain builder configs: 3 architectures {amd64,arm,arm64}
### x 1 toolchains {llvm-next}
### All of these builders should be slaves of 'master-toolchain'.
### Master toolchain config.
master = site_config.Add(
'master-toolchain',
site_config.templates.toolchain,
boards=[],
description='Toolchain master (all others are slaves).',
master=True,
sync_chrome=True,
health_alert_recipients=['c-compiler-chrome@google.com'],
health_threshold=1,
afdo_use=False,
active_waterfall=waterfall.WATERFALL_SWARMING,
buildslave_type=constants.GCE_BEEFY_BUILD_SLAVE_TYPE,
slave_configs=[],
# 3 PM UTC is 7 AM PST (no daylight savings)
schedule="0 15 * * *",
)
def toolchainSlaveHelper(name, board, *args, **kwargs):
master.AddSlaves([
site_config.Add(
name + '-llvm-next-toolchain',
site_config.templates.llvm_next_toolchain,
*args,
boards=[board],
active_waterfall=waterfall.WATERFALL_SWARMING,
hw_tests=hw_test_list.ToolchainTestMedium(
constants.HWTEST_MACH_POOL),
hw_tests_override=hw_test_list.ToolchainTestMedium(
constants.HWTEST_MACH_POOL),
**kwargs
)
])
# Create all waterfall slave builders.
toolchainSlaveHelper('amd64', 'samus')
toolchainSlaveHelper('arm', 'veyron_jaq')
toolchainSlaveHelper('arm64', 'elm')
#
# Create toolchain tryjob builders.
#
builder_to_boards_dict = config_lib.GroupBoardsByBuilder(
ge_build_config[config_lib.CONFIG_TEMPLATE_BOARDS])
toolchain_tryjob_boards = builder_to_boards_dict[
config_lib.CONFIG_TEMPLATE_RELEASE] | boards_dict['all_boards']
site_config.AddForBoards(
'llvm-toolchain',
toolchain_tryjob_boards,
board_configs,
site_config.templates.llvm_toolchain,
)
site_config.AddForBoards(
'llvm-next-toolchain',
toolchain_tryjob_boards,
board_configs,
site_config.templates.llvm_next_toolchain,
)
site_config.Add(
'llvm-clang-tidy-toolchain',
site_config.templates.toolchain,
site_config.templates.no_hwtest_builder,
description='Full release build with LLVM toolchain, with clang-tidy)',
chrome_sdk=False,
# Run clang-tidy specific stages.
builder_class_name='clang_tidy_builders.ClangTidyBuilder',
useflags=append_useflags(['clang_tidy']),
boards=['grunt'],
active_waterfall=waterfall.WATERFALL_SWARMING,
# Weekly on Sunday 3 AM UTC
schedule='0 0 3 * * 0 *',
)
def PreCqBuilders(site_config, boards_dict, ge_build_config):
"""Create all build configs associated with the PreCQ.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
# The PreCQ Launcher doesn't limit eternal PreCQ builds to external
# CLs. as a hack, use internal checkouts for external builds so
# they can apply (and ignore) internal CLs. crbug.com/882965
for b in _arm_external_boards | _x86_external_boards:
board_configs[b].apply(site_config.templates.internal)
site_config.AddTemplate(
'pre_cq',
site_config.templates.paladin,
display_label=config_lib.DISPLAY_LABEL_PRECQ,
luci_builder=config_lib.LUCI_BUILDER_PRECQ,
active_waterfall=waterfall.WATERFALL_SWARMING,
build_type=constants.PRE_CQ_TYPE,
pre_cq=True,
archive=False,
chrome_sdk=False,
chroot_replace=True,
debug_symbols=False,
prebuilts=False,
cpe_export=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)],
vm_tests_override=None,
description='Verifies compilation, building an image, and vm/unit tests '
'if supported.',
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Pre-CQ',
sanity_check_threshold=3,
)
# Pre-CQ targets that only check compilation and unit tests.
site_config.AddTemplate(
'unittest_only_pre_cq',
site_config.templates.pre_cq,
site_config.templates.no_vmtest_builder,
description='Verifies compilation and unit tests only',
compilecheck=True,
)
# Pre-CQ targets that don't run VMTests.
site_config.AddTemplate(
'no_vmtest_pre_cq',
site_config.templates.pre_cq,
site_config.templates.no_vmtest_builder,
description='Verifies compilation, building an image, and unit tests '
'if supported.',
)
# Pre-CQ targets that only check compilation.
site_config.AddTemplate(
'compile_only_pre_cq',
site_config.templates.unittest_only_pre_cq,
description='Verifies compilation only',
unittests=False,
)
site_config.AddWithoutTemplate(
'pre-cq-launcher',
site_config.templates.paladin,
site_config.templates.internal_paladin,
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
boards=[],
display_label=config_lib.DISPLAY_LABEL_PRECQ,
build_type=constants.PRE_CQ_LAUNCHER_TYPE,
active_waterfall=waterfall.WATERFALL_SWARMING,
description='Launcher for Pre-CQ builders',
manifest_version=False,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Pre-CQ',
schedule='with 3m interval',
)
# Add a pre-cq config for every board.
site_config.AddForBoards(
'pre-cq',
boards_dict['all_boards'],
board_configs,
site_config.templates.pre_cq,
)
site_config.AddForBoards(
'no-vmtest-pre-cq',
boards_dict['all_boards'],
board_configs,
site_config.templates.no_vmtest_pre_cq,
)
site_config.AddForBoards(
'compile-only-pre-cq',
boards_dict['all_boards'],
board_configs,
site_config.templates.compile_only_pre_cq,
)
site_config.Add(
constants.BINHOST_PRE_CQ,
site_config.templates.pre_cq,
site_config.templates.no_vmtest_pre_cq,
site_config.templates.internal,
boards=[],
binhost_test=True,
)
# Wifi specific PreCQ.
site_config.AddTemplate(
'wificell_pre_cq',
site_config.templates.pre_cq,
unittests=False,
hw_tests=hw_test_list.WiFiCellPoolPreCQ(),
hw_tests_override=hw_test_list.WiFiCellPoolPreCQ(),
archive=True,
image_test=False,
description='WiFi tests acting as pre-cq for WiFi related changes',
)
site_config.AddGroup(
'mixed-wificell-pre-cq',
site_config.Add(
'winky-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['winky']),
site_config.Add(
'veyron_speedy-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['veyron_speedy']),
site_config.Add(
'veyron_jerry-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['veyron_jerry']),
site_config.Add(
'daisy-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['daisy']),
site_config.Add(
'lulu-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['lulu']),
site_config.Add(
'cyan-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['cyan']),
site_config.Add(
'elm-wificell-pre-cq',
site_config.templates.wificell_pre_cq,
board_configs['elm']),
)
# Bluestreak specific PreCQ.
site_config.Add(
'bluestreak-pre-cq',
board_configs['guado'],
site_config.templates.pre_cq,
hw_tests=hw_test_list.BluestreakPoolPreCQ(),
hw_tests_override=hw_test_list.BluestreakPoolPreCQ(),
archive=True,
description='Bluestreak tests as pre-cq for CFM related changes',
)
site_config.Add(
'signer-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='test_builders.SignerTestsBuilder',
description='Run the signer unittests.',
)
site_config.Add(
'chromite-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='test_builders.ChromiteTestsBuilder',
description='Run the chromite network unittests.',
)
site_config.Add(
'cbuildbot-launch-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='test_builders.CbuildbotLaunchTestBuilder',
description='Run cbuildbot_launch test builds.',
)
# Pre-cq for lakitu's public overlay.
site_config.Add(
'lakitu-external-pre-cq',
site_config.templates.pre_cq,
board_configs['lakitu'],
site_config.templates.lakitu,
site_config.templates.external,
useflags=append_useflags(['-chrome_internal']),
)
site_config.AddWithoutTemplate(
'chromeos-infra-puppet-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='infra_builders.PuppetPreCqBuilder',
use_sdk=True,
build_timeout=60 * 60,
description='Test Puppet specs',
doc='https://chrome-internal.googlesource.com/'
'chromeos/chromeos-admin/+/HEAD/puppet/README.md',
)
site_config.AddWithoutTemplate(
'chromeos-infra-go-pre-cq',
site_config.templates.pre_cq,
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='infra_builders.InfraGoPreCqBuilder',
use_sdk=True,
build_timeout=60 * 60,
description='Test building Chromium OS infra Go binaries',
doc='https://goto.google.com/cros-infra-go-packaging',
)
site_config.AddWithoutTemplate(
'chromeos-infra-unittests-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='infra_builders.InfraUnittestsPreCqBuilder',
use_sdk=True,
build_timeout=60 * 60,
description='Run unittests for infra repositories',
)
def AndroidPfqBuilders(site_config, boards_dict, ge_build_config):
"""Create all build configs associated with the Android PFQ.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
# Generic template shared by all Android versions.
site_config.AddTemplate(
'generic_android_pfq',
site_config.templates.no_vmtest_builder,
site_config.templates.default_hw_tests_override,
build_type=constants.ANDROID_PFQ_TYPE,
uprev=False,
overlays=constants.BOTH_OVERLAYS,
manifest_version=True,
android_rev=constants.ANDROID_REV_LATEST,
description='Preflight Android Uprev & Build (internal)',
active_waterfall=waterfall.WATERFALL_SWARMING,
)
# Template for Android NYC.
site_config.AddTemplate(
'nyc_android_pfq',
site_config.templates.generic_android_pfq,
display_label=config_lib.DISPLAY_LABEL_NYC_ANDROID_PFQ,
android_package='android-container-nyc',
android_import_branch=constants.ANDROID_NYC_BUILD_BRANCH,
android_gts_build_branch='git_nyc-mr2-dev',
)
# Template for Android Master.
site_config.AddTemplate(
'pi_android_pfq',
site_config.templates.generic_android_pfq,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_PI_ANDROID_PFQ,
android_package='android-container-pi',
android_import_branch=constants.ANDROID_PI_BUILD_BRANCH,
)
# Template for Android Master.
site_config.AddTemplate(
'mst_android_pfq',
site_config.templates.generic_android_pfq,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_MST_ANDROID_PFQ,
android_package='android-container-master-arc-dev',
android_import_branch=constants.ANDROID_MST_BUILD_BRANCH,
)
# Mixin for masters.
site_config.AddTemplate(
'master_android_pfq_mixin',
site_config.templates.internal,
site_config.templates.no_vmtest_builder,
boards=[],
master=True,
push_overlays=constants.BOTH_OVERLAYS,
)
# Android PFQ masters.
# Any additions of Android PFQ masters should be reflected by a
# change in lib/constants.py SOM_BUILDS to add Sheriff-o-Matic coverage.
# Android MST master.
mst_master_config = site_config.Add(
constants.MST_ANDROID_PFQ_MASTER,
site_config.templates.mst_android_pfq,
site_config.templates.master_android_pfq_mixin,
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 150m interval',
)
_mst_hwtest_boards = frozenset([])
_mst_no_hwtest_boards = frozenset([])
_mst_no_hwtest_experimental_boards = frozenset([])
_mst_no_hwtest_experimental_boards = frozenset([])
_mst_vmtest_boards = frozenset([])
# Android PI master.
pi_master_config = site_config.Add(
constants.PI_ANDROID_PFQ_MASTER,
site_config.templates.pi_android_pfq,
site_config.templates.master_android_pfq_mixin,
buildslave_type=constants.GCE_BEEFY_BUILD_SLAVE_TYPE,
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 60m interval',
)
_pi_no_hwtest_boards = frozenset([])
_pi_no_hwtest_experimental_boards = frozenset([])
_pi_hwtest_boards = frozenset([
'caroline-arcnext',
'grunt',
'kevin-arcnext',
])
_pi_hwtest_experimental_boards = frozenset([
'eve-arcnext',
])
_pi_vmtest_boards = frozenset([
'betty-arcnext'
])
_pi_vmtest_experimental_boards = frozenset([])
# Android NYC master.
nyc_master_config = site_config.Add(
constants.NYC_ANDROID_PFQ_MASTER,
site_config.templates.nyc_android_pfq,
site_config.templates.master_android_pfq_mixin,
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 150m interval',
)
_nyc_hwtest_boards = frozenset([
'cyan',
'samus',
'veyron_minnie',
])
_nyc_no_hwtest_boards = frozenset([
'bob',
'caroline',
'coral',
'hana',
'reef',
])
_nyc_no_hwtest_experimental_boards = frozenset([])
_nyc_vmtest_boards = frozenset([
'betty',
'betty-arc64',
])
# Android MST slaves.
mst_master_config.AddSlaves(
site_config.AddForBoards(
'mst-android-pfq',
_mst_hwtest_boards,
board_configs,
site_config.templates.mst_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_no_hwtest_boards,
board_configs,
site_config.templates.mst_android_pfq,
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_no_hwtest_experimental_boards,
board_configs,
site_config.templates.mst_android_pfq,
important=False,
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_vmtest_boards,
board_configs,
site_config.templates.mst_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
)
)
# Android PI slaves.
pi_master_config.AddSlaves(
site_config.AddForBoards(
'pi-android-pfq',
_pi_hwtest_boards,
board_configs,
site_config.templates.pi_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_no_hwtest_boards,
board_configs,
site_config.templates.pi_android_pfq,
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_no_hwtest_experimental_boards,
board_configs,
site_config.templates.pi_android_pfq,
important=False,
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_hwtest_experimental_boards,
board_configs,
site_config.templates.pi_android_pfq,
important=False,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_vmtest_boards,
board_configs,
site_config.templates.pi_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_vmtest_experimental_boards,
board_configs,
site_config.templates.pi_android_pfq,
important=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
)
)
# Android NYC slaves.
nyc_master_config.AddSlaves(
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_hwtest_boards,
board_configs,
site_config.templates.nyc_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_no_hwtest_boards,
board_configs,
site_config.templates.nyc_android_pfq,
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_no_hwtest_experimental_boards,
board_configs,
site_config.templates.nyc_android_pfq,
important=False,
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_vmtest_boards,
board_configs,
site_config.templates.nyc_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'),],
)
)
def FullBuilders(site_config, boards_dict, ge_build_config):
"""Create all full builders.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
active_builders = frozenset([
'amd64-generic',
'arm-generic',
'daisy',
'kevin',
'kevin64',
'lakitu',
'oak',
'tael',
'tatl',
])
external_board_configs = CreateBoardConfigs(
site_config, boards_dict, ge_build_config)
site_config.AddForBoards(
config_lib.CONFIG_TYPE_FULL,
boards_dict['all_full_boards'],
external_board_configs,
site_config.templates.full,
site_config.templates.build_external_chrome,
internal=False,
manifest_repo_url=site_config.params['MANIFEST_URL'],
overlays=constants.PUBLIC_OVERLAYS,
prebuilts=constants.PUBLIC)
master_config = site_config.Add(
'master-full',
site_config.templates.full,
site_config.templates.internal,
site_config.templates.build_external_chrome,
boards=[],
master=True,
manifest_version=True,
overlays=constants.PUBLIC_OVERLAYS,
slave_configs=[],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='0 */3 * * *',
)
master_config.AddSlaves(
site_config.ApplyForBoards(
config_lib.CONFIG_TYPE_FULL,
active_builders,
active_waterfall=waterfall.WATERFALL_SWARMING,
manifest_version=True,
)
)
def CqBuilders(site_config, boards_dict, ge_build_config):
"""Create all CQ build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
site_config.AddTemplate(
'cq_luci_slave',
active_waterfall=waterfall.WATERFALL_SWARMING,
build_affinity=True,
luci_builder=config_lib.LUCI_BUILDER_CQ,
)
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
_separate_boards = boards_dict['all_boards']
_unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config)
_unified_board_names = set([b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]
for b in _unified_builds])
_paladin_boards = _separate_boards | _unified_board_names
# List of paladin boards where the regular paladin config is important.
_paladin_important_boards = frozenset([
'amd64-generic',
'arm-generic',
'auron_paine',
'auron_yuna',
'atlas',
'beaglebone',
'betty',
'betty-arc64',
'betty-arcnext',
'bob',
'caroline',
'caroline-arcnext',
'cave',
'chell',
'cheza',
'coral',
'cyan',
'daisy',
'daisy_skate',
'daisy_spring',
'edgar',
'elm',
'eve',
'falco',
'fizz',
'gale',
'glados',
'grunt',
'guado_moblab',
'hana',
'kevin',
'kevin-arcnext',
'kip',
'kukui',
'lakitu',
'lakitu-gpu',
'lakitu-st',
'leon',
'link',
'monroe',
'moblab-generic-vm',
'nami',
'nyan_big',
'nyan_kitty',
'oak',
'octopus',
'panther',
'peach_pit',
'peppy',
'poppy',
'quawks',
'reef',
'samus',
'scarlet',
'sentry',
'stout',
'tael',
'tatl',
'terra',
'tidus',
'tricky',
'veyron_jaq',
'veyron_jerry',
'veyron_mighty',
'veyron_minnie',
'veyron_rialto',
'veyron_speedy',
'veyron_tiger',
'winky',
'wizpig',
'wolf',
])
# Paladin configs that exist and should be important as soon as they are
# shown to be green. All new paladins should start in this group and get
# promoted to _paladin_important_boards.
#
# A paladin is generally considered good enough for important if it can
# pass the last ~20 build attempts, e.g. the builder page shows mostly green.
# Note that paladins are expected to fail occasionally as they block bad CLs
# from landing, a red paladin from a bad CL in the CQ is a working paladin.
#
# If the device is to be used with HW testing, the standard is higher, and
# the device should be proven to be stable for at least three weeks.
# Generally only PVT/MP stage devices should be used with HW testing due
# to stability requirements for the commit queue. If a EVT/DVT device is
# to be deployed in a HW test capacity as a paladin, it must be approved
# by a member of the associated product team, the test infrastructure team,
# and the CI team.
#
# The definition of what paladins run HW tests are in the
# _paladin_hwtest_assignments table further down this script.
_paladin_new_boards = frozenset([
'dragonegg', # contact:yueherngl@
'eve-campfire', # contact:yueherngl@
'nocturne',
'rammus', # contact:zhuohao@
'kalista', # contact:zhuohao@
])
# Paladin configs that exist and should stay as experimental until further
# notice, preferably with a comment indicating why and a bug.
_paladin_experimental_boards = _paladin_new_boards | frozenset([
'capri', # contact:ghines@
'capri-zfpga', # contact:victoryang@
'cobblepot', # contact:jkoleszar@
'eve-arcnext', # contact: ihf@ (crbug.com/826755)
'fizz-accelerator', # contact:perley@
'gonzo', # contact:icoolidge@
'guado', # contact:egemih@
'guado-accelerator', # contact:perley@ (crbug.com/748635)
'lasilla-ground', # contact:jemele@
'lasilla-sky', # contact:jemele@
'macchiato-ground', # contact:jemele@
'octavius', # contact:dpjacques@
'romer', # contact:michaelho@
'whirlwind', # contact:mxt@ (crbug.com/794242, crbug.com/819287)
'wooten', # contact:icoolidge@
'nyan_blaze', # contact: xixuan@
])
assert not (_paladin_experimental_boards & _paladin_important_boards), (
'Experimental and important paladin board sets must be disjoint.')
_paladin_active = _paladin_important_boards | _paladin_experimental_boards
_paladin_simple_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
])
_paladin_devmode_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
])
_paladin_cros_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
])
_paladin_smoke_vmtest_boards = frozenset([
'betty',
'betty-arc64',
'betty-arcnext',
])
_paladin_default_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
])
# Jetstream devices run unique hw tests
_paladin_jetstream_hwtest_boards = frozenset([
'whirlwind',
'gale',
])
_paladin_moblab_hwtest_boards = frozenset([
'guado_moblab',
])
_paladin_chroot_replace_boards = frozenset([
'daisy_spring',
])
_paladin_separate_symbols = frozenset([
'amd64-generic',
])
_paladin_bluestreak_hwtest_boards = frozenset([
'guado',
])
_paladin_enable_skylab_hwtest = frozenset([
'nyan_blaze',
'reef',
])
### Master paladin (CQ builder).
master_config = site_config.Add(
'master-paladin',
site_config.templates.paladin,
site_config.templates.internal_paladin,
boards=[],
master=True,
binhost_test=True,
push_overlays=constants.BOTH_OVERLAYS,
description='Commit Queue master (all others are slaves)',
auto_reboot=True, # TODO(dgarrett): Disable chroot.img stable.
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 2m interval',
)
### Other paladins (CQ builders).
# These are slaves of the master paladin by virtue of matching
# in a few config values (e.g. 'build_type', 'branch', etc). If
# they are not 'important' then they are ignored slaves.
# TODO(mtennant): This master-slave relationship should be specified
# here in the configuration, rather than GetSlavesForMaster().
# Something like the following:
# master_paladin = site_config.AddConfig(internal_paladin, ...)
# master_paladin.AddSlave(site_config.AddConfig(internal_paladin, ...))
for board in _paladin_boards:
assert board in board_configs, '%s not in board_configs' % board
config_name = '%s-%s' % (board, constants.PALADIN_TYPE)
customizations = config_lib.BuildConfig()
base_config = board_configs[board]
if board in _unified_board_names:
for unibuild in _unified_builds:
if board == unibuild[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]:
models = []
for model in unibuild[config_lib.CONFIG_TEMPLATE_MODELS]:
name = model[config_lib.CONFIG_TEMPLATE_MODEL_NAME]
lab_board_name = model[config_lib.CONFIG_TEMPLATE_MODEL_BOARD_NAME]
if (config_lib.CONFIG_TEMPLATE_MODEL_CQ_TEST_ENABLED in model
and model[config_lib.CONFIG_TEMPLATE_MODEL_CQ_TEST_ENABLED]):
models.append(config_lib.ModelTestConfig(name, lab_board_name))
customizations.update(models=models)
if board in _paladin_enable_skylab_hwtest:
customizations.update(enable_skylab_hw_tests=True)
if board in _paladin_moblab_hwtest_boards:
customizations.update(
hw_tests=[
config_lib.HWTestConfig(
constants.HWTEST_MOBLAB_QUICK_SUITE,
timeout=90*60,
pool=constants.HWTEST_PALADIN_POOL)
],
hw_tests_override=None)
if board in _paladin_jetstream_hwtest_boards:
customizations.update(
hw_tests=[
config_lib.HWTestConfig(
constants.HWTEST_JETSTREAM_COMMIT_SUITE,
pool=constants.HWTEST_PALADIN_POOL)
],
hw_tests_override=None)
if board in _paladin_bluestreak_hwtest_boards:
customizations.update(
hw_tests=hw_test_list.BluestreakPoolPreCQ(),
hw_tests_override=hw_test_list.BluestreakPoolPreCQ())
if board in _paladin_experimental_boards:
customizations.update(important=False)
if board in _paladin_chroot_replace_boards:
customizations.update(chroot_replace=True)
if (board in boards_dict['internal_boards']
or board in _unified_board_names):
customizations = customizations.derive(
site_config.templates.internal,
site_config.templates.official_chrome,
manifest=constants.OFFICIAL_MANIFEST)
if board in _paladin_separate_symbols:
customizations.update(separate_debug_symbols=True)
if board not in _paladin_default_vmtest_boards:
vm_tests = []
if board in _paladin_simple_vmtest_boards:
vm_tests.append(
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE))
if board in _paladin_cros_vmtest_boards:
vm_tests.append(config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE))
if board in _paladin_devmode_vmtest_boards:
vm_tests.append(config_lib.VMTestConfig(constants.DEV_MODE_TEST_TYPE))
if board in _paladin_smoke_vmtest_boards:
vm_tests.append(
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'))
customizations.update(vm_tests=vm_tests)
if site_config.templates.paladin.vm_tests_override is not None:
# Make sure any new tests are also in override.
override = site_config.templates.paladin.vm_tests_override[:]
for test in vm_tests:
if test not in override:
override.append(test)
customizations.update(vm_tests_override=override)
if base_config.get('internal'):
customizations.update(
prebuilts=constants.PRIVATE,
description=(site_config.templates.paladin.description +
' (internal)'))
else:
customizations.update(prebuilts=constants.PUBLIC)
if board in _paladin_active:
customizations.update(
site_config.templates.cq_luci_slave,
)
if board in _lakitu_boards:
customizations.update(
site_config.templates.lakitu_paladin_test_customizations)
config = site_config.Add(
config_name,
site_config.templates.paladin,
customizations,
base_config,
)
if config.active_waterfall:
master_config.AddSlave(config)
# N.B. The ordering of columns here is coupled to the ordering of
# suites returned by DefaultListCQ(). If you change the ordering here,
# you must also change the ordering there.
#
# CAUTION: Only add devices to this table which are known to be stable in
# the HW test lab, even low rates of flake from these devices quickly
# add up and can destabilize the commit queue.
#
# TODO: Fill in any notable gaps in this table. crbug.com/730076
# pylint: disable=bad-continuation, bad-whitespace, line-too-long
_paladin_hwtest_assignments = frozenset([
# bvt-inline bvt-cq bvt-arc bvt-tast-cq family
('link', None, None, None), # ivybridge
(None, None, None, None), # daisy (Exynos5250)
('wolf', 'peppy', None, 'peppy'), # slippy (HSW)
('peach_pit', None, None, None), # peach (Exynos5420)
('winky', 'kip', None, 'kip'), # rambi (BYT)
('nyan_big', 'nyan_kitty', None, 'nyan_kitty'), # nyan (K1)
('auron_paine', 'tidus', 'auron_yuna', 'tidus'), # auron (BDW)
('veyron_mighty', 'veyron_speedy', 'veyron_minnie', 'veyron_speedy'), # pinky (RK3288)
('wizpig', 'edgar', 'cyan', 'edgar'), # strago (BSW)
('cave', 'sentry', None, 'sentry'), # glados (SKL)
('elm', None, 'hana', None), # oak (MTK8173)
('bob', None, 'kevin', None), # gru (RK3399)
('reef', None, None, None), # reef (APL)
('coral', None, None, None), # coral (APL)
(None, 'eve', 'soraka', 'eve'), # poppy (KBL)
('nocturne', None, None, 'nocturne'), # Nocturne (KBL)
(None, None, 'kevin-arcnext', None), # gru + arcnext
(None, None, 'caroline-arcnext', None), # arcnext
('nyan_blaze', None, None, None), # Add for Skylab test
('scarlet' , None, None, None), # scarlet (RK3399 unibuild)
('grunt', None, 'grunt', None), # grunt (AMD unibuild)
])
# pylint: enable=bad-continuation, bad-whitespace, line-too-long
sharded_hw_tests = hw_test_list.DefaultListCQ()
# Run provision suite first everywhere.
default_tests = [sharded_hw_tests.pop(0)]
config_default_reset = set()
for board_assignments in _paladin_hwtest_assignments:
assert len(board_assignments) == len(sharded_hw_tests)
for board, suite in zip(board_assignments, sharded_hw_tests):
if board is None:
continue
config_name = '%s-%s' % (board, constants.PALADIN_TYPE)
# Only configurate hw_tests for unified builds if they have specified
# models they want to test against (based on lab provisioning)
if (board in _unified_board_names and
not site_config[config_name]['models']):
continue
if config_name not in config_default_reset:
site_config[config_name]['hw_tests'] = default_tests[:]
config_default_reset.add(config_name)
site_config[config_name]['hw_tests'] += [suite]
#
# Paladins with alternative configs.
#
site_config.AddForBoards(
'nowithdebug-paladin',
['amd64-generic'],
board_configs,
site_config.templates.paladin,
site_config.templates.internal_nowithdebug_paladin,
)
master_config.AddSlaves([
site_config.Add(
'chell-nowithdebug-paladin',
site_config.templates.paladin,
site_config.templates.internal_nowithdebug_paladin,
site_config.templates.cq_luci_slave,
boards=['chell'],
),
site_config.Add(
'incremental-paladin',
site_config.templates.paladin,
site_config.templates.internal_paladin,
site_config.templates.no_hwtest_builder,
site_config.templates.cq_luci_slave,
boards=['reef'],
build_before_patching=True,
prebuilts=False,
compilecheck=True,
unittests=False,
important=False,
),
])
# Used for builders which build completely from source except Chrome.
# These boards pass with -clang-clean CFLAG, so ensure they stay that way.
master_config.AddSlaves(
site_config.AddForBoards(
'full-compile-paladin',
['falco', 'nyan_kitty'],
board_configs,
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
site_config.templates.cq_luci_slave,
board_replace=True,
chrome_sdk=False,
compilecheck=True,
cpe_export=False,
debug_symbols=False,
prebuilts=False,
unittests=False,
upload_hw_test_artifacts=False,
vm_tests=[],
)
)
site_config.Add(
'amd64-generic-asan-paladin',
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
board_configs['amd64-generic'],
site_config.templates.asan,
description='Paladin build with Address Sanitizer (Clang)',
)
site_config.Add(
'betty-asan-paladin',
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
board_configs['betty'],
site_config.templates.asan,
description='Paladin build with Address Sanitizer (Clang)',
)
site_config.Add(
'amd64-generic-ubsan-paladin',
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
board_configs['amd64-generic'],
site_config.templates.ubsan,
description='Paladin build with Undefined Behavior Sanitizer (Clang)',
)
def IncrementalBuilders(site_config, boards_dict, ge_build_config):
"""Create all incremental build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
site_config.AddTemplate(
'incremental_affinity',
active_waterfall=waterfall.WATERFALL_SWARMING,
build_affinity=True,
luci_builder=config_lib.LUCI_BUILDER_INCREMENTAL,
)
master_config = site_config.Add(
'master-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
boards=[],
master=True,
manifest_version=True,
slave_configs=[],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 10m interval',
)
# Build external source, for an internal board.
master_config.AddSlave(
site_config.Add(
'daisy-incremental',
site_config.templates.incremental,
site_config.templates.incremental_affinity,
board_configs['daisy'],
site_config.templates.external,
manifest_version=True,
useflags=append_useflags(['-chrome_internal']),
)
)
master_config.AddSlave(
site_config.Add(
'amd64-generic-incremental',
site_config.templates.incremental,
site_config.templates.incremental_affinity,
site_config.templates.no_vmtest_builder,
board_configs['amd64-generic'],
manifest_version=True,
)
)
master_config.AddSlave(
site_config.Add(
'betty-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.incremental_affinity,
boards=['betty'],
manifest_version=True,
vm_tests=getInfoVMTest(),
vm_tests_override=getInfoVMTest(),
)
)
master_config.AddSlave(
site_config.Add(
'chell-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.incremental_affinity,
boards=['chell'],
manifest_version=True,
)
)
master_config.AddSlave(
site_config.Add(
'lakitu-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.incremental_affinity,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu'],
site_config.templates.lakitu_test_customizations,
manifest_version=True,
)
)
#
# Available, but not regularly scheduled.
#
site_config.Add(
'x32-generic-incremental',
site_config.templates.incremental,
site_config.templates.no_vmtest_builder,
board_configs['x32-generic'],
)
site_config.Add(
'beaglebone-incremental',
site_config.templates.incremental,
site_config.templates.beaglebone,
boards=['beaglebone'],
description='Incremental Beaglebone Builder',
)
site_config.Add(
'lakitu-gpu-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu-gpu'],
site_config.templates.lakitu_test_customizations,
)
site_config.Add(
'lakitu-st-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu-st'],
site_config.templates.lakitu_test_customizations,
)
site_config.Add(
'lakitu_next-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu_next'],
site_config.templates.lakitu_test_customizations,
)
def ReleaseAfdoBuilders(site_config, boards_dict, ge_build_config):
"""Create AFDO Performance tryjobs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
# Now generate generic release-afdo configs if we haven't created anything
# more specific above already. release-afdo configs are builders that do AFDO
# profile collection and optimization in the same builder. Used by developers
# that want to measure performance changes caused by their changes.
for board in boards_dict['all_release_boards']:
base = board_configs[board]
config_name = '%s-%s' % (board, config_lib.CONFIG_TYPE_RELEASE_AFDO)
if config_name in site_config:
continue
generate_config_name = (
'%s-%s-%s' % (board,
config_lib.CONFIG_TYPE_RELEASE_AFDO,
'generate'))
use_config_name = '%s-%s-%s' % (board,
config_lib.CONFIG_TYPE_RELEASE_AFDO,
'use')
# We can't use AFDO data if afdo_use is disabled for this board.
if not base.get('afdo_use', True):
continue
site_config.AddGroup(
config_name,
site_config.Add(
generate_config_name,
site_config.templates.release_afdo_generate,
base
),
site_config.Add(
use_config_name,
site_config.templates.release_afdo_use,
base
),
)
def InformationalBuilders(site_config, boards_dict, ge_build_config):
"""Create all informational builders.
We have a number of informational builders that are built, but whose output is
not directly used for anything other than reporting success or failure.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
external_board_configs = CreateBoardConfigs(
site_config, boards_dict, ge_build_config)
internal_board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
_chrome_boards = frozenset(
board for board, config in internal_board_configs.iteritems()
if config.get('sync_chrome', True))
hw_test_list = HWTestList(ge_build_config)
_chrome_informational_hwtest_boards = frozenset([
'caroline',
'eve',
'peach_pit',
'reks',
'tricky',
'veyron_minnie',
])
_chrome_informational_swarming_boards = frozenset([
'eve',
])
# We have to mark all autogenerated PFQs as not important so the master
# does not wait for them. https://crbug.com/386214
# If you want an important PFQ, you'll have to declare it yourself.
informational_boards = (
(boards_dict['all_release_boards'] & _chrome_boards))
_tot_chrome_pfq_informational_board_configs = UpdateBoardConfigs(
internal_board_configs,
_chrome_informational_hwtest_boards,
hw_tests=hw_test_list.DefaultListChromePFQInformational(
pool=constants.HWTEST_CONTINUOUS_POOL))
_tot_chrome_pfq_informational_board_configs = UpdateBoardConfigs(
_tot_chrome_pfq_informational_board_configs,
_chrome_informational_swarming_boards,
active_waterfall=waterfall.WATERFALL_SWARMING)
site_config.AddForBoards(
'tot-chrome-pfq-informational',
informational_boards,
_tot_chrome_pfq_informational_board_configs,
site_config.templates.chrome_pfq_informational)
site_config.Add(
'amd64-generic-asan',
site_config.templates.asan,
site_config.templates.incremental,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_CHROME_INFORMATIONAL,
boards=['amd64-generic'],
description='Build with Address Sanitizer (Clang)',
# THESE IMAGES CAN DAMAGE THE LAB and cannot be used for hardware testing.
disk_layout='4gb-rootfs',
active_waterfall=waterfall.WATERFALL_SWARMING,
# Every 3 hours.
schedule='0 */3 * * *',
)
site_config.Add(
'amd64-generic-tot-asan-informational',
site_config.templates.tot_asan_informational,
site_config.templates.no_hwtest_builder,
boards=['amd64-generic'],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 30m interval',
)
site_config.Add(
'betty-asan',
site_config.templates.asan,
site_config.templates.incremental,
site_config.templates.no_hwtest_builder,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_CHROME_INFORMATIONAL,
boards=['betty'],
description='Build with Address Sanitizer (Clang)',
active_waterfall=waterfall.WATERFALL_SWARMING,
# Once every day. 3 PM UTC is 7 AM PST (no daylight savings).
schedule='0 15 * * *'
)
site_config.Add(
'betty-tot-asan-informational',
site_config.templates.tot_asan_informational,
site_config.templates.no_hwtest_builder,
boards=['betty'],
)
site_config.Add(
'amd64-generic-fuzzer',
site_config.templates.fuzzer,
boards=['amd64-generic'],
description='Build for fuzzing testing',
# THESE IMAGES CAN DAMAGE THE LAB and cannot be used for hardware testing.
disk_layout='4gb-rootfs',
active_waterfall=waterfall.WATERFALL_SWARMING,
# Every 3 hours.
schedule='0 */3 * * *'
)
site_config.Add(
'amd64-generic-ubsan',
site_config.templates.ubsan,
site_config.templates.incremental,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_CHROME_INFORMATIONAL,
boards=['amd64-generic'],
description='Build with Undefined Behavior Sanitizer (Clang)',
# THESE IMAGES CAN DAMAGE THE LAB and cannot be used for hardware testing.
disk_layout='16gb-rootfs',
active_waterfall=waterfall.WATERFALL_SWARMING,
# Every 3 hours.
schedule='0 */3 * * *',
)
site_config.Add(
'amd64-generic-ubsan-fuzzer',
site_config.templates.fuzzer,
boards=['amd64-generic'],
profile='ubsan-fuzzer',
description='Build for fuzzing testing',
gs_path='gs://chromeos-fuzzing-artifacts/libfuzzer-ubsan',
disk_layout='4gb-rootfs',
active_waterfall=waterfall.WATERFALL_SWARMING,
# Every 3 hours.
schedule='0 */3 * * *'
)
site_config.Add(
'amd64-generic-goma-canary-chromium-pfq-informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
description='Test canary versions of goma.',
boards=[
'amd64-generic',
],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 30m interval',
goma_client_type='candidate',
)
_chrome_perf_boards = frozenset([
'daisy',
'reef',
])
site_config.AddForBoards(
'chrome-perf',
_chrome_perf_boards,
internal_board_configs,
site_config.templates.chrome_perf,
)
site_config.AddForBoards(
'telem-chromium-pfq-informational',
['amd64-generic'],
internal_board_configs,
site_config.templates.chromium_pfq_informational,
site_config.templates.telemetry,
site_config.templates.chrome_try,
)
_tot_chromium_pfq_informational_swarming_boards = frozenset([
'amd64-generic',
'daisy',
])
_tot_chromium_pfq_informational_board_configs = UpdateBoardConfigs(
external_board_configs,
_tot_chromium_pfq_informational_swarming_boards,
active_waterfall=waterfall.WATERFALL_SWARMING)
site_config.AddForBoards(
'tot-chromium-pfq-informational',
(boards_dict['all_full_boards'] & _chrome_boards),
_tot_chromium_pfq_informational_board_configs,
site_config.templates.chromium_pfq_informational,
site_config.templates.build_external_chrome,
internal=False,
manifest_repo_url=site_config.params['MANIFEST_URL'],
overlays=constants.PUBLIC_OVERLAYS,
)
site_config.ApplyForBoards(
'tot-chromium-pfq-informational',
['amd64-generic', 'daisy'],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 30m interval',
)
site_config.ApplyForBoards(
'tot-chrome-pfq-informational',
['caroline', 'eve', 'peach_pit', 'tricky', 'veyron_minnie',],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 30m interval',
)
_telemetry_boards = frozenset([
'amd64-generic',
'arm-generic',
'betty',
'betty-arcnext',
])
site_config.AddForBoards(
'telemetry',
_telemetry_boards,
internal_board_configs,
site_config.templates.telemetry,
)
site_config['amd64-generic-telemetry'].apply(
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='with 30m interval',
)
def ChromePfqBuilders(site_config, boards_dict, ge_build_config):
"""Create all Chrome PFQ build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
external_board_configs = CreateBoardConfigs(
site_config, boards_dict, ge_build_config)
internal_board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
_chrome_boards = frozenset(
board for board, config in internal_board_configs.iteritems()
if config.get('sync_chrome', True))
_chromium_pfq_important_boards = frozenset([
'arm-generic',
'daisy',
'veyron_jerry',
'amd64-generic',
])
master_config = site_config.Add(
'master-chromium-pfq',
site_config.templates.chromium_pfq,
boards=[],
master=True,
slave_configs=[],
binhost_test=True,
push_overlays=constants.BOTH_OVERLAYS,
afdo_update_ebuild=True,
chrome_sdk=False,
health_alert_recipients=['chromeos-infra-eng@grotations.appspotmail.com',
'chrome'],
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='triggered',
triggered_gitiles=[[
'https://chromium.googlesource.com/chromium/src',
['regexp:refs/tags/[^/]+']
]],
)
# Create important configs, then non-important configs.
master_config.AddSlaves(
site_config.AddForBoards(
'chromium-pfq',
_chromium_pfq_important_boards,
external_board_configs,
site_config.templates.chromium_pfq,
site_config.templates.build_external_chrome,
active_waterfall=waterfall.WATERFALL_SWARMING,
)
)
site_config.AddForBoards(
'chromium-pfq',
((boards_dict['all_full_boards'] & _chrome_boards) -
_chromium_pfq_important_boards),
external_board_configs,
site_config.templates.chromium_pfq,
site_config.templates.build_external_chrome,
)
_chrome_pfq_important_boards = frozenset([
'betty',
'betty-arcnext',
'caroline',
'caroline-arcnext',
'chell',
'coral',
'cyan',
'daisy_skate',
'eve',
'eve-arcnext',
'grunt',
'kevin-arcnext',
'nocturne',
'peach_pit',
'peppy',
'reef',
'terra',
'tricky',
'veyron_minnie',
'veyron_rialto',
])
_chrome_pfq_experimental_boards = frozenset([
'bob',
'hana',
'nyan_big',
'scarlet',
])
_chrome_pfq_skylab_boards = frozenset([
'reef',
])
_chrome_pfq_tryjob_boards = (
(boards_dict['all_release_boards'] & _chrome_boards) -
(_chrome_pfq_important_boards | _chrome_pfq_experimental_boards)
)
master_config.AddSlaves(
site_config.AddForBoards(
'chrome-pfq',
_chrome_pfq_important_boards - _chrome_pfq_skylab_boards,
internal_board_configs,
site_config.templates.chrome_pfq,
active_waterfall=waterfall.WATERFALL_SWARMING,
)
)
master_config.AddSlaves(
site_config.AddForBoards(
'chrome-pfq',
_chrome_pfq_experimental_boards - _chrome_pfq_skylab_boards,
internal_board_configs,
site_config.templates.chrome_pfq,
important=False,
active_waterfall=waterfall.WATERFALL_SWARMING,
)
)
master_config.AddSlaves(
site_config.AddForBoards(
'chrome-pfq',
_chrome_pfq_skylab_boards,
internal_board_configs,
site_config.templates.chrome_pfq,
enable_skylab_hw_tests=True,
active_waterfall=waterfall.WATERFALL_SWARMING,
)
)
# Define the result of the build configs for tryjob purposes.
site_config.AddForBoards(
'chrome-pfq',
_chrome_pfq_tryjob_boards,
internal_board_configs,
site_config.templates.chrome_pfq,
)
def FirmwareBuilders(site_config, boards_dict, ge_build_config):
"""Create all firmware build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
_firmware_boards = _arm_internal_release_boards | _x86_internal_release_boards
_x86_depthcharge_firmware_boards = frozenset([
'link',
])
# Add x86 and arm firmware configs.
for board in _firmware_boards:
site_config.Add(
'%s-%s' % (board, config_lib.CONFIG_TYPE_FIRMWARE),
site_config.templates.firmware,
board_configs[board],
site_config.templates.no_vmtest_builder,
)
for board in _x86_depthcharge_firmware_boards:
site_config.Add(
'%s-%s-%s' % (board, 'depthcharge', config_lib.CONFIG_TYPE_FIRMWARE),
site_config.templates.depthcharge_firmware,
board_configs[board],
site_config.templates.no_vmtest_builder,
)
site_config.Add(
'%s-%s-%s-%s' % (board, 'depthcharge', config_lib.CONFIG_TYPE_FULL,
config_lib.CONFIG_TYPE_FIRMWARE),
site_config.templates.depthcharge_full_firmware,
board_configs[board],
site_config.templates.no_vmtest_builder,
)
# Requires that you set boards, and workspace_branch.
site_config.AddTemplate(
'firmwarebranch',
site_config.templates.firmware,
build_type=constants.GENERIC_TYPE,
uprev=True,
overlays=constants.BOTH_OVERLAYS,
push_overlays=constants.BOTH_OVERLAYS,
builder_class_name='workspace_builders.FirmwareBranchBuilder',
build_timeout=6*60 * 60,
description='TOT builder to build a firmware branch.',
doc='https://goto.google.com/tot-for-firmware-branches',
)
# Defines "active", "branch", "boards" for firmwarebranch builds.
#
# Active:
# None: Do not schedule automatically.
ACTIVE = 'with 24h interval' # 1 day interval
INACTIVE = 'with 720h interval' # 30 day interval
firmware_branch_builders = [
(INACTIVE, 'firmware-falco_peppy-4389.B', ['falco', 'peppy']),
(INACTIVE, 'firmware-pit-4482.B', ['peach_pit', 'peach_pi']),
(INACTIVE, 'firmware-wolf-4389.24.B', ['wolf']),
(INACTIVE, 'firmware-leon-4389.61.B', ['leon']),
(INACTIVE, 'firmware-monroe-4921.B', ['monroe']),
(INACTIVE, 'firmware-panther-4920.24.B', ['panther']),
(INACTIVE, 'firmware-skate-3824.129.B', ['daisy_skate']),
(INACTIVE, 'firmware-zako-5219.B', ['zako']),
(INACTIVE, 'firmware-squawks-5216.152.B', ['squawks']),
(INACTIVE, 'firmware-nyan-5771.B', ['nyan_big', 'nyan_blaze']),
(INACTIVE, 'firmware-glimmer-5216.198.B', ['glimmer']),
(INACTIVE, 'firmware-clapper-5216.199.B', ['clapper']),
(INACTIVE, 'firmware-enguarde-5216.201.B', ['enguarde']),
(INACTIVE, 'firmware-zako-5219.17.B', ['zako']),
(INACTIVE, 'firmware-mccloud-5827.B', ['mccloud']),
(INACTIVE, 'firmware-tricky-5829.B', ['tricky']),
(INACTIVE, 'firmware-quawks-5216.204.B', ['quawks']),
(INACTIVE, 'firmware-expresso-5216.223.B', ['expresso']),
(INACTIVE, 'firmware-kip-5216.227.B', ['kip']),
(INACTIVE, 'firmware-swanky-5216.238.B', ['swanky']),
(INACTIVE, 'firmware-gnawty-5216.239.B', ['gnawty']),
(INACTIVE, 'firmware-winky-5216.265.B', ['winky']),
(INACTIVE, 'firmware-kitty-5771.61.B', ['nyan_kitty']),
(INACTIVE, 'firmware-samus-6300.B', ['samus']),
(INACTIVE, 'firmware-auron-6301.B', ['jecht']),
(INACTIVE, 'firmware-candy-5216.310.B', ['candy']),
(INACTIVE, 'firmware-veyron-6588.B', [
'veyron_jerry', 'veyron_mighty',
'veyron_speedy', 'veyron_jaq',
'veyron_minnie',
'veyron_mickey', 'veyron_rialto', 'veyron_tiger',
'veyron_fievel']),
(INACTIVE, 'firmware-paine-6301.58.B', ['auron_paine']),
(INACTIVE, 'firmware-yuna-6301.59.B', ['auron_yuna']),
(INACTIVE, 'firmware-guado-6301.108.B', ['guado']),
(INACTIVE, 'firmware-tidus-6301.109.B', ['tidus']),
(INACTIVE, 'firmware-rikku-6301.110.B', ['rikku']),
(INACTIVE, 'firmware-banjo-5216.334.B', ['banjo']),
(INACTIVE, 'firmware-lulu-6301.136.B', ['lulu']),
(INACTIVE, 'firmware-orco-5216.362.B', ['orco']),
(INACTIVE, 'firmware-gandof-6301.155.B', ['gandof']),
(INACTIVE, 'firmware-ninja-5216.383.B', ['ninja']),
(INACTIVE, 'firmware-sumo-5216.382.B', ['sumo']),
(INACTIVE, 'firmware-strago-7287.B', [
'wizpig', 'setzer', 'banon', 'kefka', 'relm']),
(INACTIVE, 'firmware-heli-5216.392.B', ['heli']),
(INACTIVE, 'firmware-cyan-7287.57.B', ['cyan']),
(INACTIVE, 'firmware-celes-7287.92.B', ['celes']),
(INACTIVE, 'firmware-ultima-7287.131.B', ['ultima']),
(INACTIVE, 'firmware-reks-7287.133.B', ['reks']),
(INACTIVE, 'firmware-buddy-6301.202.B', ['buddy']),
(INACTIVE, 'firmware-glados-7820.B', [
'glados', 'chell', 'lars',
'sentry', 'cave', 'asuka', 'caroline']),
(INACTIVE, 'firmware-terra-7287.154.B', ['terra']),
(INACTIVE, 'firmware-edgar-7287.167.B', ['edgar']),
(INACTIVE, 'firmware-gale-8281.B', ['gale']),
(INACTIVE, 'firmware-oak-8438.B', ['oak', 'elm', 'hana']),
(INACTIVE, 'firmware-gru-8785.B', ['gru', 'kevin', 'bob']),
(INACTIVE, 'firmware-reef-9042.B', ['reef', 'pyro', 'sand', 'snappy']),
(INACTIVE, 'firmware-cr50-9308.B', ['reef']),
(INACTIVE, 'firmware-eve-9584.B', ['eve']),
(INACTIVE, 'firmware-eve-campfire-9584.131.B', ['eve']),
(INACTIVE, 'firmware-cr50-9308.24.B', ['reef']),
(INACTIVE, 'firmware-cr50-release-9308.25.B', ['reef']),
(INACTIVE, 'firmware-cr50-mp-release-9308.87.B', ['reef']),
(INACTIVE, 'firmware-coral-10068.B', ['coral']),
(INACTIVE, 'firmware-servo-9040.B', ['falco']),
(INACTIVE, 'firmware-servo-11011.B', ['oak']),
(INACTIVE, 'firmware-fizz-10139.B', ['fizz']),
(INACTIVE, 'firmware-fizz-10139.94.B', ['fizz']),
(INACTIVE, 'firmware-scarlet-10388.B', ['scarlet']),
(INACTIVE, 'firmware-poppy-10431.B', ['poppy', 'soraka', 'nautilus']),
(ACTIVE, 'firmware-nami-10775.B', ['nami']),
# (ACTIVE, 'firmware-nocturne-10984.B', ['nocturne']),
# (ACTIVE, 'firmware-grunt-11031.B', ['grunt']),
]
for active, branch, boards in firmware_branch_builders:
schedule = {}
if active:
schedule = {
'active_waterfall': waterfall.WATERFALL_SWARMING,
'schedule': active,
}
site_config.Add(
'%s-firmwarebranch' % branch,
site_config.templates.firmwarebranch,
boards=boards,
workspace_branch=branch,
**schedule
)
def FactoryBuilders(site_config, boards_dict, ge_build_config):
"""Create all factory build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
_factory_boards = _arm_internal_release_boards | _x86_internal_release_boards
_factory_boards -= _nofactory_boards
for board in _factory_boards:
site_config.Add(
'%s-%s' % (board, config_lib.CONFIG_TYPE_FACTORY),
site_config.templates.factory,
board_configs[board],
site_config.templates.no_vmtest_builder,
)
def ReleaseBuilders(site_config, boards_dict, ge_build_config):
"""Create all release builders.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config)
unified_board_names = set([b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]
for b in unified_builds])
### Master release config.
master_config = site_config.Add(
'master-release',
site_config.templates.release,
boards=[],
master=True,
slave_configs=[],
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
branch_util_test=True,
active_waterfall=waterfall.WATERFALL_SWARMING,
# Because PST is 8 hours from UTC, these times are the same in both. But
# daylight savings time is NOT adjusted for
schedule=' 0 2,10,18 * * *',
)
### Release configs.
# These currently do not support unified build boards.
_critical_for_chrome_boards = frozenset([
'daisy',
'eve',
])
_release_experimental_boards = frozenset([
'nyan_blaze',
])
_release_enable_skylab_hwtest = frozenset([
'nyan_blaze',
'reef',
])
def _get_skylab_settings(board_name):
"""Get skylab settings for release builder.
Args:
board_name: A string board name.
Returns:
enable_skylab_hw_tests: a boolean var to indicate whether this
board is moved to skylab.
"""
if board_name in _release_enable_skylab_hwtest:
return True
return False
site_config.AddForBoards(
config_lib.CONFIG_TYPE_RELEASE,
_critical_for_chrome_boards,
board_configs,
site_config.templates.release,
critical_for_chrome=True,
)
builder_to_boards_dict = config_lib.GroupBoardsByBuilder(
ge_build_config[config_lib.CONFIG_TEMPLATE_BOARDS])
_all_release_builder_boards = builder_to_boards_dict[
config_lib.CONFIG_TEMPLATE_RELEASE]
site_config.AddForBoards(
config_lib.CONFIG_TYPE_RELEASE,
(((boards_dict['all_release_boards'] | _all_release_builder_boards)
- _critical_for_chrome_boards) - unified_board_names),
board_configs,
site_config.templates.release,
)
hw_test_list = HWTestList(ge_build_config)
for unibuild in config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config):
models = []
for model in unibuild[config_lib.CONFIG_TEMPLATE_MODELS]:
name = model[config_lib.CONFIG_TEMPLATE_MODEL_NAME]
lab_board_name = model[config_lib.CONFIG_TEMPLATE_MODEL_BOARD_NAME]
if config_lib.CONFIG_TEMPLATE_MODEL_TEST_SUITES in model:
test_suites = model[config_lib.CONFIG_TEMPLATE_MODEL_TEST_SUITES]
if 'bvt-arc' in test_suites:
# TODO(crbug.com/814793)
# We're tying these test suites to bvt-arc because it's not worth
# plumbing this all the way through the GE UI since that architecture
# was never fully implemented and we shouldn't have tied to it in
# the first place.
# Once test planning is properly implemented, this will fall out.
test_suites.append('arc-cts-qual')
test_suites.append('arc-gts-qual')
models.append(config_lib.ModelTestConfig(
name,
lab_board_name,
test_suites))
else:
no_model_test_suites = []
models.append(config_lib.ModelTestConfig(
name, lab_board_name, no_model_test_suites))
reference_board_name = unibuild[
config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]
pool = constants.HWTEST_MACH_POOL
config_name = '%s-release' % reference_board_name
# Move unibuild to skylab.
important = not unibuild[config_lib.CONFIG_TEMPLATE_EXPERIMENTAL]
if reference_board_name in _release_experimental_boards:
important = False
enable_skylab_hw_tests = _get_skylab_settings(reference_board_name)
site_config.AddForBoards(
config_lib.CONFIG_TYPE_RELEASE,
[reference_board_name],
board_configs,
site_config.templates.release,
models=models,
important=important,
enable_skylab_hw_tests=enable_skylab_hw_tests,
active_waterfall=waterfall.WATERFALL_SWARMING,
hw_tests=(hw_test_list.SharedPoolCanary(pool=pool) +
hw_test_list.CtsGtsQualTests()),
)
master_config.AddSlave(site_config[config_name])
def GetReleaseConfigName(board):
"""Convert a board name into a release config name."""
return '%s-release' % board
def GetConfigName(builder, board):
"""Convert a board name into a config name."""
if builder == config_lib.CONFIG_TEMPLATE_RELEASE:
return GetReleaseConfigName(board)
else:
# Currently just support RELEASE builders
raise Exception('Do not support other builders.')
def _GetConfigValues(board):
"""Get and return config values from template and user definitions."""
important = not board[config_lib.CONFIG_TEMPLATE_EXPERIMENTAL]
if board['name'] in _release_experimental_boards:
important = False
enable_skylab_hw_tests = _get_skylab_settings(board['name'])
# Move non-unibuild to skylab.
config_values = {
'important': important,
'active_waterfall': waterfall.WATERFALL_SWARMING,
'enable_skylab_hw_tests': enable_skylab_hw_tests,
}
return config_values
def _AdjustUngroupedReleaseConfigs(builder_ungrouped_dict):
"""Adjust for ungrouped release boards"""
for builder in builder_ungrouped_dict:
for board in builder_ungrouped_dict[builder]:
config_name = GetConfigName(builder,
board[config_lib.CONFIG_TEMPLATE_NAME])
site_config[config_name].apply(
_GetConfigValues(board),
)
master_config.AddSlave(site_config[config_name])
def _AdjustGroupedReleaseConfigs(builder_group_dict):
"""Adjust leader and follower configs for grouped boards"""
for builder in builder_group_dict:
for group in builder_group_dict[builder]:
board_group = builder_group_dict[builder][group]
# Leaders are built on baremetal builders and run all tests needed by
# the related boards.
for board in board_group.leader_boards:
config_name = GetConfigName(builder,
board[config_lib.CONFIG_TEMPLATE_NAME])
site_config[config_name].apply(
_GetConfigValues(board),
)
master_config.AddSlave(site_config[config_name])
# Followers are built on GCE instances, and turn off testing that breaks
# on GCE. The missing tests run on the leader board.
for board in board_group.follower_boards:
config_name = GetConfigName(builder,
board[config_lib.CONFIG_TEMPLATE_NAME])
site_config[config_name].apply(
_GetConfigValues(board),
buildslave_type=constants.GCE_BEEFY_BUILD_SLAVE_TYPE,
chrome_sdk_build_chrome=False,
vm_tests=[],
)
master_config.AddSlave(site_config[config_name])
def _AdjustReleaseConfigs():
"""Adjust ungrouped and grouped release configs"""
(builder_group_dict, builder_ungrouped_dict) = (
config_lib.GroupBoardsByBuilderAndBoardGroup(
ge_build_config[config_lib.CONFIG_TEMPLATE_BOARDS]))
_AdjustUngroupedReleaseConfigs(builder_ungrouped_dict)
_AdjustGroupedReleaseConfigs(builder_group_dict)
for board in _moblab_boards:
config_name = GetReleaseConfigName(board)
if config_name not in site_config:
continue
# If the board is in _moblab_boards, use moblab_release template
site_config[config_name].apply(
site_config.templates.moblab_release,
board_configs[board],
)
_AdjustReleaseConfigs()
def PayloadBuilders(site_config, boards_dict):
"""Create <board>-payloads configs for all payload generating boards.
We create a config named 'board-payloads' for every board which has a
config with 'paygen' True. The idea is that we have a build that generates
payloads, we need to have a tryjob to re-attempt them on failure.
"""
for board in boards_dict['all_release_boards']:
if site_config['%s-release' % board].paygen:
site_config.Add(
'%s-payloads' % board,
site_config.templates.payloads,
boards=[board],
luci_builder=config_lib.LUCI_BUILDER_TRY,
)
def InsertHwTestsOverrideDefaults(build):
"""Insert default hw_tests values for a given build.
Also updates child builds.
Args:
build: BuildConfig instance to modify in place.
"""
for child in build['child_configs']:
InsertHwTestsOverrideDefaults(child)
if build['hw_tests_override'] is not None:
# Explicitly set, no need to insert defaults.
return
if build['hw_tests']:
# Copy over base tests.
build['hw_tests_override'] = [copy.copy(x) for x in build['hw_tests']]
# Adjust for manual test environment.
for hw_config in build['hw_tests_override']:
hw_config.pool = constants.HWTEST_TRYBOT_POOL
hw_config.file_bugs = False
hw_config.priority = constants.HWTEST_DEFAULT_PRIORITY
def ApplyCustomOverrides(site_config, ge_build_config):
"""Method with to override specific flags for specific builders.
Generally try really hard to avoid putting anything here that isn't
a really special case for a single specific builder. This is performed
after every other bit of processing, so it always has the final say.
Args:
site_config: config_lib.SiteConfig containing builds to have their
waterfall values updated.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
hw_test_list = HWTestList(ge_build_config)
overwritten_configs = {
'amd64-generic-chromium-pfq': {
'useflags': [],
},
'lakitu-pre-cq':
site_config.templates.lakitu_test_customizations,
'lakitu-gpu-pre-cq':
site_config.templates.lakitu_test_customizations,
'lakitu-nc-pre-cq':
site_config.templates.lakitu_nc_customizations,
'lakitu-st-pre-cq':
site_config.templates.lakitu_test_customizations,
'lakitu_next-pre-cq':
site_config.templates.lakitu_test_customizations,
'whirlwind-release': {
'dev_installer_prebuilts':True,
},
'gale-release': {
'dev_installer_prebuilts':True,
},
'lakitu-release': config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations,
site_config.templates.lakitu_notification_emails,
sign_types=['base'],
),
# This is the full build of open-source overlay.
'lakitu-full': config_lib.BuildConfig().apply(
site_config.templates.lakitu_notification_emails,
# logging_CrashSender is expected to fail for lakitu-full.
# See b/111567339 for more details.
useflags=append_useflags(['-tests_logging_CrashSender']),
),
'lakitu-gpu-release': config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations,
site_config.templates.lakitu_notification_emails,
sign_types=['base'],
paygen=False,
),
'lakitu-nc-release': config_lib.BuildConfig().apply(
site_config.templates.lakitu_nc_customizations,
site_config.templates.lakitu_notification_emails,
signer_tests=False,
paygen=False,
),
'lakitu-st-release': config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations,
site_config.templates.lakitu_notification_emails,
sign_types=['base'],
paygen=False,
),
'lakitu_next-release': config_lib.BuildConfig().apply(
site_config.templates.lakitu_test_customizations,
site_config.templates.lakitu_notification_emails,
signer_tests=False,
),
'guado_labstation-release': {
'hw_tests': [],
'hwqual':False,
'image_test':False,
'images':['base', 'test'],
'signer_tests':False,
'paygen':False,
'vm_tests':[],
},
'peach_pit-release': {
'useflags': append_useflags(['cfi']),
},
'kevin-release': {
'useflags': append_useflags(['cfi']),
},
'chell-chrome-pfq': {
'afdo_generate': True,
'archive_build_debug': True,
# Disable hugepages before collecting AFDO profile.
# Disable debug fission before collecting AFDO profile.
# Disable thinlto before collecting AFDO profile.
# Disable cfi before collecting AFDO profile.
'useflags': append_useflags(['-transparent_hugepage',
'-debug_fission',
'-thinlto',
'-cfi']),
'hw_tests': [hw_test_list.AFDORecordTest()]
},
'cyan-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolAndroidPFQ(),
},
'caroline-arcnext-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolAndroidPFQ(),
},
'grunt-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolAndroidPFQ(),
},
'kevin-arcnext-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolAndroidPFQ(),
},
'peppy-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolPFQ(),
},
'reef-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolAndroidPFQ(),
},
'veyron_minnie-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolAndroidPFQ(),
},
'peach_pit-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolPFQ(),
},
'tricky-chrome-pfq': {
'hw_tests': hw_test_list.SharedPoolPFQ(),
},
# Currently factory and firmware branches will be created after DVT stage
# therefore we need signed factory shim or accessory_rwsig firmware from
# ToT temporarily.
#
# After factory and firmware branches are created, the configuation of
# this project should be removed.
# --- start from here ---
'cheza-release': {
'sign_types': ['recovery', 'factory'],
},
'poppy-release': {
'sign_types': ['recovery', 'accessory_rwsig', 'factory'],
},
'nautilus-release': {
'sign_types': ['recovery', 'accessory_rwsig', 'factory'],
},
'nami-release': {
'sign_types': ['recovery', 'factory'],
},
'octopus-release': {
'sign_types': ['recovery', 'factory'],
},
# --- end from here ---
# Disabled pending resolution of crbug.com/894820.
#'amd64-generic-paladin': site_config.templates.tast_vm_paladin_tests,
#'betty-paladin': site_config.templates.tast_vm_paladin_tests,
'betty-release': site_config.templates.tast_vm_canary_tests,
# Enable the new tcmalloc version on certain boards.
'elm-release': {
'useflags': append_useflags(['new_tcmalloc']),
},
'kip-release': {
'useflags': append_useflags(['new_tcmalloc']),
},
'veyron_minnie-release': {
'useflags': append_useflags(['new_tcmalloc']),
},
}
for config_name, overrides in overwritten_configs.iteritems():
# TODO: Turn this assert into a unittest.
# config = site_config[config_name]
# for k, v in overrides.iteritems():
# assert config[k] != v, ('Unnecessary override: %s: %s' %
# (config_name, k))
site_config[config_name].apply(**overrides)
def SpecialtyBuilders(site_config, boards_dict, ge_build_config):
"""Add a variety of specialized builders or tryjobs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
site_config.AddWithoutTemplate(
'success-build',
site_config.templates.external,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
luci_builder=config_lib.LUCI_BUILDER_TRY,
builder_class_name='test_builders.SucessBuilder',
description='Builder always passes as quickly as possible.',
active_waterfall=waterfall.WATERFALL_SWARMING,
)
# Used by cbuildbot/stages/sync_stages_unittest
site_config.AddWithoutTemplate(
'sync-test-cbuildbot',
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
luci_builder=config_lib.LUCI_BUILDER_PROD,
builder_class_name='test_builders.SucessBuilder',
description='Used by sync_stages_unittest.',
)
site_config.AddWithoutTemplate(
'fail-build',
site_config.templates.external,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
luci_builder=config_lib.LUCI_BUILDER_TRY,
builder_class_name='test_builders.FailBuilder',
description='Builder always fails as quickly as possible.',
active_waterfall=waterfall.WATERFALL_SWARMING,
)
site_config.AddWithoutTemplate(
'chromiumos-sdk',
site_config.templates.full,
site_config.templates.no_hwtest_builder,
# The amd64-host has to be last as that is when the toolchains
# are bundled up for inclusion in the sdk.
boards=[
'arm-generic', 'amd64-generic'
],
display_label=config_lib.DISPLAY_LABEL_UTILITY,
build_type=constants.CHROOT_BUILDER_TYPE,
active_waterfall=waterfall.WATERFALL_SWARMING,
buildslave_type=constants.GCE_BEEFY_BUILD_SLAVE_TYPE,
builder_class_name='sdk_builders.ChrootSdkBuilder',
use_sdk=False,
prebuilts=constants.PUBLIC,
build_timeout=18 * 60 * 60,
description='Build the SDK and all the cross-compilers',
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Continuous',
schedule='with 30m interval',
)
site_config.AddWithoutTemplate(
'config-updater',
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
display_label=config_lib.DISPLAY_LABEL_UTILITY,
description=('Build Config Updater reads updated GE config files from'
' GS, and commits them to chromite after running tests.'),
build_type=constants.GENERIC_TYPE,
boards=[],
builder_class_name='config_builders.UpdateConfigBuilder',
active_waterfall=constants.WATERFALL_SWARMING,
binhost_test=True,
schedule='@hourly',
)
site_config.AddWithoutTemplate(
'luci-scheduler-updater',
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
display_label=config_lib.DISPLAY_LABEL_UTILITY,
description=('Deploy changes to luci_scheduler.cfg.'),
build_type=constants.GENERIC_TYPE,
boards=[],
builder_class_name='config_builders.LuciSchedulerBuilder',
active_waterfall=constants.WATERFALL_SWARMING,
binhost_test=True,
schedule='triggered',
triggered_gitiles=[[
'https://chromium.googlesource.com/chromiumos/chromite',
['refs/heads/master']
]],
)
site_config.AddWithoutTemplate(
constants.BRANCH_UTIL_CONFIG,
site_config.templates.paladin,
site_config.templates.internal_paladin,
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_UTILITY,
boards=[],
# Disable postsync_patch to prevent conflicting patches from being applied
# - e.g., patches from 'master' branch being applied to a branch.
postsync_patch=False,
# Disable postsync_reexec to continue running the 'master' branch chromite
# for all stages, rather than the chromite in the branch buildroot.
postsync_reexec=False,
# Need to reset the paladin build_type we inherited.
build_type=None,
builder_class_name='release_builders.CreateBranchBuilder',
description='Used for creating/deleting branches (TPMs only)',
# This very weird tryjob is only run locally. It should never upload
# build artifacts.
archive=False,
gs_path=os.path.join(constants.TRASH_BUCKET, 'branch-util-noise/')
)
site_config.AddWithoutTemplate(
'betty-vmtest-informational',
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
display_label=config_lib.DISPLAY_LABEL_INFORMATIONAL,
description='VMTest Informational Builder for running long run tests.',
build_type=constants.GENERIC_TYPE,
boards=['betty'],
builder_class_name='test_builders.VMInformationalBuilder',
vm_tests=getInfoVMTest(),
vm_tests_override=getInfoVMTest(),
vm_test_report_to_dashboards=True,
active_waterfall=waterfall.WATERFALL_SWARMING,
# 3 PM UTC is 7 AM PST (no daylight savings).
schedule='0 15 * * *',
)
# Create our unittest stress build configs (used for tryjobs only)
site_config.AddForBoards(
'unittest-stress',
boards_dict['all_boards'],
board_configs,
site_config.templates.unittest_stress,
luci_builder=config_lib.LUCI_BUILDER_TRY,
)
site_config.AddGroup(
'test-ap-group',
site_config.Add('panther-test-ap',
site_config.templates.test_ap,
boards=['panther']),
site_config.Add('whirlwind-test-ap',
site_config.templates.test_ap,
boards=['whirlwind']),
site_config.Add('gale-test-ap',
site_config.templates.test_ap,
boards=['gale']),
description='Create images used to power access points in WiFi lab.',
)
# *-pre-flight-branch builders are in chromeos_release waterfall.
site_config.Add(
'samus-chrome-pre-flight-branch',
site_config.templates.pre_flight_branch,
display_label=config_lib.DISPLAY_LABEL_CHROME_PFQ,
boards=['samus'],
afdo_generate=True,
afdo_update_ebuild=True,
sync_chrome=True,
chrome_rev=constants.CHROME_REV_STICKY,
hw_tests=[hw_test_list.AFDORecordTest()],
useflags=append_useflags(['-transparent_hugepage',
'-debug_fission',
'-thinlto',
'-cfi']),
prebuilts=constants.PRIVATE,
archive_build_debug=True,
)
site_config.Add(
'reef-android-nyc-pre-flight-branch',
site_config.templates.pre_flight_branch,
display_label=config_lib.DISPLAY_LABEL_NYC_ANDROID_PFQ,
boards=['reef'],
sync_chrome=True,
android_rev=constants.ANDROID_REV_LATEST,
android_package='android-container-nyc',
android_import_branch=constants.ANDROID_NYC_BUILD_BRANCH,
prebuilts=False,
)
site_config.Add(
'grunt-android-pi-pre-flight-branch',
site_config.templates.pre_flight_branch,
display_label=config_lib.DISPLAY_LABEL_PI_ANDROID_PFQ,
boards=['grunt'],
sync_chrome=True,
android_rev=constants.ANDROID_REV_LATEST,
android_package='android-container-pi',
android_import_branch=constants.ANDROID_PI_BUILD_BRANCH,
prebuilts=False,
)
site_config.AddWithoutTemplate(
'chromeos-infra-go',
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
display_label=config_lib.DISPLAY_LABEL_UTILITY,
build_type=constants.GENERIC_TYPE,
buildslave_type=constants.GCE_BEEFY_BUILD_SLAVE_TYPE,
builder_class_name='infra_builders.InfraGoBuilder',
use_sdk=True,
prebuilts=constants.PUBLIC,
build_timeout=60 * 60,
description='Build Chromium OS infra Go binaries',
doc='https://goto.google.com/cros-infra-go-packaging',
active_waterfall=waterfall.WATERFALL_SWARMING,
schedule='triggered',
triggered_gitiles=[[
'https://chromium.googlesource.com/chromiumos/infra/lucifer',
['refs/heads/master']
], [
'https://chromium.googlesource.com/chromiumos/infra/skylab_inventory',
['refs/heads/master'],
]],
)
def EnsureVmTestsOnVmTestBoards(site_config, boards_dict, _gs_build_config):
"""Make sure VMTests are only enabled on boards that support them.
Args:
site_config: config_lib.SiteConfig containing builds to have their
waterfall values updated.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
for c in site_config.itervalues():
if set(c['boards']).intersection(set(boards_dict['no_vmtest_boards'])):
c.apply(site_config.templates.no_vmtest_builder)
if c.child_configs:
for cc in c.child_configs:
cc.apply(site_config.templates.no_vmtest_builder)
def EnsureVmTestsOnBaremetal(site_config, _gs_build_config):
"""Make sure VMTests have a builder than can run them.
Args:
site_config: config_lib.SiteConfig containing builds to have their
waterfall values updated.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
for c in site_config.itervalues():
# We can run vmtests on GCE because we are whitelisted for GCE L2 VM support
# and are migrating from baremetal to GCE.
if c.vm_tests or c.moblab_vm_tests:
# Special case betty-incremental which we want on gce, crbug.com/795976
if c['name'] == 'betty-incremental':
continue
c['buildslave_type'] = constants.BAREMETAL_BUILD_SLAVE_TYPE
def TryjobMirrors(site_config):
"""Create tryjob specialized variants of every build config.
This creates a new 'tryjob' config for every existing config, unless the
existing config is already a tryjob config.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
"""
# Tryjobs which are unsafe to talk to Prod CIDB.
cidb_unsafe = frozenset(['pre-cq-launcher'])
tryjob_configs = {}
for build_name, config in site_config.iteritems():
# Don't mirror builds that are already tryjob safe.
if config_lib.isTryjobConfig(config):
config.apply(hw_tests_override=None, vm_tests_override=None)
continue
tryjob_name = build_name + '-tryjob'
# Don't overwrite mirrored versions that were explicitly created earlier.
if tryjob_name in site_config:
continue
tryjob_config = copy.deepcopy(config)
tryjob_config.apply(
name=tryjob_name,
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
luci_builder=config_lib.LUCI_BUILDER_TRY,
# Generally make tryjobs safer.
chroot_replace=True,
debug=True,
paygen=False,
push_image=False,
# Force uprev. This is so patched in changes are always built.
uprev=True,
active_waterfall=waterfall.WATERFALL_SWARMING,
gs_path=config_lib.GS_PATH_DEFAULT,
schedule=None,
suite_scheduling=False,
triggered_gitiles=None,
important=True,
build_affinity=False,
)
# Force uprev. This is so patched in changes are always built.
if tryjob_config.internal:
tryjob_config.apply(overlays=constants.BOTH_OVERLAYS)
# In trybots, we want to always run VM tests and all unit tests, so that
# developers will get better testing for their changes.
if tryjob_config.hw_tests_override is not None:
tryjob_config.apply(hw_tests=tryjob_config.hw_tests_override,
hw_tests_override=None)
if tryjob_config.vm_tests_override is not None:
tryjob_config.apply(vm_tests=tryjob_config.vm_tests_override,
vm_tests_override=None)
if build_name in cidb_unsafe or tryjob_config.master:
tryjob_config.apply(debug_cidb=True)
# Save off the new config so we can insert into site_config.
tryjob_configs[tryjob_name] = tryjob_config
for tryjob_name, tryjob_config in tryjob_configs.iteritems():
site_config[tryjob_name] = tryjob_config
def BranchScheduleConfig():
"""Create a list of configs to schedule for branch builds.
This function returns a list of build configs with just enough
information to correctly schedule builds on branches. This function
is only used by scripts/gen_luci_scheduler.
After making changes to this function, they must be deployed to take
effect. See gen_luci_scheduler --help for details.
Returns:
List of config_lib.BuildConfig instances.
"""
release_label = config_lib.DISPLAY_LABEL_RELEASE
#
# Define each branched schedule with:
# branch_name: Name of the branch to build as a string.
# config_name: Name of the buidl config already present on the branch.
# label: Display label for UI use. Usually release, factory, firmware.
# schedule: When to do the build. Can take several formats.
# 'triggered' for manual builds.
# Cron style in UTC timezone: '0 15 * * *'
# 'with 30d interval' to run X time after previous build.
#
# https://github.com/luci/luci-go/blob/master/scheduler/
# appengine/messages/config.proto
#
# When updating this be sure to run
# `config/chromeos_config_unittest --update`
# or the change will fail chromite unittests.
branch_builds = (
('release-R71-11151.B', 'master-release',
release_label, '0 7 * * *', None),
('release-R71-11151.B', 'reef-android-nyc-pre-flight-branch',
release_label, '0 3,7,11,15,19,23 * * *', None),
('release-R71-11151.B', 'grunt-android-pi-pre-flight-branch',
release_label, '0 3,7,11,15,19,23 * * *', None),
('release-R71-11151.B', 'samus-chrome-pre-flight-branch',
release_label, '0 3,7,11,15,19,23 * * *', None),
('release-R70-11021.B', 'master-release',
release_label, '0 5 * * *', None),
('release-R70-11021.B', 'reef-android-nyc-pre-flight-branch',
release_label, '0 1,5,9,13,17,21 * * *', None),
('release-R70-11021.B', 'grunt-android-pi-pre-flight-branch',
release_label, '0 1,5,9,13,17,21 * * *', None),
('release-R70-11021.B', 'samus-chrome-pre-flight-branch',
release_label, '0 1,5,9,13,17,21 * * *', None),
('release-R69-10895.B', 'master-release',
release_label, 'triggered', None),
('release-R69-10895.B', 'reef-android-nyc-pre-flight-branch',
release_label, '0 2,6,10,14,18,22 * * *', None),
('release-R69-10895.B', 'samus-chrome-pre-flight-branch',
release_label, '0 2,6,10,14,18,22 * * *', None),
)
default_config = config_lib.GetConfig().GetDefault()
result = []
for branch, config_name, label, schedule, trigger in branch_builds:
result.append(default_config.derive(
name=config_name,
display_label=label,
luci_builder=config_lib.LUCI_BUILDER_PROD,
schedule_branch=branch,
schedule=schedule,
triggered_gitiles=trigger,
))
return result
@factory.CachedFunctionCall
def GetConfig():
"""Create the Site configuration for all ChromeOS builds.
Returns:
A config_lib.SiteConfig.
"""
site_params = config_lib.DefaultSiteParameters()
defaults = DefaultSettings(site_params)
ge_build_config = config_lib.LoadGEBuildConfigFromFile()
# site_config with no templates or build configurations.
site_config = config_lib.SiteConfig(defaults=defaults,
site_params=site_params)
boards_dict = GetBoardTypeToBoardsDict(ge_build_config)
GeneralTemplates(site_config, ge_build_config)
ToolchainBuilders(site_config, boards_dict, ge_build_config)
ReleaseBuilders(site_config, boards_dict, ge_build_config)
PayloadBuilders(site_config, boards_dict)
SpecialtyBuilders(site_config, boards_dict, ge_build_config)
PreCqBuilders(site_config, boards_dict, ge_build_config)
CqBuilders(site_config, boards_dict, ge_build_config)
IncrementalBuilders(site_config, boards_dict, ge_build_config)
ReleaseAfdoBuilders(site_config, boards_dict, ge_build_config)
InformationalBuilders(site_config, boards_dict, ge_build_config)
FirmwareBuilders(site_config, boards_dict, ge_build_config)
FactoryBuilders(site_config, boards_dict, ge_build_config)
AndroidPfqBuilders(site_config, boards_dict, ge_build_config)
ChromePfqBuilders(site_config, boards_dict, ge_build_config)
FullBuilders(site_config, boards_dict, ge_build_config)
# Insert default HwTests for tryjobs.
for build in site_config.itervalues():
InsertHwTestsOverrideDefaults(build)
ApplyCustomOverrides(site_config, ge_build_config)
EnsureVmTestsOnVmTestBoards(site_config, boards_dict, ge_build_config)
EnsureVmTestsOnBaremetal(site_config, ge_build_config)
TryjobMirrors(site_config)
return site_config