Google Git
Sign in
cos / mirrors / cros / chromiumos / chromite / refs/heads/stabilize-kernelnext-broadwell-12434.B / . / config / chromeos_config.py
blob: f70bd2da98cfe13ed05bfbe07a3015b53846a986 [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Configuration options for various cbuildbot builders."""
from __future__ import print_function
import copy
import re
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import factory
from chromite.config import chromeos_config_boards as chromeos_boards
from chromite.config import chromeos_test_config as chromeos_test
# TODO(yshaul): Restrict the import when we're done splitting
from chromite.config.chromeos_test_config import HWTestList
from chromite.config.chromeos_test_config import TRADITIONAL_VM_TESTS_SUPPORTED
from chromite.config.chromeos_test_config import getInfoVMTest
def remove_images(unsupported_images):
"""Remove unsupported images when applying changes to a BuildConfig.
Used similarly to append_useflags.
Args:
unsupported_images: A list of image names that should not be present
in the final build config.
Returns:
A callable suitable for use with BuildConfig.apply.
"""
unsupported = set(unsupported_images)
def handler(old_images):
if not old_images:
old_images = []
return [i for i in old_images if i not in unsupported]
return handler
def GetBoardTypeToBoardsDict(ge_build_config):
"""Get board type to board names dict.
Args:
ge_build_config: Dictionary containing the decoded GE configuration file.
Returns:
A dict mapping board types to board name collections.
The dict contains board types including distinct_board_sets,
all_release_boards, all_full_boards, all_boards, internal_boards,
and no_vmtest_boards.
"""
ge_arch_board_dict = config_lib.GetArchBoardDict(ge_build_config)
boards_dict = {}
arm_internal_release_boards = (
chromeos_boards.arm_internal_release_boards |
ge_arch_board_dict.get(config_lib.CONFIG_ARM_INTERNAL, set())
)
arm_external_boards = (
chromeos_boards.arm_external_boards |
ge_arch_board_dict.get(config_lib.CONFIG_ARM_EXTERNAL, set())
)
x86_internal_release_boards = (
chromeos_boards.x86_internal_release_boards |
ge_arch_board_dict.get(config_lib.CONFIG_X86_INTERNAL, set())
)
x86_external_boards = (
chromeos_boards.x86_external_boards |
ge_arch_board_dict.get(config_lib.CONFIG_X86_EXTERNAL, set())
)
# Every board should be in only 1 of the above sets.
boards_dict['distinct_board_sets'] = [
arm_internal_release_boards,
arm_external_boards,
x86_internal_release_boards,
x86_external_boards,
]
arm_full_boards = (
arm_internal_release_boards |
arm_external_boards)
x86_full_boards = (
x86_internal_release_boards |
x86_external_boards)
arm_boards = arm_full_boards
x86_boards = x86_full_boards
boards_dict['all_release_boards'] = (
chromeos_boards.arm_internal_release_boards |
chromeos_boards.x86_internal_release_boards
)
boards_dict['all_full_boards'] = (
arm_full_boards |
x86_full_boards
)
all_boards = x86_boards | arm_boards
boards_dict['all_boards'] = (
all_boards
)
boards_dict['internal_boards'] = boards_dict['all_release_boards']
# This set controls the final vmtest override. It allows us to specify
# vm_tests for each class of builders, but only execute on vmtest_boards.
boards_dict['no_vmtest_boards'] = (
all_boards - chromeos_test.vmtest_boards
)
boards_dict['generic_kernel_boards'] = frozenset([
'amd64-generic',
'arm-generic',
'arm64-generic'
])
return boards_dict
def DefaultSettings():
"""Create the default build config values for this site.
Returns:
dict: of default config_lib.BuildConfig values to use for this site.
"""
# Site specific adjustments for default BuildConfig values.
defaults = config_lib.DefaultSettings()
# Git repository URL for our manifests.
# https://chromium.googlesource.com/chromiumos/manifest
# https://chrome-internal.googlesource.com/chromeos/manifest-internal
defaults['manifest_repo_url'] = config_lib.GetSiteParams().MANIFEST_URL
return defaults
def GeneralTemplates(site_config):
"""Defines templates that are shared between categories of builders.
Args:
site_config: A SiteConfig object to add the templates too.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
# Config parameters for builders that do not run tests on the builder.
site_config.AddTemplate(
'no_unittest_builder',
unittests=False,
)
# Builder type templates.
site_config.AddTemplate(
'full',
# Full builds are test builds to show that we can build from scratch,
# so use settings to build from scratch, and archive the results.
usepkg_build_packages=False,
chrome_sdk=True,
# Increase master timeout: crbug.com/927886
build_timeout=6 * 60 * 60,
display_label=config_lib.DISPLAY_LABEL_FULL,
build_type=constants.FULL_TYPE,
luci_builder=config_lib.LUCI_BUILDER_FULL,
archive_build_debug=True,
images=['base', 'recovery', 'test', 'factory_install'],
git_sync=True,
description='Full Builds',
image_test=True,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Continuous',
)
site_config.AddTemplate(
'paladin',
chroot_replace=False,
display_label=config_lib.DISPLAY_LABEL_CQ,
build_type=constants.PALADIN_TYPE,
overlays=constants.PUBLIC_OVERLAYS,
luci_builder=config_lib.LUCI_BUILDER_COMMITQUEUE,
manifest_version=True,
description='Commit Queue',
upload_standalone_images=False,
images=['base', 'test'],
image_test=True,
chrome_sdk=True,
chrome_sdk_build_chrome=False,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#TOC-CQ',
# This only applies to vmtest enabled boards like betty and novato.
vm_tests=[config_lib.VMTestConfig(
constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)],
vm_tests_override=TRADITIONAL_VM_TESTS_SUPPORTED,
)
site_config.AddTemplate(
'unittest_only_paladin',
chroot_replace=False,
board_replace=True,
display_label=config_lib.DISPLAY_LABEL_CQ,
build_type=constants.PALADIN_TYPE,
overlays=constants.PUBLIC_OVERLAYS,
luci_builder=config_lib.LUCI_BUILDER_COMMITQUEUE,
manifest_version=True,
unittests=True,
compilecheck=True,
build_packages=False,
upload_standalone_images=False,
sync_chrome=False,
# The unittest only paladin needs to build everything from source
# as a way to ensure ISA compatibility with the remote builder and
# our binpkg selection mechanisms will provide incorrect binaries
# in this case, so we need to disable them entirely.
# TODO(cjmcdonald): Remove this once binpkg fallback selection
# correctly respects -march flags.
usepkg_build_packages=False,
profile='generic_build'
)
# Incremental builders are intended to test the developer workflow.
# For that reason, they don't uprev.
site_config.AddTemplate(
'incremental',
display_label=config_lib.DISPLAY_LABEL_INCREMENATAL,
build_type=constants.INCREMENTAL_TYPE,
luci_builder=config_lib.LUCI_BUILDER_INCREMENTAL,
chroot_replace=False,
uprev=False,
overlays=constants.PUBLIC_OVERLAYS,
description='Incremental Builds',
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Continuous',
)
site_config.AddTemplate(
'informational',
display_label=config_lib.DISPLAY_LABEL_INFORMATIONAL,
description='Informational Builds',
luci_builder=config_lib.LUCI_BUILDER_INFORMATIONAL,
)
site_config.AddTemplate(
'external',
internal=False,
overlays=constants.PUBLIC_OVERLAYS,
manifest_repo_url=config_lib.GetSiteParams().MANIFEST_URL,
manifest=constants.DEFAULT_MANIFEST,
)
# This builds with more source available.
site_config.AddTemplate(
'internal',
internal=True,
overlays=constants.BOTH_OVERLAYS,
manifest_repo_url=config_lib.GetSiteParams().MANIFEST_INT_URL,
)
site_config.AddTemplate(
'infra_builder',
luci_builder=config_lib.LUCI_BUILDER_INFRA,
)
site_config.AddTemplate(
'accelerator',
sync_chrome=False,
chrome_sdk=False,
)
site_config.AddTemplate(
'brillo',
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
)
site_config.AddTemplate(
'lakitu',
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
paygen_skip_testing=True,
)
site_config.AddTemplate(
'lassen',
sync_chrome=False,
chrome_sdk=False,
image_test=False,
)
site_config.AddTemplate(
'x30evb',
sync_chrome=False,
chrome_sdk=False,
signer_tests=False,
paygen=False,
upload_hw_test_artifacts=False,
image_test=False,
images=['base', 'test'],
packages=['virtual/target-os',
'virtual/target-os-dev',
'virtual/target-os-test'],
)
site_config.AddTemplate(
'termina',
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
signer_tests=False,
sign_types=None,
paygen=False,
upload_hw_test_artifacts=False,
upload_stripped_packages=['sys-kernel/*kernel*'],
image_test=False,
images=['base', 'test'],
packages=['virtual/target-os',
'virtual/target-os-dev',
'virtual/target-os-test'],
)
site_config.AddTemplate(
'loonix',
sync_chrome=False,
chrome_sdk=False,
afdo_use=False,
dev_installer_prebuilts=False,
# TODO(harshmodi): Re-enable this when we start using vboot
signer_tests=False,
paygen=False,
upload_hw_test_artifacts=False,
image_test=False,
images=remove_images(['recovery', 'factory_install'])
)
site_config.AddTemplate(
'wshwos',
site_config.templates.loonix
)
site_config.AddTemplate(
'dustbuster',
# TODO(ehislen): Starting with loonix but will diverge later.
site_config.templates.loonix
)
# An anchor of Laktiu' test customizations.
# TODO: renable SIMPLE_AU_TEST_TYPE once b/67510964 is fixed.
site_config.AddTemplate(
'lakitu_test_customizations',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
gce_tests=[config_lib.GCETestConfig(constants.GCE_SUITE_TEST_TYPE,
test_suite='gce-sanity'),
config_lib.GCETestConfig(constants.GCE_SUITE_TEST_TYPE,
test_suite='gce-smoke')],
)
# No GCE tests for lakitu-nc; Enable 'hsm' profile by default.
site_config.AddTemplate(
'lakitu_nc_customizations',
profile='hsm',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
)
# Test customizations for lakitu boards' paladin builders.
site_config.AddTemplate(
'lakitu_paladin_test_customizations',
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
gce_tests=[config_lib.GCETestConfig(constants.GCE_SUITE_TEST_TYPE,
test_suite='gce-sanity')],
)
# An anchor of Laktiu' notification email settings.
site_config.AddTemplate(
'lakitu_notification_emails',
# Send an email on build failures.
health_threshold=1,
health_alert_recipients=['gci-alerts+buildbots@google.com'],
)
site_config.AddTemplate(
'beaglebone',
site_config.templates.brillo,
image_test=False,
rootfs_verification=False,
paygen=False,
signer_tests=False,
images=remove_images(['dev', 'test', 'recovery', 'factory_install']),
)
# This adds Chrome branding.
site_config.AddTemplate(
'official_chrome',
useflags=config_lib.append_useflags([constants.USE_CHROME_INTERNAL]),
)
# This sets chromeos_official.
site_config.AddTemplate(
'official',
site_config.templates.official_chrome,
chromeos_official=True,
)
site_config.AddTemplate(
'asan',
site_config.templates.full,
profile='asan',
luci_builder=config_lib.LUCI_BUILDER_COMMITQUEUE,
# THESE IMAGES CAN DAMAGE THE LAB and cannot be used for hardware testing.
disk_layout='16gb-rootfs',
# TODO(deymo): ASan builders generate bigger files, in particular a bigger
# Chrome binary, that update_engine can't handle in delta payloads due to
# memory limits. Remove the following lines once crbug.com/329248 is
# fixed.
chrome_sdk=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-ASAN',
)
site_config.AddTemplate(
'ubsan',
profile='ubsan',
# Need larger rootfs for ubsan builds.
disk_layout='16gb-rootfs',
chrome_sdk=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
vm_tests_override=None,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-ASAN',
)
site_config.AddTemplate(
'fuzzer',
site_config.templates.full,
site_config.templates.informational,
profile='fuzzer',
chrome_sdk=False,
sync_chrome=False,
# Run fuzzer builder specific stages.
builder_class_name='fuzzer_builders.FuzzerBuilder',
# Need larger rootfs since fuzzing also enables asan.
disk_layout='2gb-rootfs',
gs_path='gs://chromeos-fuzzing-artifacts/libfuzzer-asan',
images=[],
image_test=None,
packages=['virtual/target-fuzzers'],
)
site_config.AddTemplate(
'external_chromium_pfq',
build_type=constants.CHROME_PFQ_TYPE,
uprev=False,
# Increase the master timeout to 8 hours crbug.com/933284.
build_timeout=12 * 60 * 60,
overlays=constants.PUBLIC_OVERLAYS,
manifest_version=True,
chrome_rev=constants.CHROME_REV_LATEST,
chrome_sdk=True,
unittests=False,
description='Preflight Chromium Uprev & Build (public)',
# Add betty smoke VMTest crbug.com/710629.
vm_tests=[config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE)],
vm_tests_override=None,
)
# TODO(davidjames): Convert this to an external config once the unified master
# logic is ready.
site_config.AddTemplate(
'chromium_pfq',
site_config.templates.internal,
site_config.templates.external_chromium_pfq,
display_label=config_lib.DISPLAY_LABEL_CHROME_PFQ,
description='Preflight Chromium Uprev & Build (internal)',
overlays=constants.BOTH_OVERLAYS,
prebuilts=constants.PUBLIC,
luci_builder=config_lib.LUCI_BUILDER_PFQ,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Chrome-PFQ',
)
site_config.AddTemplate(
'chrome_pfq',
site_config.templates.chromium_pfq,
site_config.templates.official,
overlays=constants.BOTH_OVERLAYS,
description='Preflight Chrome Uprev & Build (internal)',
prebuilts=constants.PRIVATE,
)
site_config.AddTemplate(
'chrome_try',
build_type=constants.CHROME_PFQ_TYPE,
chrome_rev=constants.CHROME_REV_TOT,
manifest_version=False,
)
site_config.AddTemplate(
'chromium_pfq_informational',
site_config.templates.external_chromium_pfq,
site_config.templates.chrome_try,
site_config.templates.informational,
display_label=config_lib.DISPLAY_LABEL_CHROME_INFORMATIONAL,
chrome_sdk=False,
unittests=False,
description='Informational Chromium Uprev & Build (public)',
)
site_config.AddTemplate(
'chrome_pfq_informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.internal,
site_config.templates.official,
unittests=False,
description='Informational Chrome Uprev & Build (internal)',
)
site_config.AddTemplate(
'chrome_pfq_cheets_informational',
site_config.templates.chrome_pfq_informational,
)
site_config.AddTemplate(
'telemetry',
site_config.templates.chromium_pfq_informational,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='telemetry_unit_server',
# Add an extra 60 minutes.
timeout=120 * 60)],
description='Telemetry Builds',
)
site_config.AddTemplate(
'tot_asan_informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.asan,
site_config.templates.informational,
unittests=True,
description='Build TOT Chrome with Address Sanitizer (Clang)',
board_replace=True,
)
site_config.AddTemplate(
'tot_ubsan_informational',
site_config.templates.chromium_pfq_informational,
site_config.templates.ubsan,
site_config.templates.informational,
unittests=True,
description='Build TOT Chrome with Undefined Behavior Sanitizer (Clang)',
board_replace=True,
)
site_config.AddTemplate(
'chrome_perf',
site_config.templates.chrome_pfq_informational,
site_config.templates.no_unittest_builder,
description='Chrome Performance test bot',
hw_tests=[config_lib.HWTestConfig(
'perf_v2', pool=constants.HWTEST_CHROME_PERF_POOL,
timeout=90 * 60, critical=True)],
use_chrome_lkgm=True,
useflags=config_lib.append_useflags(['-cros-debug']),
)
site_config.AddTemplate(
'pre_flight_branch',
site_config.templates.internal,
site_config.templates.official_chrome,
build_type=constants.PFQ_TYPE,
luci_builder=config_lib.LUCI_BUILDER_PFQ,
build_timeout=20 * 60,
manifest_version=True,
branch=True,
master=True,
slave_configs=[],
vm_tests=[],
vm_tests_override=TRADITIONAL_VM_TESTS_SUPPORTED,
hw_tests=[],
hw_tests_override=[],
unittests=False,
uprev=True,
overlays=constants.BOTH_OVERLAYS,
push_overlays=constants.BOTH_OVERLAYS,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Chrome-PFQ')
site_config.AddTemplate(
'internal_paladin',
site_config.templates.paladin,
site_config.templates.internal,
site_config.templates.official_chrome,
manifest=constants.OFFICIAL_MANIFEST,
overlays=constants.BOTH_OVERLAYS,
vm_tests=[],
description=site_config.templates.paladin.description + ' (internal)',
)
# Used for paladin builders with nowithdebug flag (a.k.a -cros-debug)
site_config.AddTemplate(
'internal_nowithdebug_paladin',
site_config.templates.internal_paladin,
useflags=config_lib.append_useflags(['-cros-debug']),
description=(site_config.templates.paladin.description +
' (internal, nowithdebug)'),
)
# Internal incremental builders don't use official chrome because we want
# to test the developer workflow.
site_config.AddTemplate(
'internal_incremental',
site_config.templates.internal,
site_config.templates.incremental,
overlays=constants.BOTH_OVERLAYS,
description='Incremental Builds (internal)',
)
# A test-ap image is just a test image with a special profile enabled.
# Note that each board enabled for test-ap use has to have the testbed-ap
# profile linked to from its private overlay.
site_config.AddTemplate(
'test_ap',
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_UTILITY,
build_type=constants.INCREMENTAL_TYPE,
description='WiFi AP images used in testing',
profile='testbed-ap',
)
# Create tryjob build configs to help with stress testing.
site_config.AddTemplate(
'unittest_stress',
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
build_type=constants.TRYJOB_TYPE,
description='Run Unittests repeatedly to look for flake.',
builder_class_name='test_builders.UnittestStressBuilder',
# Make this available, so we can stress a previous build.
manifest_version=True,
)
site_config.AddTemplate(
'release_common',
site_config.templates.full,
site_config.templates.official,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_RELEASE,
build_type=constants.CANARY_TYPE,
luci_builder=config_lib.LUCI_BUILDER_PROD,
chroot_use_image=False,
suite_scheduling=True,
# Because release builders never use prebuilts, they need the
# longer timeout. See crbug.com/938958.
build_timeout=12 * 60 * 60,
useflags=config_lib.append_useflags(['-cros-debug']),
afdo_use=True,
manifest=constants.OFFICIAL_MANIFEST,
manifest_version=True,
images=['base', 'recovery', 'test', 'factory_install'],
sign_types=['recovery'],
push_image=True,
upload_symbols=True,
run_cpeexport=True,
run_build_configs_export=True,
binhost_bucket='gs://chromeos-dev-installer',
binhost_key='RELEASE_BINHOST',
binhost_base_url='https://commondatastorage.googleapis.com/'
'chromeos-dev-installer',
dev_installer_prebuilts=True,
git_sync=False,
vm_tests=[
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'),
config_lib.VMTestConfig(constants.DEV_MODE_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)],
# Some release builders disable VMTests to be able to build on GCE, but
# still want VMTests enabled on trybot builders.
vm_tests_override=[
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'),
config_lib.VMTestConfig(constants.DEV_MODE_TEST_TYPE),
config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE)],
paygen=True,
signer_tests=True,
hwqual=True,
description='Release Builds (canary) (internal)',
chrome_sdk=True,
image_test=True,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Canaries',
)
site_config.AddTemplate(
'release',
site_config.templates.release_common,
luci_builder=config_lib.LUCI_BUILDER_LEGACY_RELEASE,
)
site_config.AddTemplate(
'factory_firmware',
site_config.templates.release_common,
luci_builder=config_lib.LUCI_BUILDER_FACTORY,
)
### Release AFDO configs.
site_config.AddTemplate(
'release_afdo',
site_config.templates.release,
suite_scheduling=False,
push_image=False,
paygen=False,
dev_installer_prebuilts=False,
)
site_config.AddTemplate(
'release_afdo_generate',
site_config.templates.release_afdo,
afdo_generate_min=True,
afdo_use=False,
afdo_update_ebuild=True,
)
site_config.AddTemplate(
'release_afdo_use',
site_config.templates.release_afdo,
afdo_use=True,
)
site_config.AddTemplate(
'moblab_release',
site_config.templates.release,
description='Moblab release builders',
images=['base', 'recovery', 'test'],
afdo_use=False,
signer_tests=False,
)
# Factory and Firmware releases much inherit from these classes.
# Modifications for these release builders should go here.
# Naming conventions also must be followed. Factory and firmware branches
# must end in -factory or -firmware suffixes.
site_config.AddTemplate(
'factory',
site_config.templates.factory_firmware,
display_label=config_lib.DISPLAY_LABEL_FACTORY,
afdo_use=False,
chrome_sdk=False,
chrome_sdk_build_chrome=False,
description='Factory Builds',
dev_installer_prebuilts=False,
factory_toolkit=True,
hwqual=False,
images=['test', 'factory_install'],
image_test=False,
paygen=False,
signer_tests=False,
sign_types=['factory'],
upload_hw_test_artifacts=False,
upload_symbols=False,
)
# This should be used by any "workspace_builders."
site_config.AddTemplate(
'workspace',
postsync_patch=False,
)
# Requires that you set boards, and workspace_branch.
site_config.AddTemplate(
'firmwarebranch',
site_config.templates.factory_firmware,
site_config.templates.workspace,
display_label=config_lib.DISPLAY_LABEL_FIRMWARE,
images=[],
hwqual=False,
factory_toolkit=False,
packages=['virtual/chromeos-firmware'],
usepkg_build_packages=False,
sync_chrome=False,
chrome_sdk=False,
unittests=False,
dev_installer_prebuilts=False,
upload_hw_test_artifacts=False,
upload_symbols=False,
useflags=config_lib.append_useflags(['chromeless_tty']),
signer_tests=False,
paygen=False,
image_test=False,
manifest=constants.DEFAULT_MANIFEST,
afdo_use=False,
sign_types=['firmware', 'accessory_rwsig'],
build_type=constants.GENERIC_TYPE,
uprev=True,
overlays=constants.BOTH_OVERLAYS,
push_overlays=constants.BOTH_OVERLAYS,
builder_class_name='workspace_builders.FirmwareBranchBuilder',
build_timeout=6*60 * 60,
description='TOT builder to build a firmware branch.',
doc='https://goto.google.com/tot-for-firmware-branches',
)
site_config.AddTemplate(
'payloads',
site_config.templates.internal,
site_config.templates.no_unittest_builder,
display_label=config_lib.DISPLAY_LABEL_TRYJOB,
build_type=constants.PAYLOADS_TYPE,
luci_builder=config_lib.LUCI_BUILDER_RELEASE,
builder_class_name='release_builders.GeneratePayloadsBuilder',
description='Regenerate release payloads.',
# Sync to the code used to do the build the first time.
manifest_version=True,
# This is the actual work we want to do.
paygen=True,
upload_hw_test_artifacts=False,
)
site_config.AddTemplate(
'build_external_chrome',
useflags=config_lib.append_useflags(
['-%s' % constants.USE_CHROME_INTERNAL]),
)
# Tast is an alternate system for running integration tests.
# The expression specified here matches the union of the tast.critical-*
# Autotest server tests, which are executed by the bvt-tast-cq suite on real
# hardware in the lab.
site_config.AddTemplate(
'tast_vm_paladin_tests',
tast_vm_tests=[
config_lib.TastVMTestConfig(
'tast_vm_paladin',
['(!disabled && !"group:*" && !informational)'])],
)
# The expression specified here matches the union of the
# tast.critical-{android,chrome} Autotest server tests, which are executed by
# the bvt-tast-chrome-pfq suite on real hardware in the lab.
site_config.AddTemplate(
'tast_vm_chrome_pfq_tests',
tast_vm_tests=[
config_lib.TastVMTestConfig(
'tast_vm_chrome_pfq',
['(!disabled && !"group:*" && !informational && '
'("dep:android" || "dep:android_all" || "dep:chrome"))'])],
)
# The expression specified here matches the tast.critical-android Autotest
# server test, which is executed by the bvt-tast-android-pfq suite on real
# hardware in the lab.
site_config.AddTemplate(
'tast_vm_android_pfq_tests',
tast_vm_tests=[
config_lib.TastVMTestConfig(
'tast_vm_android_pfq',
['(!disabled && !"group:*" && !informational && '
'("dep:android" || "dep:android_all"))'])],
)
# The expression specified here matches the union of the tast.critical-* and
# tast.informational-* Autotest server tests, which are executed by the
# bvt-tast-cq and bvt-tast-informational suites on real hardware in the lab.
site_config.AddTemplate(
'tast_vm_canary_tests',
tast_vm_tests=[config_lib.TastVMTestConfig(
'tast_vm_canary', ['(!disabled && !"group:*")'])],
)
site_config.AddTemplate(
'moblab_vm_tests',
moblab_vm_tests=[
config_lib.MoblabVMTestConfig(constants.MOBLAB_VM_SMOKE_TEST_TYPE)],
)
site_config.AddTemplate(
'buildspec',
site_config.templates.workspace,
site_config.templates.internal,
luci_builder=config_lib.LUCI_BUILDER_FACTORY,
master=True,
boards=[],
build_type=constants.GENERIC_TYPE,
uprev=True,
overlays=constants.BOTH_OVERLAYS,
push_overlays=constants.BOTH_OVERLAYS,
builder_class_name='workspace_builders.BuildSpecBuilder',
build_timeout=4*60 * 60,
description='Buildspec creator.',
)
def CreateBoardConfigs(site_config, boards_dict, ge_build_config):
"""Create mixin templates for each board."""
# Extract the full list of board names from GE data.
separate_board_names = set(config_lib.GeBuildConfigAllBoards(ge_build_config))
unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config)
unified_board_names = set([b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]
for b in unified_builds])
board_names = separate_board_names | unified_board_names
# TODO(crbug.com/648473): Remove these, after GE adds them to their data set.
board_names = board_names.union(boards_dict['all_boards'])
result = dict()
for board in board_names:
board_config = config_lib.BuildConfig(boards=[board])
if board in chromeos_boards.brillo_boards:
board_config.apply(site_config.templates.brillo)
if board in chromeos_boards.lakitu_boards:
board_config.apply(site_config.templates.lakitu)
if board in chromeos_boards.lassen_boards:
board_config.apply(site_config.templates.lassen)
if board in ['x30evb']:
board_config.apply(site_config.templates.x30evb)
if board in chromeos_boards.loonix_boards:
board_config.apply(site_config.templates.loonix)
if board in chromeos_boards.wshwos_boards:
board_config.apply(site_config.templates.wshwos)
if board in chromeos_boards.dustbuster_boards:
board_config.apply(site_config.templates.dustbuster)
if board in chromeos_boards.moblab_boards:
board_config.apply(site_config.templates.moblab)
if board in chromeos_boards.accelerator_boards:
board_config.apply(site_config.templates.accelerator)
if board in chromeos_boards.termina_boards:
board_config.apply(site_config.templates.termina)
if board in chromeos_boards.nofactory_boards:
board_config.apply(factory=False,
factory_toolkit=False,
factory_install_netboot=False,
images=remove_images(['factory_install']))
if board in chromeos_boards.toolchains_from_source:
board_config.apply(usepkg_toolchain=False)
if board in chromeos_boards.noimagetest_boards:
board_config.apply(image_test=False)
if board in chromeos_boards.nohwqual_boards:
board_config.apply(hwqual=False)
if board in chromeos_boards.norootfs_verification_boards:
board_config.apply(rootfs_verification=False)
if board in chromeos_boards.base_layout_boards:
board_config.apply(disk_layout='base')
if board in chromeos_boards.beaglebone_boards:
board_config.apply(site_config.templates.beaglebone)
if board in chromeos_boards.builder_incompatible_binaries_boards:
board_config.apply(unittests=False)
result[board] = board_config
return result
def CreateInternalBoardConfigs(site_config, boards_dict, ge_build_config):
"""Create mixin templates for each board."""
result = CreateBoardConfigs(
site_config, boards_dict, ge_build_config)
for board in boards_dict['internal_boards']:
if board in result:
result[board].apply(site_config.templates.internal,
site_config.templates.official_chrome,
manifest=constants.OFFICIAL_MANIFEST)
return result
def UpdateBoardConfigs(board_configs, boards, *args, **kwargs):
"""Update "board_configs" for selected chromeos_boards.
Args:
board_configs: Dict in CreateBoardConfigs format to filter from.
boards: Iterable of boards to update in the dict.
args: List of templates to apply.
kwargs: Individual keys to update.
Returns:
Copy of board_configs dict with boards boards update with templates
and values applied.
"""
result = board_configs.copy()
for b in boards:
result[b] = result[b].derive(*args, **kwargs)
return result
def ToolchainBuilders(site_config, boards_dict, ge_build_config):
"""Define templates used for toolchain builders.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
site_config.AddTemplate(
'base_toolchain',
# Full build, AFDO, latest-toolchain, -cros-debug, and simple-chrome.
site_config.templates.full,
display_label=config_lib.DISPLAY_LABEL_TOOLCHAIN,
build_type=constants.TOOLCHAIN_TYPE,
build_timeout=(15 * 60 + 50) * 60,
# Need to re-enable platform_SyncCrash after issue crosbug/658864 is
# fixed. Need to re-enable network_VPNConnect.* tests after issue
# crosbug/585936 is fixed. Need to re-enable
# power_DarkResumeShutdownServer after issue crosbug/689598 is fixed.
# According to crosbug/653496 security_OpenFDs will not work for
# non-official builds, so we need to leave it permanently disabled.
# Need to reenable power_DarkResumeDisplay after crosbug/703250 is fixed.
# Need to reenable cheets_SELinuxTest after crosbug/693308 is fixed.
# Need to reenable security_SMMLocked when crosbug/654610 is fixed.
# Add strict_toolchain_checks to perform toolchain-related checks
useflags=config_lib.append_useflags([
'-cros-debug',
'-tests_security_OpenFDs',
'-tests_platform_SyncCrash',
'-tests_network_VPNConnect.l2tpipsec_xauth',
'-tests_network_VPNConnect.l2tpipsec_psk',
'-tests_power_DarkResumeShutdownServer',
'-tests_power_DarkResumeDisplay',
'-tests_security_SMMLocked',
'-tests_cheets_SELinuxTest',
'thinlto',
'strict_toolchain_checks']),
afdo_use=True,
latest_toolchain=True,
enable_skylab_hw_tests=True,
)
site_config.AddTemplate(
'toolchain',
site_config.templates.base_toolchain,
site_config.templates.internal,
site_config.templates.official_chrome,
site_config.templates.no_vmtest_builder,
images=['base', 'test', 'recovery'],
manifest=constants.OFFICIAL_MANIFEST,
manifest_version=True,
git_sync=False,
description='Toolchain Builds (internal)',
)
site_config.AddTemplate(
'gcc_toolchain',
site_config.templates.toolchain,
description='Full release build with next minor GCC toolchain revision',
useflags=config_lib.append_useflags(['next_gcc']),
hw_tests=hw_test_list.ToolchainTestFull(
constants.HWTEST_QUOTA_POOL,
quota_account='toolchain'
),
hw_tests_override=hw_test_list.ToolchainTestFull(
constants.HWTEST_QUOTA_POOL,
quota_account='toolchain'
),
)
site_config.AddTemplate(
'llvm_toolchain',
site_config.templates.toolchain,
description='Full release build with LLVM toolchain',
hw_tests=hw_test_list.ToolchainTestMedium(
constants.HWTEST_QUOTA_POOL,
quota_account='toolchain'
),
hw_tests_override=hw_test_list.ToolchainTestMedium(
constants.HWTEST_QUOTA_POOL,
quota_account='toolchain'
),
)
site_config.AddTemplate(
'llvm_next_toolchain',
site_config.templates.llvm_toolchain,
description='Full release build with LLVM (next) toolchain',
useflags=config_lib.append_useflags(['llvm-next']),
)
# This builds everything with ToT LLVM. LLVM is a fast-moving target, so we
# may consider dropping expensive things (e.g. ThinLTO) to increase coverage.
#
# Since the most recent LLVM has a nonzero chance of being broken, it has a
# nonzero chance of producing very broken images. Only VMTests are targeted
# with this for now, so we don't put hardware in a bad state.
#
# This should only be used with external configs.
site_config.AddTemplate(
'llvm_tot_toolchain',
site_config.templates.base_toolchain,
site_config.templates.no_hwtest_builder,
images=['base', 'test'],
useflags=config_lib.append_useflags(['llvm-tot']),
description='Builds Chrome OS using top-of-tree LLVM',
)
site_config.AddTemplate(
'afdo_toolchain',
site_config.templates.full,
site_config.templates.official,
site_config.templates.internal,
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
git_sync=False,
images=[],
unittests=False,
image_test=False,
chrome_sdk=False,
paygen=False,
signer_tests=False,
useflags=config_lib.append_useflags(['-cros-debug']),
display_label=config_lib.DISPLAY_LABEL_TOOLCHAIN,
)
site_config.AddTemplate(
'afdo_verify',
site_config.templates.afdo_toolchain,
images=['base', 'test'],
image_test=True,
unittests=True,
afdo_use=True,
)
# This build config is dedicated to improve code layout of Chrome. It adopts
# the Call-Chain Clustering (C3) approach and other techniques to improve
# the code layout. It builds Chrome and generates an orderfile as a result.
# The orderfile will be uploaded so Chrome in the future production will use
# the orderfile to improve code layout.
#
# This builder is not a toolchain builder, i.e. it won't build all the
# toolchain. Instead, it's a release builder. It's put here because it
# belongs to the toolchain team.
site_config.AddTemplate(
'orderfile_generate_toolchain',
site_config.templates.afdo_toolchain,
afdo_use=True,
orderfile_generate=True,
useflags=config_lib.append_useflags(['orderfile_generate']),
description='Build Chrome and generate an orderfile for better layout',
)
site_config.AddTemplate(
'orderfile_verify_toolchain',
site_config.templates.afdo_verify,
orderfile_verify=True,
useflags=config_lib.append_useflags(['orderfile_verify',
'-reorder_text_sections',
'strict_toolchain_checks']),
description='Verify the most recent orderfile is building correctly',
)
# This build config is used to asynchronously generate benchmark
# AFDO profile. It inherites from afdo_toolchain template, i.e.
# it is essentially a release builder but doesn't run hwtests, etc.
# But it should run a special HWTest "AFDOGenerate" to collect
# profiling data on a device. So this template needs to build image.
site_config.AddTemplate(
'benchmark_afdo_generate',
site_config.templates.afdo_toolchain,
images=['test'],
hw_tests=[config_lib.HWTestConfig(constants.HWTEST_AFDO_SUITE,
file_bugs=True)],
hw_tests_override=[config_lib.HWTestConfig(constants.HWTEST_AFDO_SUITE,
file_bugs=True)],
description='Generate AFDO profile based on benchmarks.'
)
# This build config is used to verify merged and redacted AFDO profile,
# aka. profiles for release, can build packages without any problems.
site_config.AddTemplate(
'release_afdo_verify',
site_config.templates.afdo_verify,
description='Verify the most recent AFDO profile for release can '
'build Chrome images correctly.'
)
### Toolchain waterfall entries.
### Toolchain builder configs: 3 architectures {amd64,arm,arm64}
### x 1 toolchains {llvm-next}
### All of these builders should be slaves of 'master-toolchain'.
### Master toolchain config.
master = site_config.Add(
'master-toolchain',
site_config.templates.toolchain,
boards=[],
description='Toolchain master (all others are slaves).',
master=True,
sync_chrome=True,
health_alert_recipients=['c-compiler-chrome@google.com'],
health_threshold=1,
afdo_use=False,
slave_configs=[],
# 3 PM UTC is 7 AM PST (no daylight savings)
schedule='0 15 * * *',
)
def toolchainSlaveHelper(name, board, *args, **kwargs):
master.AddSlaves([
site_config.Add(
name + '-llvm-next-toolchain',
site_config.templates.llvm_next_toolchain,
*args,
boards=[board],
**kwargs
)
])
# Create all waterfall slave builders.
toolchainSlaveHelper('amd64', 'eve')
toolchainSlaveHelper('arm', 'veyron_mighty')
toolchainSlaveHelper('arm64', 'kevin')
#
# Create toolchain tryjob builders.
#
builder_to_boards_dict = config_lib.GroupBoardsByBuilder(
ge_build_config[config_lib.CONFIG_TEMPLATE_BOARDS])
toolchain_tryjob_boards = builder_to_boards_dict[
config_lib.CONFIG_TEMPLATE_RELEASE] | boards_dict['all_boards']
site_config.AddForBoards(
'llvm-toolchain',
toolchain_tryjob_boards,
board_configs,
site_config.templates.llvm_toolchain,
)
site_config.AddForBoards(
'llvm-next-toolchain',
toolchain_tryjob_boards,
board_configs,
site_config.templates.llvm_next_toolchain,
)
site_config.Add(
'llvm-clang-tidy-toolchain',
site_config.templates.toolchain,
site_config.templates.no_hwtest_builder,
description='Full release build with LLVM toolchain, with clang-tidy)',
chrome_sdk=False,
# Run clang-tidy specific stages.
builder_class_name='clang_tidy_builders.ClangTidyBuilder',
useflags=config_lib.append_useflags(['clang_tidy']),
boards=['grunt'],
# Weekly on Sunday 3 AM UTC
schedule='0 0 3 * * 0 *',
)
def PGOBuilders(name, board):
site_config.Add(
name + '-pgo-generate-llvm-next-toolchain',
site_config.templates.llvm_next_toolchain,
site_config.templates.no_hwtest_builder,
description='Full release build with PGO instrumented LLVM toolchain)',
chrome_sdk=False,
# Run PGO generate specific stages.
builder_class_name='pgo_generate_builders.PGOGenerateBuilder',
useflags=config_lib.append_useflags(['llvm_pgo_generate',
'-llvm_pgo_use']),
boards=[board],
images=['base'],
# Build chrome as C++ training set, and kernel as C training set.
packages=[
'chromeos-base/chromeos-chrome',
'virtual/linux-sources'
],
# Weekly on Sunday 1 AM UTC
schedule='0 0 1 * * 0 *',
)
# Create three PGO profile collecting builders.
PGOBuilders('amd64', 'eve')
PGOBuilders('arm', 'kevin')
PGOBuilders('arm64', 'kevin64')
# All *-generic boards are external.
site_config.Add(
'amd64-generic-llvm-tot-toolchain',
site_config.templates.llvm_tot_toolchain,
display_label=config_lib.DISPLAY_LABEL_TOOLCHAIN,
vm_tests=[config_lib.VMTestConfig(
constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)],
vm_tests_override=TRADITIONAL_VM_TESTS_SUPPORTED,
boards=['amd64-generic'],
)
site_config.Add(
'arm-generic-llvm-tot-toolchain',
site_config.templates.llvm_tot_toolchain,
site_config.templates.no_vmtest_builder,
boards=['arm-generic'],
)
site_config.Add(
'arm64-generic-llvm-tot-toolchain',
site_config.templates.llvm_tot_toolchain,
site_config.templates.no_vmtest_builder,
boards=['arm64-generic'],
)
site_config.Add(
'benchmark-afdo-generate',
site_config.templates.benchmark_afdo_generate,
boards=['chell'],
# TODO: Add a schedule
)
site_config.Add(
'release-afdo-verify',
site_config.templates.release_afdo_verify,
boards=['eve'],
# TODO: Add a schedule
)
site_config.Add(
'orderfile-generate-toolchain',
site_config.templates.orderfile_generate_toolchain,
# The board should not matter much, since we are not running
# anything on the board.
boards=['terra'],
# 10 AM UTC is 2 AM PST (no daylight savings)
schedule='0 10 * * *',
health_alert_recipients=['c-compiler-chrome@google.com'],
# Send emails if this builder fails once
health_threshold=1,
)
site_config.Add(
'orderfile-verify-toolchain',
site_config.templates.orderfile_verify_toolchain,
# Only test on X86 for now.
boards=['eve'],
# 2 PM UTC is 6 AM PST (no daylight savings)
# Start this builder 4 hours after orderfile-generate-toolchain
schedule='0 14 * * *',
health_alert_recipients=['c-compiler-chrome@google.com'],
# Send emails if this builder fails once
health_threshold=1,
)
# This config is manually run when we want to generate a 'release' AFDO
# profile, which is a mixture of CWP/benchmark profiles. This is done
# manually once per branch, since these are (currently) hand-rolled into
# branches for Android and Linux builds of Chrome. See crbug.com/858856 for
# context (comment #4 in particular).
#
# FIXME(gbiv): Ideally, this should be done as a part of the regular AFDO
# pipeline if Chrome OS finds these profiles useful.
site_config.Add(
'release-afdo-profile-generate',
# No board is necessary; this just runs a few commands inside of the SDK.
boards=[],
build_type=constants.TOOLCHAIN_TYPE,
builder_class_name='release_profile_builders.ReleaseProfileMergeBuilder',
chrome_sdk=False,
description='Generates merged AFDO profiles for select Chrome releases',
display_label=config_lib.DISPLAY_LABEL_TOOLCHAIN,
)
def PreCqBuilders(site_config, boards_dict, ge_build_config):
"""Create all build configs associated with the PreCQ.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
# The PreCQ Launcher doesn't limit eternal PreCQ builds to external
# CLs. as a hack, use internal checkouts for external builds so
# they can apply (and ignore) internal CLs. crbug.com/882965
for b in (chromeos_boards.arm_external_boards
| chromeos_boards.x86_external_boards):
board_configs[b].apply(site_config.templates.internal)
site_config.AddTemplate(
'pre_cq',
site_config.templates.paladin,
display_label=config_lib.DISPLAY_LABEL_PRECQ,
luci_builder=config_lib.LUCI_BUILDER_PRECQ,
build_type=constants.PRE_CQ_TYPE,
build_timeout=4 * 60 * 60,
pre_cq=True,
archive=False,
chrome_sdk=False,
chroot_replace=True,
debug_symbols=False,
prebuilts=False,
cpe_export=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke',
use_ctest=False)],
vm_tests_override=None,
description='Verifies compilation, building an image, and vm/unit tests '
'if supported.',
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Pre-CQ',
sanity_check_threshold=3,
)
# Pre-CQ targets that only check compilation and unit tests.
site_config.AddTemplate(
'unittest_only_pre_cq',
site_config.templates.pre_cq,
site_config.templates.no_vmtest_builder,
description='Verifies compilation and unit tests only',
compilecheck=True,
)
# Pre-CQ targets that don't run VMTests.
site_config.AddTemplate(
'no_vmtest_pre_cq',
site_config.templates.pre_cq,
site_config.templates.no_vmtest_builder,
description='Verifies compilation, building an image, and unit tests '
'if supported.',
)
# Pre-CQ targets that only check compilation.
site_config.AddTemplate(
'compile_only_pre_cq',
site_config.templates.unittest_only_pre_cq,
description='Verifies compilation only',
unittests=False,
)
site_config.AddWithoutTemplate(
'pre-cq-launcher',
site_config.templates.paladin,
site_config.templates.internal_paladin,
site_config.templates.no_vmtest_builder,
site_config.templates.no_hwtest_builder,
boards=[],
display_label=config_lib.DISPLAY_LABEL_PRECQ,
build_type=constants.PRE_CQ_LAUNCHER_TYPE,
luci_builder=config_lib.LUCI_BUILDER_PRECQ_LAUNCHER,
description='Launcher for Pre-CQ builders',
manifest_version=False,
doc='https://dev.chromium.org/chromium-os/build/builder-overview#'
'TOC-Pre-CQ',
schedule='with 3m interval',
)
# Add a pre-cq config for every board.
site_config.AddForBoards(
'pre-cq',
boards_dict['all_boards'],
board_configs,
site_config.templates.pre_cq,
)
# Add special pre-cq for generic build tests with non-default kernel version
site_config.AddForBoards(
'linux-v4_4-pre-cq',
boards_dict['generic_kernel_boards'],
board_configs,
site_config.templates.pre_cq,
useflags=config_lib.append_useflags(['kernel-4_4', '-kernel-4_14',
'-kernel-4_19']),
)
site_config.AddForBoards(
'linux-v4_4-buildtest-compile-only-pre-cq',
boards_dict['generic_kernel_boards'],
board_configs,
site_config.templates.compile_only_pre_cq,
useflags=config_lib.append_useflags(['kernel-4_4', '-kernel-4_14',
'-kernel-4_19',
'-clang', 'buildtest',
'-chrome_internal']),
packages=['sys-kernel/chromeos-kernel-4_4'],
)
site_config.AddForBoards(
'linux-v4_14-pre-cq',
boards_dict['generic_kernel_boards'],
board_configs,
site_config.templates.pre_cq,
useflags=config_lib.append_useflags(['-kernel-4_4', 'kernel-4_14',
'-kernel-4_19']),
)
site_config.AddForBoards(
'linux-v4_14-buildtest-compile-only-pre-cq',
boards_dict['generic_kernel_boards'],
board_configs,
site_config.templates.compile_only_pre_cq,
useflags=config_lib.append_useflags(['-kernel-4_4', 'kernel-4_14',
'-kernel-4_19',
'-clang', 'buildtest']),
packages=['sys-kernel/chromeos-kernel-4_14'],
)
site_config.AddForBoards(
'linux-v4_19-pre-cq',
boards_dict['generic_kernel_boards'],
board_configs,
site_config.templates.pre_cq,
useflags=config_lib.append_useflags(['-kernel-4_4', '-kernel-4_14',
'kernel-4_19']),
)
site_config.AddForBoards(
'linux-v4_19-buildtest-compile-only-pre-cq',
boards_dict['generic_kernel_boards'],
board_configs,
site_config.templates.compile_only_pre_cq,
useflags=config_lib.append_useflags(['-kernel-4_4', '-kernel-4_14',
'kernel-4_19',
'-clang', 'buildtest']),
packages=['sys-kernel/chromeos-kernel-4_19'],
)
site_config.AddForBoards(
'no-vmtest-pre-cq',
boards_dict['all_boards'],
board_configs,
site_config.templates.no_vmtest_pre_cq,
)
site_config.AddForBoards(
'compile-only-pre-cq',
boards_dict['all_boards'],
board_configs,
site_config.templates.compile_only_pre_cq,
)
site_config.Add(
constants.BINHOST_PRE_CQ,
site_config.templates.pre_cq,
site_config.templates.no_vmtest_pre_cq,
site_config.templates.internal,
boards=[],
binhost_test=True,
)
# Wifi specific PreCQ.
site_config.AddTemplate(
'wificell_pre_cq',
site_config.templates.pre_cq,
unittests=False,
hw_tests=hw_test_list.WiFiCellPoolPreCQ(),
hw_tests_override=hw_test_list.WiFiCellPoolPreCQ(),
archive=True,
image_test=False,
description='WiFi tests acting as pre-cq for WiFi related changes',
)
_wifi_boards = frozenset([
'winky',
'veyron_speedy',
'veyron_jerry',
'lulu',
'cyan',
'elm',
])
site_config.AddForBoards(
'wificell-pre-cq',
_wifi_boards,
board_configs,
site_config.templates.wificell_pre_cq,
)
# Bluestreak specific PreCQ.
site_config.Add(
'bluestreak-pre-cq',
board_configs['guado'],
site_config.templates.pre_cq,
hw_tests=hw_test_list.BluestreakPoolPreCQ(),
hw_tests_override=hw_test_list.BluestreakPoolPreCQ(),
archive=True,
description='Bluestreak tests as pre-cq for CFM related changes',
)
site_config.Add(
'signer-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='test_builders.SignerTestsBuilder',
description='Run the signer unittests.',
)
site_config.Add(
'chromite-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='test_builders.ChromiteTestsBuilder',
description='Run the chromite unittests.',
)
site_config.Add(
'cbuildbot-launch-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='test_builders.CbuildbotLaunchTestBuilder',
description='Run cbuildbot_launch test builds.',
)
# Pre-cq for lakitu's public overlay.
site_config.Add(
'lakitu-external-pre-cq',
site_config.templates.pre_cq,
board_configs['lakitu'],
site_config.templates.lakitu,
site_config.templates.external,
useflags=config_lib.append_useflags(['-chrome_internal']),
)
site_config.AddWithoutTemplate(
'chromeos-infra-puppet-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='infra_builders.PuppetPreCqBuilder',
use_sdk=True,
build_timeout=60 * 60,
description='Test Puppet specs',
doc='https://chrome-internal.googlesource.com/'
'chromeos/chromeos-admin/+/HEAD/puppet/README.md',
)
site_config.AddWithoutTemplate(
'chromeos-infra-go-pre-cq',
site_config.templates.pre_cq,
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='infra_builders.InfraGoPreCqBuilder',
use_sdk=True,
build_timeout=60 * 60,
description='Test building Chromium OS infra Go binaries',
doc='https://goto.google.com/cros-infra-go-packaging',
)
site_config.AddWithoutTemplate(
'chromeos-infra-unittests-pre-cq',
site_config.templates.pre_cq,
site_config.templates.internal,
site_config.templates.no_hwtest_builder,
site_config.templates.no_unittest_builder,
site_config.templates.no_vmtest_builder,
boards=[],
builder_class_name='infra_builders.InfraUnittestsPreCqBuilder',
use_sdk=True,
build_timeout=60 * 60,
description='Run unittests for infra repositories',
)
def AndroidTemplates(site_config):
"""Apply Android specific config to site_config
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
"""
# Generic template shared by all Android versions.
site_config.AddTemplate(
'generic_android_pfq',
site_config.templates.no_vmtest_builder,
build_type=constants.ANDROID_PFQ_TYPE,
uprev=False,
overlays=constants.BOTH_OVERLAYS,
manifest_version=True,
android_rev=constants.ANDROID_REV_LATEST,
description='Preflight Android Uprev & Build (internal)',
luci_builder=config_lib.LUCI_BUILDER_PFQ,
)
# Template for Android NYC.
site_config.AddTemplate(
'nyc_android_pfq',
site_config.templates.generic_android_pfq,
display_label=config_lib.DISPLAY_LABEL_NYC_ANDROID_PFQ,
android_package='android-container-nyc',
android_import_branch=constants.ANDROID_NYC_BUILD_BRANCH,
android_gts_build_branch='git_nyc-mr2-dev',
)
# Template for Android Pi.
site_config.AddTemplate(
'pi_android_pfq',
site_config.templates.generic_android_pfq,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_PI_ANDROID_PFQ,
android_package='android-container-pi',
android_import_branch=constants.ANDROID_PI_BUILD_BRANCH,
)
# Template for Android VM Pi.
site_config.AddTemplate(
'vmpi_android_pfq',
site_config.templates.generic_android_pfq,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_VMPI_ANDROID_PFQ,
android_package='android-vm-pi',
android_import_branch=constants.ANDROID_VMPI_BUILD_BRANCH,
)
# Template for Android Qt.
site_config.AddTemplate(
'qt_android_pfq',
site_config.templates.generic_android_pfq,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_QT_ANDROID_PFQ,
android_package='android-container-qt',
android_import_branch=constants.ANDROID_QT_BUILD_BRANCH,
)
# Template for Android Master.
site_config.AddTemplate(
'mst_android_pfq',
site_config.templates.generic_android_pfq,
site_config.templates.internal,
display_label=config_lib.DISPLAY_LABEL_MST_ANDROID_PFQ,
android_package='android-container-master-arc-dev',
android_import_branch=constants.ANDROID_MST_BUILD_BRANCH,
)
# Mixin for masters.
site_config.AddTemplate(
'master_android_pfq_mixin',
site_config.templates.internal,
site_config.templates.no_vmtest_builder,
boards=[],
master=True,
push_overlays=constants.BOTH_OVERLAYS,
)
def AndroidPfqBuilders(site_config, boards_dict, ge_build_config):
"""Create all build configs associated with the Android PFQ.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
# Android MST master.
mst_master_config = site_config.Add(
constants.MST_ANDROID_PFQ_MASTER,
site_config.templates.mst_android_pfq,
site_config.templates.master_android_pfq_mixin,
schedule='with 150m interval',
)
_mst_hwtest_boards = frozenset([])
_mst_hwtest_skylab_boards = frozenset([])
_mst_no_hwtest_boards = frozenset([])
_mst_no_hwtest_experimental_boards = frozenset([])
_mst_vmtest_boards = frozenset([
'betty-arcmaster',
])
# Android QT master.
qt_master_config = site_config.Add(
constants.QT_ANDROID_PFQ_MASTER,
site_config.templates.qt_android_pfq,
site_config.templates.master_android_pfq_mixin,
schedule='with 150m interval',
)
_qt_hwtest_boards = frozenset([
'eve-arcnext',
])
_qt_hwtest_skylab_boards = frozenset([
'eve-arcnext',
])
_qt_no_hwtest_boards = frozenset([])
_qt_no_hwtest_experimental_boards = frozenset([])
_qt_vmtest_boards = frozenset([
'betty-qt-arc',
])
# Android PI master.
pi_master_config = site_config.Add(
constants.PI_ANDROID_PFQ_MASTER,
site_config.templates.pi_android_pfq,
site_config.templates.master_android_pfq_mixin,
schedule='with 60m interval',
)
_pi_no_hwtest_boards = frozenset([
'caroline',
'coral',
'reef',
])
_pi_no_hwtest_experimental_boards = frozenset([])
_pi_hwtest_boards = frozenset([
'caroline-arcnext',
'eve',
'grunt',
'kevin',
])
_pi_hwtest_experimental_boards = frozenset([])
_pi_hwtest_skylab_boards = frozenset([
'caroline-arcnext',
'eve',
'grunt',
'kevin',
])
_pi_vmtest_boards = frozenset([
'betty-arcnext',
'betty-pi-arc',
])
_pi_vmtest_experimental_boards = frozenset([
])
# Android VM PI master.
vmpi_master_config = site_config.Add(
constants.VMPI_ANDROID_PFQ_MASTER,
site_config.templates.vmpi_android_pfq,
site_config.templates.master_android_pfq_mixin,
schedule='with 1440m interval',
)
_vmpi_no_hwtest_boards = frozenset([
'eve-arcvm',
])
_vmpi_no_hwtest_experimental_boards = frozenset([])
_vmpi_hwtest_boards = frozenset([])
_vmpi_hwtest_experimental_boards = frozenset([])
_vmpi_vmtest_boards = frozenset([])
_vmpi_vmtest_experimental_boards = frozenset([])
# Android NYC master.
nyc_master_config = site_config.Add(
constants.NYC_ANDROID_PFQ_MASTER,
site_config.templates.nyc_android_pfq,
site_config.templates.master_android_pfq_mixin,
schedule='with 150m interval',
)
_nyc_hwtest_boards = frozenset([
'veyron_jerry',
'gandof',
])
_nyc_hwtest_skylab_boards = frozenset([])
_nyc_no_hwtest_boards = frozenset([
'veyron_tiger',
])
_nyc_no_hwtest_experimental_boards = frozenset([])
_nyc_vmtest_boards = frozenset([
'betty',
'betty-arc64',
])
# Android MST slaves.
mst_master_config.AddSlaves(
site_config.AddForBoards(
'mst-android-pfq',
_mst_hwtest_boards - _mst_hwtest_skylab_boards,
board_configs,
site_config.templates.mst_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_hwtest_skylab_boards,
board_configs,
site_config.templates.mst_android_pfq,
enable_skylab_hw_tests=True,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_no_hwtest_boards,
board_configs,
site_config.templates.mst_android_pfq,
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_no_hwtest_experimental_boards,
board_configs,
site_config.templates.mst_android_pfq,
important=False,
) +
site_config.AddForBoards(
'mst-android-pfq',
_mst_vmtest_boards,
board_configs,
site_config.templates.mst_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
)
)
# Android QT slaves.
qt_master_config.AddSlaves(
site_config.AddForBoards(
'qt-android-pfq',
_qt_hwtest_boards - _qt_hwtest_skylab_boards,
board_configs,
site_config.templates.qt_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'qt-android-pfq',
_qt_hwtest_skylab_boards,
board_configs,
site_config.templates.qt_android_pfq,
enable_skylab_hw_tests=True,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'qt-android-pfq',
_qt_no_hwtest_boards,
board_configs,
site_config.templates.qt_android_pfq,
) +
site_config.AddForBoards(
'qt-android-pfq',
_qt_no_hwtest_experimental_boards,
board_configs,
site_config.templates.qt_android_pfq,
important=False,
) +
site_config.AddForBoards(
'qt-android-pfq',
_qt_vmtest_boards,
board_configs,
site_config.templates.qt_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
)
)
# Android PI slaves.
pi_master_config.AddSlaves(
site_config.AddForBoards(
'pi-android-pfq',
_pi_hwtest_boards - _pi_hwtest_skylab_boards,
board_configs,
site_config.templates.pi_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_hwtest_skylab_boards,
board_configs,
site_config.templates.pi_android_pfq,
enable_skylab_hw_tests=True,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_no_hwtest_boards,
board_configs,
site_config.templates.pi_android_pfq,
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_no_hwtest_experimental_boards,
board_configs,
site_config.templates.pi_android_pfq,
important=False,
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_hwtest_experimental_boards,
board_configs,
site_config.templates.pi_android_pfq,
important=False,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_vmtest_boards,
board_configs,
site_config.templates.pi_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
) +
site_config.AddForBoards(
'pi-android-pfq',
_pi_vmtest_experimental_boards,
board_configs,
site_config.templates.pi_android_pfq,
important=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
)
)
# Android VM PI slaves.
vmpi_master_config.AddSlaves(
site_config.AddForBoards(
'vmpi-android-pfq',
_vmpi_hwtest_boards,
board_configs,
site_config.templates.vmpi_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'vmpi-android-pfq',
_vmpi_no_hwtest_boards,
board_configs,
site_config.templates.vmpi_android_pfq,
) +
site_config.AddForBoards(
'vmpi-android-pfq',
_vmpi_no_hwtest_experimental_boards,
board_configs,
site_config.templates.vmpi_android_pfq,
important=False,
) +
site_config.AddForBoards(
'vmpi-android-pfq',
_vmpi_hwtest_experimental_boards,
board_configs,
site_config.templates.vmpi_android_pfq,
important=False,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'vmpi-android-pfq',
_vmpi_vmtest_boards,
board_configs,
site_config.templates.vmpi_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
) +
site_config.AddForBoards(
'vmpi-android-pfq',
_vmpi_vmtest_experimental_boards,
board_configs,
site_config.templates.vmpi_android_pfq,
important=False,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke')],
)
)
# Android NYC slaves.
nyc_master_config.AddSlaves(
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_hwtest_boards - _nyc_hwtest_skylab_boards,
board_configs,
site_config.templates.nyc_android_pfq,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_hwtest_skylab_boards,
board_configs,
site_config.templates.nyc_android_pfq,
enable_skylab_hw_tests=True,
hw_tests=hw_test_list.SharedPoolAndroidPFQ(),
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_no_hwtest_boards,
board_configs,
site_config.templates.nyc_android_pfq,
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_no_hwtest_experimental_boards,
board_configs,
site_config.templates.nyc_android_pfq,
important=False,
) +
site_config.AddForBoards(
'nyc-android-pfq',
_nyc_vmtest_boards,
board_configs,
site_config.templates.nyc_android_pfq,
vm_tests=[config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'),],
)
)
def FullBuilders(site_config, boards_dict, ge_build_config):
"""Create all full builders.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
active_builders = frozenset([
'amd64-generic',
'arm-generic',
'arm64-generic',
'kevin',
'kevin64',
'oak',
'tael',
'tatl',
])
# Move the following builders to active_builders once they are consistently
# green.
unstable_builders = frozenset([
'lakitu', # TODO: Re-enable after crbug.com/919630 resolved.
])
external_board_configs = CreateBoardConfigs(
site_config, boards_dict, ge_build_config)
site_config.AddForBoards(
config_lib.CONFIG_TYPE_FULL,
boards_dict['all_full_boards'],
external_board_configs,
site_config.templates.full,
site_config.templates.build_external_chrome,
internal=False,
manifest_repo_url=config_lib.GetSiteParams().MANIFEST_URL,
overlays=constants.PUBLIC_OVERLAYS,
prebuilts=constants.PUBLIC)
master_config = site_config.Add(
'master-full',
site_config.templates.full,
site_config.templates.internal,
site_config.templates.build_external_chrome,
boards=[],
master=True,
manifest_version=True,
overlays=constants.PUBLIC_OVERLAYS,
slave_configs=[],
schedule='0 */3 * * *',
)
master_config.AddSlaves(
site_config.ApplyForBoards(
config_lib.CONFIG_TYPE_FULL,
active_builders,
manifest_version=True,
)
)
master_config.AddSlaves(
site_config.ApplyForBoards(
config_lib.CONFIG_TYPE_FULL,
unstable_builders,
manifest_version=True,
important=False,
)
)
def CqBuilders(site_config, boards_dict, ge_build_config):
"""Create all CQ build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
site_config.AddTemplate(
'cq_luci_slave',
build_affinity=False,
)
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
hw_test_list = HWTestList(ge_build_config)
_separate_boards = boards_dict['all_boards']
_unified_builds = config_lib.GetUnifiedBuildConfigAllBuilds(ge_build_config)
_unified_board_names = set([b[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]
for b in _unified_builds])
_paladin_boards = _separate_boards | _unified_board_names
# List of paladin boards where the regular paladin config is important.
_paladin_important_boards = frozenset([
'amd64-generic',
'arm-generic',
'arm64-generic',
'auron_paine',
'auron_yuna',
'beaglebone',
'betty',
'betty-arc64',
'betty-arcnext',
'betty-pi-arc',
'bob',
'capri',
'capri-zfpga',
'caroline',
'caroline-arcnext',
'cave',
'chell',
'cheza',
'cobblepot',
'coral',
'cyan',
'edgar',
'elm',
'eve',
'eve-arcnext',
'fizz',
'fizz-accelerator',
'fizz-moblab',
'flapjack',
'gale',
'gandof',
'glados',
'gonzo',
'guado',
'guado-accelerator',
'hana',
'hatch',
'kalista',
'kevin',
'kevin64',
'kevin-arcnext',
'kip',
'kukui',
'lakitu',
'lakitu-gpu',
'lakitu-st',
'lasilla-ground',
'littlejoe',
'mistral',
'monroe',
'nami',
'nocturne',
'nyan_big',
'nyan_blaze',
'nyan_kitty',
'octavius',
'octopus',
'rammus',
'reef',
'romer',
'sarien',
'samus',
'samus-kernelnext',
'scarlet',
'sentry',
'sludge',
'tael',
'tatl',
'terra',
'tidus',
'veyron_jaq',
'veyron_jerry',
'veyron_mighty',
'veyron_minnie',
'veyron_rialto',
'veyron_speedy',
'veyron_tiger',
'viking',
'whirlwind',
'winky',
'wizpig',
'wooten',
])
# Paladin configs that exist and should be important as soon as they are
# shown to be green. All new paladins should start in this group and get
# promoted to _paladin_important_boards.
#
# A paladin is generally considered good enough for important if it can
# pass the last ~20 build attempts, e.g. the builder page shows mostly green.
# Note that paladins are expected to fail occasionally as they block bad CLs
# from landing, a red paladin from a bad CL in the CQ is a working paladin.
#
# If the device is to be used with HW testing, the standard is higher, and
# the device should be proven to be stable for at least three weeks.
# Generally only PVT/MP stage devices should be used with HW testing due
# to stability requirements for the commit queue. If a EVT/DVT device is
# to be deployed in a HW test capacity as a paladin, it must be approved
# by a member of the associated product team, the test infrastructure team,
# and the CI team.
#
# The definition of what paladins run HW tests are in the
# _paladin_hwtest_assignments table further down this script.
_paladin_new_boards = frozenset([
'betty-kernelnext',
'bubs',
'drallion',
'jacuzzi',
'kefka-kernelnext',
'kumo',
'veyron_jerry-kernelnext',
'veyron_minnie-kernelnext',
'zork',
])
# Paladin configs that exist and should stay as experimental until further
# notice, preferably with a comment indicating why and a bug.
_paladin_experimental_boards = _paladin_new_boards | frozenset([
'atlas', # crbug.com/994783
'moblab-generic-vm', # crbug.com/960998, crbug.com/976297
'grunt', # crbug.com/984614
])
assert not (_paladin_experimental_boards & _paladin_important_boards), (
'Experimental and important paladin board sets must be disjoint.')
_paladin_active = _paladin_important_boards | _paladin_experimental_boards
_paladin_simple_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
'betty-pi-arc',
])
_paladin_devmode_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
'betty-pi-arc',
])
_paladin_cros_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
'betty-pi-arc',
])
_paladin_smoke_vmtest_boards = frozenset([
'betty',
'betty-arc64',
'betty-arcnext',
'betty-pi-arc',
])
_paladin_default_vmtest_boards = frozenset([
'betty',
'betty-arcnext',
'betty-pi-arc',
])
# Jetstream devices run unique hw tests
_paladin_jetstream_hwtest_boards = frozenset([
'whirlwind',
'gale',
'mistral',
])
_paladin_moblab_hwtest_boards = frozenset([
'fizz-moblab',
])
_paladin_chroot_replace_boards = frozenset([])
_paladin_separate_symbols = frozenset([
'amd64-generic',
])
_paladin_bluestreak_hwtest_boards = frozenset([
'guado',
])
_paladin_enable_skylab_hwtest = frozenset([
'auron_paine',
'auron_yuna',
'atlas',
'bob',
'caroline',
'caroline-arcnext',
'cave',
'coral',
'cyan',
'edgar',
'elm',
'eve',
'gale',
'grunt',
'hana',
'kevin',
'kip',
'mistral',
'nocturne',
'nyan_blaze',
'nyan_big',
'nyan_kitty',
'octopus',
'reef',
'scarlet',
'sentry',
'tidus',
'veyron_mighty',
'veyron_minnie',
'veyron_speedy',
'whirlwind',
'winky',
'wizpig',
])
_paladin_separate_unittest_phase = frozenset([
'grunt',
])
### Master paladin (CQ builder).
master_config = site_config.Add(
'master-paladin',
site_config.templates.paladin,
site_config.templates.internal_paladin,
boards=[],
master=True,
binhost_test=True,
push_overlays=constants.BOTH_OVERLAYS,
description='Commit Queue master (all others are slaves)',
schedule='with 2m interval',
)
### Other paladins (CQ builders).
# These are slaves of the master paladin by virtue of matching
# in a few config values (e.g. 'build_type', 'branch', etc). If
# they are not 'important' then they are ignored slaves.
# TODO(mtennant): This master-slave relationship should be specified
# here in the configuration, rather than GetSlavesForMaster().
# Something like the following:
# master_paladin = site_config.AddConfig(internal_paladin, ...)
# master_paladin.AddSlave(site_config.AddConfig(internal_paladin, ...))
for board in _paladin_boards:
assert board in board_configs, '%s not in board_configs' % board
config_name = '%s-%s' % (board, constants.PALADIN_TYPE)
customizations = config_lib.BuildConfig()
base_config = board_configs[board]
if board in _unified_board_names:
for unibuild in _unified_builds:
if board == unibuild[config_lib.CONFIG_TEMPLATE_REFERENCE_BOARD_NAME]:
models = []
for model in unibuild[config_lib.CONFIG_TEMPLATE_MODELS]:
name = model[config_lib.CONFIG_TEMPLATE_MODEL_NAME]
lab_board_name = model[config_lib.CONFIG_TEMPLATE_MODEL_BOARD_NAME]
if (config_lib.CONFIG_TEMPLATE_MODEL_CQ_TEST_ENABLED in model
and model[config_lib.CONFIG_TEMPLATE_MODEL_CQ_TEST_ENABLED]):
models.append(config_lib.ModelTestConfig(
name, lab_board_name, enable_skylab=True))
customizations.update(models=models)
if board in _paladin_enable_skylab_hwtest:
customizations.update(enable_skylab_hw_tests=True)
if board in _paladin_moblab_hwtest_boards:
customizations.update(
hw_tests=[
config_lib.HWTestConfig(
constants.HWTEST_MOBLAB_QUICK_SUITE,
timeout=90*60,
pool=constants.HWTEST_QUOTA_POOL),
config_lib.HWTestConfig(
constants.HWTEST_TAST_CQ_SUITE,
pool=constants.HWTEST_QUOTA_POOL),
],
hw_tests_override=None)
if board in _paladin_jetstream_hwtest_boards:
customizations.update(
hw_tests=[
hw_test_list.DefaultListCQ()[0],
config_lib.HWTestConfig(
constants.HWTEST_JETSTREAM_COMMIT_SUITE,
pool=constants.HWTEST_QUOTA_POOL)
],
hw_tests_override=None)
if board in _paladin_bluestreak_hwtest_boards:
customizations.update(
hw_tests=hw_test_list.BluestreakPoolPreCQ(),
hw_tests_override=hw_test_list.BluestreakPoolPreCQ())
if board in _paladin_experimental_boards:
customizations.update(important=False)
if board in _paladin_chroot_replace_boards:
customizations.update(chroot_replace=True)
if (board in boards_dict['internal_boards']
or board in _unified_board_names):
customizations = customizations.derive(
site_config.templates.internal,
site_config.templates.official_chrome,
manifest=constants.OFFICIAL_MANIFEST)
if board in _paladin_separate_symbols:
customizations.update(separate_debug_symbols=True)
if board not in _paladin_default_vmtest_boards:
vm_tests = []
if board in _paladin_simple_vmtest_boards:
vm_tests.append(
config_lib.VMTestConfig(constants.SIMPLE_AU_TEST_TYPE))
if board in _paladin_cros_vmtest_boards:
vm_tests.append(config_lib.VMTestConfig(constants.CROS_VM_TEST_TYPE))
if board in _paladin_devmode_vmtest_boards:
vm_tests.append(config_lib.VMTestConfig(constants.DEV_MODE_TEST_TYPE))
if board in _paladin_smoke_vmtest_boards:
vm_tests.append(
config_lib.VMTestConfig(constants.VM_SUITE_TEST_TYPE,
test_suite='smoke'))
customizations.update(vm_tests=vm_tests)
if site_config.templates.paladin.vm_tests_override is not None:
# Make sure any new tests are also in override.
override = site_config.templates.paladin.vm_tests_override[:]
for test in vm_tests:
if test not in override:
override.append(test)
customizations.update(vm_tests_override=override)
if base_config.get('internal'):
customizations.update(
description=(site_config.templates.paladin.description +
' (internal)'))
if board in _paladin_active:
customizations.update(
site_config.templates.cq_luci_slave,
)
if board in _paladin_separate_unittest_phase:
customizations.update(unittests=False)
if board in chromeos_boards.lakitu_boards:
customizations.update(
site_config.templates.lakitu_paladin_test_customizations)
config = site_config.Add(
config_name,
site_config.templates.paladin,
customizations,
base_config,
)
if board in _paladin_active:
master_config.AddSlave(config)
# N.B. The ordering of columns here is coupled to the ordering of
# suites returned by DefaultListCQ(). If you change the ordering here,
# you must also change the ordering there.
#
# CAUTION: Only add devices to this table which are known to be stable in
# the HW test lab, even low rates of flake from these devices quickly
# add up and can destabilize the commit queue.
#
# TODO: Fill in any notable gaps in this table. crbug.com/730076
# pylint: disable=bad-continuation, bad-whitespace, line-too-long
_paladin_hwtest_assignments = frozenset([
# bvt-inline bvt-cq bvt-arc bvt-tast-cq family
('winky', 'kip', None, 'winky'), # rambi (BYT)
('nyan_big', 'nyan_kitty', None, 'nyan_big'), # nyan (K1)
('auron_paine', 'tidus', 'auron_yuna', 'auron_paine'), # auron (BDW)
('veyron_mighty', 'veyron_speedy', 'veyron_minnie', 'veyron_mighty'), # pinky (RK3288)
('wizpig', 'edgar', 'cyan', 'wizpig'), # strago (BSW)
('cave', 'sentry', None, 'cave'), # glados (SKL)
('elm', None, 'hana', 'elm'), # oak (MTK8173)
('bob', None, 'kevin', 'bob'), # gru (RK3399)
('reef', None, None, 'reef'), # reef (APL)
('coral', None, None, 'coral'), # coral (APL)
(None, 'eve', 'soraka', 'eve'), # poppy (KBL)
('nocturne', None, None, 'nocturne'), # Nocturne (KBL)
('atlas', None, None, 'atlas'), # Atlas (KBL)
('octopus', None, None, 'octopus'), # Octopus (GLK unibuild)
(None, None, 'caroline-arcnext', 'caroline-arcnext'), # arcnext
('nyan_blaze', None, None, 'nyan_blaze'), # Add for Skylab test
('scarlet' , None, None, 'scarlet'), # scarlet (RK3399 unibuild)
('grunt', None, 'grunt', 'grunt'), # grunt (AMD unibuild)
])
# pylint: enable=bad-continuation, bad-whitespace, line-too-long
sharded_hw_tests = hw_test_list.DefaultListCQ()
# Run provision suite first everywhere.
default_tests = [sharded_hw_tests.pop(0)]
config_default_reset = set()
for board_assignments in _paladin_hwtest_assignments:
assert len(board_assignments) == len(sharded_hw_tests)
for board, suite in zip(board_assignments, sharded_hw_tests):
if board is None:
continue
config_name = '%s-%s' % (board, constants.PALADIN_TYPE)
# Only configurate hw_tests for unified builds if they have specified
# models they want to test against (based on lab provisioning)
if (board in _unified_board_names and
not site_config[config_name]['models']):
continue
if config_name not in config_default_reset:
site_config[config_name]['hw_tests'] = default_tests[:]
config_default_reset.add(config_name)
site_config[config_name]['hw_tests'] += [suite]
#
# Paladins with alternative configs.
#
site_config.AddForBoards(
'nowithdebug-paladin',
['amd64-generic'],
board_configs,
site_config.templates.paladin,
site_config.templates.internal_nowithdebug_paladin,
)
master_config.AddSlaves([
site_config.Add(
'chell-nowithdebug-paladin',
site_config.templates.paladin,
site_config.templates.internal_nowithdebug_paladin,
site_config.templates.cq_luci_slave,
boards=['chell'],
),
site_config.Add(
'incremental-paladin',
site_config.templates.paladin,
site_config.templates.internal_paladin,
site_config.templates.no_hwtest_builder,
site_config.templates.cq_luci_slave,
boards=['reef'],
build_before_patching=True,
compilecheck=True,
unittests=False,
important=False, # crbug.com/984006
),
])
# Used for builders which build completely from source except Chrome.
# These boards pass with -clang-clean CFLAG, so ensure they stay that way.
master_config.AddSlaves(
site_config.AddForBoards(
'full-compile-paladin',
['nyan_kitty'],
board_configs,
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
site_config.templates.cq_luci_slave,
board_replace=True,
chrome_sdk=False,
compilecheck=True,
cpe_export=False,
debug_symbols=False,
unittests=False,
upload_hw_test_artifacts=False,
vm_tests=[],
)
)
master_config.AddSlaves(
site_config.AddForBoards(
'unittest-only-paladin',
list(_paladin_separate_unittest_phase),
board_configs,
site_config.templates.unittest_only_paladin,
site_config.templates.cq_luci_slave,
unittests=True,
)
)
site_config.Add(
'amd64-generic-asan-paladin',
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
board_configs['amd64-generic'],
site_config.templates.asan,
description='Paladin build with Address Sanitizer (Clang)',
)
site_config.Add(
'betty-asan-paladin',
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
board_configs['betty'],
site_config.templates.asan,
description='Paladin build with Address Sanitizer (Clang)',
)
site_config.Add(
'amd64-generic-ubsan-paladin',
site_config.templates.paladin,
site_config.templates.no_hwtest_builder,
board_configs['amd64-generic'],
site_config.templates.ubsan,
description='Paladin build with Undefined Behavior Sanitizer (Clang)',
)
def IncrementalBuilders(site_config, boards_dict, ge_build_config):
"""Create all incremental build configs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
site_config.AddTemplate(
'incremental_affinity',
build_affinity=True,
luci_builder=config_lib.LUCI_BUILDER_INCREMENTAL,
)
master_config = site_config.Add(
'master-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
boards=[],
master=True,
manifest_version=True,
slave_configs=[],
schedule='with 10m interval',
)
# Build external source, for an internal board.
master_config.AddSlave(
site_config.Add(
'amd64-generic-incremental',
site_config.templates.incremental,
site_config.templates.incremental_affinity,
board_configs['amd64-generic'],
manifest_version=True,
)
)
master_config.AddSlave(
site_config.Add(
'betty-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.incremental_affinity,
boards=['betty'],
manifest_version=True,
)
)
master_config.AddSlave(
site_config.Add(
'chell-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.incremental_affinity,
boards=['chell'],
manifest_version=True,
)
)
master_config.AddSlave(
site_config.Add(
'lakitu-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.incremental_affinity,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu'],
manifest_version=True,
)
)
#
# Available, but not regularly scheduled.
#
site_config.Add(
'x32-generic-incremental',
site_config.templates.incremental,
board_configs['x32-generic'],
)
site_config.Add(
'beaglebone-incremental',
site_config.templates.incremental,
site_config.templates.beaglebone,
boards=['beaglebone'],
description='Incremental Beaglebone Builder',
)
site_config.Add(
'kumo-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['kumo'],
)
site_config.Add(
'lakitu-gpu-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu-gpu'],
)
site_config.Add(
'lakitu-st-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu-st'],
)
site_config.Add(
'lakitu_next-incremental',
site_config.templates.incremental,
site_config.templates.internal_incremental,
site_config.templates.lakitu_notification_emails,
board_configs['lakitu_next'],
)
def ReleaseAfdoBuilders(site_config, boards_dict, ge_build_config):
"""Create AFDO Performance tryjobs.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
# Now generate generic release-afdo configs if we haven't created anything
# more specific above already. release-afdo configs are builders that do AFDO
# profile collection and optimization in the same builder. Used by developers
# that want to measure performance changes caused by their changes.
for board in boards_dict['all_release_boards']:
base = board_configs[board]
config_name = '%s-%s' % (board, config_lib.CONFIG_TYPE_RELEASE_AFDO)
if config_name in site_config:
continue
generate_config_name = (
'%s-%s-%s' % (board,
config_lib.CONFIG_TYPE_RELEASE_AFDO,
'generate'))
use_config_name = '%s-%s-%s' % (board,
config_lib.CONFIG_TYPE_RELEASE_AFDO,
'use')
# We can't use AFDO data if afdo_use is disabled for this board.
if not base.get('afdo_use', True):
continue
site_config.AddGroup(
config_name,
site_config.Add(
generate_config_name,
site_config.templates.release_afdo_generate,
base
),
site_config.Add(
use_config_name,
site_config.templates.release_afdo_use,
base
),
)
def InformationalBuilders(site_config, boards_dict, ge_build_config):
"""Create all informational builders.
We have a number of informational builders that are built, but whose output is
not directly used for anything other than reporting success or failure.
Args:
site_config: config_lib.SiteConfig to be modified by adding templates
and configs.
boards_dict: A dict mapping board types to board name collections.
ge_build_config: Dictionary containing the decoded GE configuration file.
"""
external_board_configs = CreateBoardConfigs(
site_config, boards_dict, ge_build_config)
internal_board_configs = CreateInternalBoardConfigs(
site_config, boards_dict, ge_build_config)
_chrome_boards = frozenset(
board for board, config in internal_board_configs.items()
if config.get('sync_chrome', True))
hw_test_list = HWTestList(ge_build_config)
_chrome_informational_hwtest_boards = frozenset([
'caroline',
'eve',
'reks',
'veyron_minnie',
])
_chrome_informational_swarming_boards = frozenset([
'eve',
])
# We have to mark all autogenerated PFQs as not important so the master
# does not wait for them. https://crbug.com/386214
# If you want an important PFQ, you'll have to declare it yourself.
informational_boards = (
(boards_dict['all_release_boards'] & _chrome_boards))
_tot_chrome_pfq_informational_board_configs = UpdateBoardConfigs(
internal_board_configs,
_chrome_informational_hwtest_boards,
hw_tests=hw_test_list.DefaultListChromePFQInformational(
pool=constants.HWTEST_CONTINUOUS_POOL))
_tot_chrome_pfq_informational_board_configs = UpdateBoardConfigs(
_tot_chrome_pfq_informational_board_configs,
_chrome_informational_swarming_boards)
site_config.AddForBoards(
'tot-chrome-pfq-informational',
informational_boards,
_tot_chrome_pfq_informational_board_configs,
site_config.templates.chrome_pfq_informational)
site_config.Add(
'amd64-generic-asan',
site_config.templates.asan,