blob: f1430432ece35a95f2b0e854f6e365bc54558220 [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""The cros chrome-sdk command for the simple chrome workflow."""
from __future__ import print_function
import argparse
import collections
import contextlib
import glob
import json
import os
import re
from chromite.cbuildbot import archive_lib
from chromite.cli import command
from chromite.lib import cache
from chromite.lib import config_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import gs
from chromite.lib import memoize
from chromite.lib import osutils
from chromite.lib import path_util
from chromite.lib import retry_util
from gn_helpers import gn_helpers
COMMAND_NAME = 'chrome-sdk'
def Log(*args, **kwargs):
"""Conditional logging.
silent: If set to True, then logs with level DEBUG. logs with level INFO
otherwise. Defaults to False.
silent = kwargs.pop('silent', False)
level = logging.DEBUG if silent else logging.INFO
logging.log(level, *args, **kwargs)
class NoChromiumSrcDir(Exception):
"""Error thrown when no chromium src dir is found."""
def __init__(self, path):
Exception.__init__(self, 'No chromium src dir found in: %s' % (path))
class MissingLKGMFile(Exception):
"""Error thrown when we cannot get the version from CHROMEOS_LKGM."""
def __init__(self, path):
Exception.__init__(self, 'Cannot parse CHROMEOS_LKGM file: %s' % (path))
class MissingSDK(Exception):
"""Error thrown when we cannot find an SDK."""
def __init__(self, board, version=None):
msg = 'Cannot find SDK for %r' % (board,)
if version is not None:
msg += ' with version %s' % (version,)
Exception.__init__(self, msg)
class SDKFetcher(object):
"""Functionality for fetching an SDK environment.
For the version of ChromeOS specified, the class downloads and caches
SDK components.
SDKContext = collections.namedtuple(
'SDKContext', ['version', 'target_tc', 'key_map'])
TARBALL_CACHE = 'tarballs'
MISC_CACHE = 'misc'
TARGET_TOOLCHAIN_KEY = 'target_toolchain'
QEMU_BIN_PATH = 'app-emulation/qemu'
SEABIOS_BIN_PATH = 'sys-firmware/seabios'
def __init__(self, cache_dir, board, clear_cache=False, chrome_src=None,
sdk_path=None, toolchain_path=None, silent=False,
use_external_config=None, require_exact_version=False):
"""Initialize the class.
cache_dir: The toplevel cache dir to use.
board: The board to manage the SDK for.
clear_cache: Clears the sdk cache during __init__.
chrome_src: The location of the chrome checkout. If unspecified, the
cwd is presumed to be within a chrome checkout.
sdk_path: The path (whether a local directory or a gs:// path) to fetch
SDK components from.
toolchain_path: The path (whether a local directory or a gs:// path) to
fetch toolchain components from.
silent: If set, the fetcher prints less output.
use_external_config: When identifying the configuration for a board,
force usage of the external configuration if both external and internal
are available.
require_exact_version: Use exact SDK version only.
site_config = config_lib.GetConfig()
self.cache_base = os.path.join(cache_dir, COMMAND_NAME)
if clear_cache:
logging.warning('Clearing the SDK cache.')
osutils.RmDir(self.cache_base, ignore_missing=True)
self.tarball_cache = cache.TarballCache(
os.path.join(self.cache_base, self.TARBALL_CACHE))
self.misc_cache = cache.DiskCache(
os.path.join(self.cache_base, self.MISC_CACHE))
self.board = board
self.config = site_config.FindCanonicalConfigForBoard(
board, allow_internal=not use_external_config)
self.gs_base = archive_lib.GetBaseUploadURI(self.config)
self.clear_cache = clear_cache
self.chrome_src = chrome_src
self.sdk_path = sdk_path
self.toolchain_path = toolchain_path
self.require_exact_version = require_exact_version
self.silent = silent
# For external configs, there is no need to run 'gsutil config', because
# the necessary files are all accessible to anonymous users.
internal = self.config['internal']
self.gs_ctx = gs.GSContext(cache_dir=cache_dir, init_boto=internal)
if self.sdk_path is None:
self.sdk_path = os.environ.get(self.SDK_PATH_ENV)
if self.toolchain_path is None:
self.toolchain_path = 'gs://%s' % constants.SDK_GS_BUCKET
def _RemoveOldCacheDir(self):
"""Deletes old cache directory."""
checkout = path_util.DetermineCheckout(os.getcwd())
if checkout.type == path_util.CHECKOUT_TYPE_GCLIENT:
old_path = os.path.join(checkout.root, path_util.OLD_CHROME_CACHE_DIR)
osutils.RmDir(old_path, ignore_missing=True)
def _UpdateTarball(self, url, ref):
"""Worker function to fetch tarballs"""
with osutils.TempDir(base_dir=self.tarball_cache.staging_dir) as tempdir:
local_path = os.path.join(tempdir, os.path.basename(url))
Log('SDK: Fetching %s', url, silent=self.silent)
self.gs_ctx.Copy(url, tempdir, debug_level=logging.DEBUG)
ref.SetDefault(local_path, lock=True)
def _GetMetadata(self, version):
"""Return metadata (in the form of a dict) for a given version."""
raw_json = None
version_base = self._GetVersionGSBase(version)
with self.misc_cache.Lookup(
self._GetCacheKeyForComponent(version, constants.METADATA_JSON)) as ref:
if ref.Exists(lock=True):
raw_json = osutils.ReadFile(ref.path)
metadata_path = os.path.join(version_base, constants.METADATA_JSON)
partial_metadata_path = os.path.join(version_base,
raw_json = self.gs_ctx.Cat(metadata_path,
except gs.GSNoSuchKey:'Could not read %s, falling back to %s',
metadata_path, partial_metadata_path)
raw_json = self.gs_ctx.Cat(partial_metadata_path,
return json.loads(raw_json)
def GetChromeLKGM(chrome_src_dir=None):
"""Get ChromeOS LKGM checked into the Chrome tree.
chrome_src_dir: chrome source directory.
Version number in format '10171.0.0'.
if not chrome_src_dir:
chrome_src_dir = path_util.DetermineCheckout(os.getcwd()).chrome_src_dir
if not chrome_src_dir:
return None
lkgm_file = os.path.join(chrome_src_dir, constants.PATH_TO_CHROME_LKGM)
version = osutils.ReadFile(lkgm_file).rstrip()
logging.debug('Read LKGM version from %s: %s', lkgm_file, version)
return version
def _GetSDKVersion(self, version):
"""Get SDK version from metadata.
version: LKGM version, e.g. 12345.0.0
sdk_version, e.g. 2018.06.04.200410
return self._GetMetadata(version)['sdk-version']
def _GetManifest(self, version):
"""Get the build manifest from the cache, downloading it if necessary.
version: LKGM version, e.g. 12345.0.0
build manifest as a python dictionary. The build manifest contains build
versions for packages built by the SDK builder.
with self.misc_cache.Lookup(('manifest', self.board, version)) as ref:
if ref.Exists(lock=True):
manifest = osutils.ReadFile(ref.path)
manifest_path = gs.GetGsURL(
suburl='cros-sdk-%s.tar.xz.Manifest' % self._GetSDKVersion(version),
manifest = self.gs_ctx.Cat(manifest_path)
return json.loads(manifest)
def _GetBinPackageGSPath(self, version, key):
"""Get google storage path of prebuilt binary package.
version: LKGM version, e.g. 12345.0.0
key: key in build manifest, for e.g. 'app-emulation/qemu'
GS path, for e.g. gs://chromeos-prebuilt/board/amd64-host/
package_version = self._GetManifest(version)['packages'][key][0][0]
return gs.GetGsURL(
suburl='board/amd64-host/chroot-%s/packages/%s-%s.tbz2' %
(self._GetSDKVersion(version), key, package_version),
def _GetTarballCachePath(self, version, key):
"""Get a path in the tarball cache.
version: LKGM version, e.g. 12345.0.0
key: tarball key, for e.g. 'app-emulation/qemu'
cache_key = self._GetCacheKeyForComponent(version, key)
with self.tarball_cache.Lookup(cache_key) as ref:
if ref.Exists(lock=True):
return ref.path
logging.warning('%s not found.', key)
return None
def _FinalizePackages(self, version):
"""Finalize downloaded packages.
Fix broken seabios symlinks in the qemu package.
version: LKGM version, e.g. 12345.0.0
def _CreateSeabiosFWSymlinks(self, version):
"""Create Seabios firmware symlinks.
tarballs/<board>+<version>+app-emulation/qemu/usr/share/qemu/ has a number
of broken symlinks, for example: bios.bin -> ../seabios/bios.bin
bios.bin is in the seabios package at <cache>/seabios/usr/share/seabios/
To resolve these symlinks, we create symlinks from
<cache>+sys-firmware/seabios/usr/share/* to
version: LKGM version, e.g. 12345.0.0
qemu_bin_path = self._GetTarballCachePath(version, self.QEMU_BIN_PATH)
seabios_bin_path = self._GetTarballCachePath(version, self.SEABIOS_BIN_PATH)
if not qemu_bin_path or not seabios_bin_path:
logging.warning('Could not create Seabios firmware links.')
# Symlink the directories in seabios/usr/share/* to qemu/usr/share/.
share_dir = 'usr/share'
seabios_share_dir = os.path.join(seabios_bin_path, share_dir)
qemu_share_dir = os.path.join(qemu_bin_path, share_dir)
for seabios_dir in os.listdir(seabios_share_dir):
src_dir = os.path.relpath(
os.path.join(seabios_share_dir, seabios_dir), qemu_share_dir)
target_dir = os.path.join(qemu_share_dir, seabios_dir)
if not os.path.exists(target_dir):
os.symlink(src_dir, target_dir)
def _GetFullVersionFromStorage(self, version_file):
"""Cat |version_file| in google storage.
version_file: google storage path of the version file.
Version number in the format 'R30-3929.0.0' or None.
# If the version doesn't exist in google storage,
# which isn't unlikely, don't waste time on retries.
full_version = self.gs_ctx.Cat(version_file, retries=0)
assert full_version.startswith('R')
return full_version
except (gs.GSNoSuchKey, gs.GSCommandError):
return None
def _GetFullVersionFromRecentLatest(self, version):
"""Gets the full version number from a recent LATEST- file.
If LATEST-{version} does not exist, we need to look for a recent
LATEST- file to get a valid full version from.
version: The version number to look backwards from. If version is not a
canary version (ending in .0.0), returns None.
Version number in the format 'R30-3929.0.0' or None.
# If version does not end in .0.0 it is not a canary so fail.
if not version.endswith('.0.0'):
return None
version_base = int(version.split('.')[0])
version_base_min = version_base - self.VERSIONS_TO_CONSIDER
for v in xrange(version_base - 1, version_base_min, -1):
version_file = '%s/LATEST-%d.0.0' % (self.gs_base, v)'Trying: %s', version_file)
full_version = self._GetFullVersionFromStorage(version_file)
if full_version is not None:
'Using cros version from most recent LATEST file: %s -> %s',
version_file, full_version)
return full_version
logging.warning('No recent LATEST file found from %d.0.0 to %d.0.0: ',
version_base_min, version_base)
return None
def _GetFullVersionFromLatest(self, version):
"""Gets the full version number from the LATEST-{version} file.
version: The version number or branch to look at.
Version number in the format 'R30-3929.0.0' or None.
version_file = '%s/LATEST-%s' % (self.gs_base, version)
full_version = self._GetFullVersionFromStorage(version_file)
if full_version is None and not self.require_exact_version:
logging.warning('No LATEST file matching SDK version %s', version)
return self._GetFullVersionFromRecentLatest(version)
return full_version
def GetDefaultVersion(self):
"""Get the default SDK version to use.
If we are in an existing SDK shell, the default version will just be
the current version. Otherwise, we will try to calculate the
appropriate version to use based on the checkout.
if os.environ.get(self.SDK_BOARD_ENV) == self.board:
sdk_version = os.environ.get(self.SDK_VERSION_ENV)
if sdk_version is not None:
return sdk_version
with self.misc_cache.Lookup((self.board, 'latest')) as ref:
if ref.Exists(lock=True):
version = osutils.ReadFile(ref.path).strip()
# Deal with the old version format.
if version.startswith('R'):
version = version.split('-')[1]
return version
return None
def _SetDefaultVersion(self, version):
"""Set the new default version."""
with self.misc_cache.Lookup((self.board, 'latest')) as ref:
def UpdateDefaultVersion(self):
"""Update the version that we default to using.
A tuple of the form (version, updated), where |version| is the
version number in the format '3929.0.0', and |updated| indicates
whether the version was indeed updated.
checkout_dir = self.chrome_src if self.chrome_src else os.getcwd()
checkout = path_util.DetermineCheckout(checkout_dir)
current = self.GetDefaultVersion() or '0'
if not checkout.chrome_src_dir:
raise NoChromiumSrcDir(checkout_dir)
target = self.GetChromeLKGM(checkout.chrome_src_dir)
if target is None:
raise MissingLKGMFile(checkout.chrome_src_dir)
return target, target != current
def GetFullVersion(self, version):
"""Add the release branch and build number to a ChromeOS platform version.
This will specify where you can get the latest build for the given version
for the current board.
version: A ChromeOS platform number of the form XXXX.XX.XX, i.e.,
3918.0.0. If a full version is provided, it will be returned unmodified.
The version with release branch and build number added, as needed. E.g.
if version.startswith('R'):
return version
with self.misc_cache.Lookup(('full-version', self.board, version)) as ref:
if ref.Exists(lock=True):
return osutils.ReadFile(ref.path).strip()
full_version = self._GetFullVersionFromLatest(version)
if full_version is None:
raise MissingSDK(self.board, version)
return full_version
def _GetVersionGSBase(self, version):
"""The base path of the SDK for a particular version."""
if self.sdk_path is not None:
return self.sdk_path
full_version = self.GetFullVersion(version)
return os.path.join(self.gs_base, full_version)
def _GetCacheKeyForComponent(self, version, component):
"""Builds the cache key tuple for an SDK component."""
version_section = version
if self.sdk_path is not None:
version_section = self.sdk_path.replace('/', '__').replace(':', '__')
return (self.board, version_section, component)
def Prepare(self, components, version=None, target_tc=None,
"""Ensures the components of an SDK exist and are read-locked.
For a given SDK version, pulls down missing components, and provides a
context where the components are read-locked, which prevents the cache from
deleting them during its purge operations.
If both target_tc and toolchain_url arguments are provided, then this
does not download metadata.json for the given version. Otherwise, this
function requires metadata.json for the given version to exist.
gs_ctx: GSContext object.
components: A list of specific components(tarballs) to prepare.
version: The version to prepare. If not set, uses the version returned by
GetDefaultVersion(). If there is no default version set (this is the
first time we are being executed), then we update the default version.
target_tc: Target toolchain name to use, e.g. x86_64-cros-linux-gnu
toolchain_url: Format pattern for path to fetch toolchain from,
e.g. 2014/04/%(target)s-2014.04.23.220740.tar.xz
An SDKFetcher.SDKContext namedtuple object. The attributes of the
object are:
version: The version that was prepared.
target_tc: Target toolchain name.
key_map: Dictionary that contains CacheReference objects for the SDK
artifacts, indexed by cache key.
if version is None and self.sdk_path is None:
version = self.GetDefaultVersion()
if version is None:
version, _ = self.UpdateDefaultVersion()
components = list(components)
key_map = {}
fetch_urls = {}
if not target_tc or not toolchain_url:
metadata = self._GetMetadata(version)
target_tc = target_tc or metadata['toolchain-tuple'][0]
toolchain_url = toolchain_url or metadata['toolchain-url']
# Fetch toolchains from separate location.
if self.TARGET_TOOLCHAIN_KEY in components:
fetch_urls[self.TARGET_TOOLCHAIN_KEY] = os.path.join(
self.toolchain_path, toolchain_url % {'target': target_tc})
# Also fetch QEMU binary if VM_IMAGE_TAR is specified.
if constants.VM_IMAGE_TAR in components:
qemu_bin_path = self._GetBinPackageGSPath(version, self.QEMU_BIN_PATH)
seabios_bin_path = self._GetBinPackageGSPath(version,
if qemu_bin_path and seabios_bin_path:
fetch_urls[self.QEMU_BIN_PATH] = qemu_bin_path
fetch_urls[self.SEABIOS_BIN_PATH] = seabios_bin_path
logging.warning('Failed to find QEMU/Seabios binaries to download.')
version_base = self._GetVersionGSBase(version)
fetch_urls.update((t, os.path.join(version_base, t)) for t in components)
for key, url in fetch_urls.iteritems():
cache_key = self._GetCacheKeyForComponent(version, key)
ref = self.tarball_cache.Lookup(cache_key)
key_map[key] = ref
if not ref.Exists(lock=True):
# TODO(rcui): Parallelize this. Requires acquiring locks *before*
# generating worker processes; therefore the functionality needs to
# be moved into the DiskCache class itself -
# i.e.,DiskCache.ParallelSetDefault().
self._UpdateTarball(url, ref)
except gs.GSNoSuchKey:
if key == constants.VM_IMAGE_TAR:
'No VM available for board %s. '
'Please try a different board, e.g. amd64-generic.',
ctx_version = version
if self.sdk_path is not None:
ctx_version = CUSTOM_VERSION
yield self.SDKContext(ctx_version, target_tc, key_map)
# TODO(rcui): Move to using cros_build_lib.ContextManagerStack()
cros_build_lib.SafeRun([ref.Release for ref in key_map.itervalues()])
class GomaError(Exception):
"""Indicates error with setting up Goma."""
class ChromeSDKCommand(command.CliCommand):
"""Set up an environment for building Chrome on Chrome OS.
Pulls down SDK components for building and testing Chrome for Chrome OS,
sets up the environment for building Chrome, and runs a command in the
environment, starting a bash session if no command is specified.
The bash session environment is set up by a user-configurable rc file located
at ~/.chromite/chrome_sdk.bashrc.
# Note, this URL is not accessible outside of corp.
_GOMA_TGZ = 'goma-goobuntu.tgz'
_CHROME_CLANG_DIR = 'third_party/llvm-build/Release+Asserts/bin'
_HOST_BINUTILS_DIR = 'third_party/binutils/Linux_x64/Release/bin/'
# Compiler tools.
# Compiler flags.
# Misc settings.
GOMACC_PORT_CMD = ['./gomacc', 'port']
# Override base class property to use cache related commandline options.
use_caching_options = True
def ValidateVersion(version):
"""Ensures that the version arg is potentially valid.
See the argument description for supported version formats.
if (not re.match(r'^[0-9]+\.0\.0$', version) and
not re.match(r'^R[0-9]+-[0-9]+\.[0-9]+\.[0-9]+', version)):
raise argparse.ArgumentTypeError(
'--version should be in the format 1234.0.0 or R56-1234.0.0')
return version
def AddParser(cls, parser):
super(ChromeSDKCommand, cls).AddParser(parser)
'--board', required=True, help='The board SDK to use.')
'--bashrc', type='path',
help='A bashrc file used to set up the SDK shell environment. '
'Defaults to %s.' % constants.CHROME_SDK_BASHRC)
'--chroot', type='path',
help='Path to a ChromeOS chroot to use. If set, '
'<chroot>/build/<board> will be used as the sysroot that Chrome '
'is built against. If chromeos-chrome was built, the build '
'environment from the chroot will also be used. The version shown '
'in the SDK shell prompt will have an asterisk prepended to it.')
'--chrome-src', type='path',
help='Specifies the location of a Chrome src/ directory. Required if '
'running with --clang if not running from a Chrome checkout.')
'--clang', action='store_true', default=False,
help='Sets up the environment for building with clang. For all '
'boards, except X86-32, clang is the default.')
'--cwd', type='path',
help='Specifies a directory to switch to after setting up the SDK '
'shell. Defaults to the current directory.')
'--internal', action='store_true', default=False,
help='Sets up SDK for building official (internal) Chrome '
'Chrome, rather than Chromium.')
'--use-external-config', action='store_true', default=False,
help='Use the external configuration for the specified board, even if '
'an internal configuration is avalable.')
'--sdk-path', type='local_or_gs_path',
help='Provides a path, whether a local directory or a gs:// path, to '
'pull SDK components from.')
'--toolchain-path', type='local_or_gs_path',
help='Provides a path, whether a local directory or a gs:// path, to '
'pull toolchain components from.')
help='Provides extra args to "gn gen". Uses the same format as '
'gn gen, e.g. "foo = true bar = 1".')
'--gn-gen', action='store_true', default=True, dest='gn_gen',
help='Run "gn gen" if is stale.')
'--nogn-gen', action='store_false', dest='gn_gen',
help='Do not run "gn gen", warns if is stale.')
'--nogoma', action='store_false', default=True, dest='goma',
help='Disables Goma in the shell by removing it from the PATH and '
'set use_goma=false to GN_ARGS.')
'--nostart-goma', action='store_false', default=True, dest='start_goma',
help='Skip starting goma and hope somebody else starts goma later.')
'--gomadir', type='path',
help='Use the goma installation at the specified PATH.')
'--version', default=None, type=cls.ValidateVersion,
help='Specify the SDK version to use. This can be a platform version '
'ending in .0.0, e.g. 1234.0.0, in which case the full version '
'will be extracted from the corresponding LATEST file for the '
'specified board. If no LATEST file exists, an older version '
'will be used if available. Alternatively, a full version may be '
'specified, e.g. R56-1234.0.0, in which case that exact version '
'will be used. Defaults to using the version specified in the '
'CHROMEOS_LKGM file in the chromium checkout.')
'--require-exact-version', default=False, action='store_true',
help='Use the exact SDK version; do not attempt to use previous '
'cmd', nargs='*', default=None,
help='The command to execute in the SDK environment. Defaults to '
'starting a bash shell.')
'--download-vm', action='store_true', default=False,
help='Additionally downloads a VM image from cloud storage.')
'--thinlto', action='store_true', default=False,
help='Enable ThinLTO in build.')
'--cfi', action='store_true', default=False,
help='Enable CFI in build.')
'--clear-sdk-cache', action='store_true',
help='Removes everything in the SDK cache before starting.')
group = parser.add_argument_group(
'Metadata Overrides (Advanced)',
description='Provide all of these overrides in order to remove '
'dependencies on metadata.json existence.')
'--target-tc', action='store', default=None,
help='Override target toolchain name, e.g. x86_64-cros-linux-gnu')
'--toolchain-url', action='store', default=None,
help='Override toolchain url format pattern, e.g. '
def __init__(self, options):
super(ChromeSDKCommand, self).__init__(options)
self.board = options.board
# Lazy initialized.
self.sdk = None
# Initialized later based on options passed in.
self.silent = True
def _PS1Prefix(board, version, chroot=None):
"""Returns a string describing the sdk environment for use in PS1."""
chroot_star = '*' if chroot else ''
return '(sdk %s %s%s)' % (board, chroot_star, version)
def _CreatePS1(board, version, chroot=None):
"""Returns PS1 string that sets commandline and xterm window caption.
If a chroot path is set, then indicate we are using the sysroot from there
instead of the stock sysroot by prepending an asterisk to the version.
board: The SDK board.
version: The SDK version.
chroot: The path to the chroot, if set.
current_ps1 = cros_build_lib.RunCommand(
['bash', '-l', '-c', 'echo "$PS1"'], print_cmd=False,
if current_ps1:
current_ps1 = current_ps1[-1]
if not current_ps1:
# Something went wrong, so use a fallback value.
current_ps1 = r'\u@\h \w $ '
ps1_prefix = ChromeSDKCommand._PS1Prefix(board, version, chroot)
return '%s %s' % (ps1_prefix, current_ps1)
def _BuildDir(self):
"""Returns a full path build directory."""
return os.path.join(self.options.chrome_src, 'out_%s' % self.board,
def _FixGoldPath(self, var_contents, toolchain_path):
"""Point to the gold linker in the toolchain tarball.
Accepts an already set environment variable in the form of '<cmd>
-B<gold_path>', and overrides the gold_path to the correct path in the
extracted toolchain tarball.
var_contents: The contents of the environment variable.
toolchain_path: Path to the extracted toolchain tarball contents.
Environment string that has correct gold path.
cmd, _, gold_path = var_contents.partition(' -B')
gold_path = os.path.join(toolchain_path, gold_path.lstrip('/'))
return '%s -B%s' % (cmd, gold_path)
def _UpdateGnArgsIfStale(self, out_dir, build_label, gn_args, board):
"""Runs 'gn gen' if gn args are stale or logs a warning."""
gn_args_file_path = os.path.join(
self.options.chrome_src, out_dir, build_label, '')
if not self._StaleGnArgs(gn_args, gn_args_file_path):
if not self.options.gn_gen:
logging.warning('To update gn args run:')
logging.warning('gn gen out_$SDK_BOARD/Release --args="$GN_ARGS"')
logging.warning('Running gn gen')
['gn', 'gen', 'out_%s/Release' % board,
'--args=%s' % gn_helpers.ToGNString(gn_args)],
def _StaleGnArgs(self, new_gn_args, gn_args_file_path):
"""Returns True if needs to be updated."""
if not os.path.exists(gn_args_file_path):
logging.warning('No file: %s', gn_args_file_path)
return True
old_gn_args = gn_helpers.FromGNArgs(osutils.ReadFile(gn_args_file_path))
if new_gn_args == old_gn_args:
return False
logging.warning('Stale file: %s', gn_args_file_path)
self._LogArgsDiff(old_gn_args, new_gn_args)
return True
def _LogArgsDiff(self, cur_args, new_args):
"""Logs the differences between |cur_args| and |new_args|."""
cur_keys = set(cur_args.keys())
new_keys = set(new_args.keys())
for k in new_keys - cur_keys:'MISSING ARG: %s = %s', k, new_args[k])
for k in cur_keys - new_keys:'EXTRA ARG: %s = %s', k, cur_args[k])
for k in new_keys & cur_keys:
v_cur = cur_args[k]
v_new = new_args[k]
if v_cur != v_new:'MISMATCHED ARG: %s: %s != %s', k, v_cur, v_new)
def _SetupTCEnvironment(self, sdk_ctx, options, env, gn_is_clang):
"""Sets up toolchain-related environment variables."""
target_tc_path = sdk_ctx.key_map[self.sdk.TARGET_TOOLCHAIN_KEY].path
tc_bin_path = os.path.join(target_tc_path, 'bin')
env['PATH'] = '%s:%s' % (tc_bin_path, os.environ['PATH'])
for var in ('CXX', 'CC', 'LD'):
env[var] = self._FixGoldPath(env[var], target_tc_path)
chrome_clang_path = os.path.join(options.chrome_src, self._CHROME_CLANG_DIR)
# Either we are forcing the use of clang through options or GN
# args say we should be using clang.
if options.clang or gn_is_clang:
clang_prepend_flags = ['-Wno-unknown-warning-option']
clang_append_flags = ['-Wno-inline-asm']
env['CC'] = ' '.join([sdk_ctx.target_tc + '-clang'] +
env['CC'].split()[1:] + clang_prepend_flags)
env['CXX'] = ' '.join([sdk_ctx.target_tc + '-clang++'] +
env['CXX'].split()[1:] + clang_prepend_flags)
env['CFLAGS'] = ' '.join(env['CFLAGS'].split() + clang_append_flags)
env['CXXFLAGS'] = ' '.join(env['CXXFLAGS'].split() + clang_append_flags)
env['LD'] = env['CXX']
# Use cros readelf for the target builds. TODO: Delete it after Jan 2019 since
# READELF env variable should already be set,
env.setdefault('READELF', sdk_ctx.target_tc + '-readelf')
# For host compiler, we use the compiler that comes with Chrome
# instead of the target compiler.
env['CC_host'] = os.path.join(chrome_clang_path, 'clang')
env['CXX_host'] = os.path.join(chrome_clang_path, 'clang++')
env['LD_host'] = env['CXX_host']
binutils_path = os.path.join(options.chrome_src, self._HOST_BINUTILS_DIR)
env['AR_host'] = os.path.join(binutils_path, 'ar')
env['NM_host'] = os.path.join(binutils_path, 'nm')
env['READELF_host'] = os.path.join(binutils_path, 'readelf')
def _ModifyPathForGomaBuild(self, compiler, tc_path=None):
"""Modify toolchain path for goma build.
This function relativizes path of -B flag, but absolutize compiler path.
Compiler path will be relativized in build/toolchain/cros/ for
directory independent cache sharing in some distributed build system.
args = []
for i, v in enumerate(compiler.split()):
if v.startswith('-B'):
v = '-B' + os.path.relpath(v[len('-B'):], self._BuildDir())
elif i == 0 and os.path.basename(v) == v:
v = os.path.join(tc_path, 'bin', v)
return ' '.join(args)
def _SetupEnvironment(self, board, sdk_ctx, options, goma_dir=None,
"""Sets environment variables to export to the SDK shell."""
if options.chroot:
sysroot = os.path.join(options.chroot, 'build', board)
if not os.path.isdir(sysroot) and not options.cmd:
logging.warning("Because --chroot is set, expected a sysroot to be at "
"%s, but couldn't find one.", sysroot)
sysroot = sdk_ctx.key_map[constants.CHROME_SYSROOT_TAR].path
environment = os.path.join(sdk_ctx.key_map[constants.CHROME_ENV_TAR].path,
if options.chroot:
# Override with the environment from the chroot if available (i.e.
# build_packages or emerge chromeos-chrome has been run for |board|).
env_path = os.path.join(sysroot, 'var', 'db', 'pkg', 'chromeos-base',
env_glob = glob.glob(env_path)
if len(env_glob) != 1:
logging.warning('Multiple Chrome versions in %s. This can be resolved'
' by running "eclean-$BOARD -d packages". Using'
' environment from: %s', env_path, environment)
elif not os.path.isdir(env_glob[0]):
logging.warning('Environment path not found: %s. Using enviroment from:'
' %s.', env_path, environment)
chroot_env_file = os.path.join(env_glob[0], 'environment.bz2')
if os.path.isfile(chroot_env_file):
# Log a warning here since this is new behavior that is not obvious.
logging.notice('Environment fetched from: %s', chroot_env_file)
# Uncompress enviornment.bz2 to pass to osutils.SourceEnvironment.
chroot_cache = os.path.join(options.cache_dir, COMMAND_NAME, 'chroot')
environment = os.path.join(chroot_cache, 'environment_%s' % board)
cros_build_lib.UncompressFile(chroot_env_file, environment)
env = osutils.SourceEnvironment(environment, self.EBUILD_ENV)
gn_args = gn_helpers.FromGNArgs(env['GN_ARGS'])
self._SetupTCEnvironment(sdk_ctx, options, env, gn_args['is_clang'])
# Add managed components to the PATH.
env['PATH'] = '%s:%s' % (constants.CHROMITE_BIN_DIR, env['PATH'])
env['PATH'] = '%s:%s' % (os.path.dirname(self.sdk.gs_ctx.gsutil_bin),
# Export internally referenced variables.
os.environ[self.sdk.SDK_BOARD_ENV] = board
if options.sdk_path:
os.environ[self.sdk.SDK_PATH_ENV] = options.sdk_path
os.environ[self.sdk.SDK_VERSION_ENV] = sdk_ctx.version
# Add board and sdk version as gn args so that tests can bind them in
# test wrappers generated at compile time.
gn_args['cros_board'] = board
gn_args['cros_sdk_version'] = sdk_ctx.version
# Export the board/version info in a more accessible way, so developers can
# reference them in their chrome_sdk.bashrc files, as well as within the
# chrome-sdk shell.
for var in [self.sdk.SDK_VERSION_ENV, self.sdk.SDK_BOARD_ENV]:
env[var.lstrip('%')] = os.environ[var]
# Export Goma information.
if goma_dir:
env[self.SDK_GOMA_DIR_ENV] = goma_dir
if goma_port:
env[self.SDK_GOMA_PORT_ENV] = goma_port
# SYSROOT is necessary for Goma and the sysroot wrapper.
env['SYSROOT'] = sysroot
gn_args['target_sysroot'] = sysroot
gn_args.pop('pkg_config', None)
# pkg_config only affects the target and comes from the sysroot.
# host_pkg_config is used for programs compiled for use later in the build.
gn_args['host_pkg_config'] = 'pkg-config'
if options.clang:
gn_args['is_clang'] = True
if options.internal:
gn_args['is_chrome_branded'] = True
gn_args['is_official_build'] = True
# TODO( We'd like to use lld for all
# configurations. However, it appears that there's a bug in how it emits
# unwinding info on ARM, and we emit unwind info on builds that aren't
# both official and branded. Until that bug is fixed, our use of lld has
# to be restricted to builds where we don't emit unwinding info.
gn_args['use_lld'] = True
gn_args.pop('is_chrome_branded', None)
gn_args.pop('is_official_build', None)
gn_args.pop('internal_gles2_conform_tests', None)
gn_args.pop('use_lld', None)
# TODO( Hack to keep Chromite rolling. Remove
# it when either we're back on lld (see the TODO above), or when we get a
# new-enough `environment` file.
ld_flags = gn_args.get('cros_target_extra_ldflags')
if ld_flags:
ld_flags = ld_flags.replace('-Wl,-z,keep-text-section-prefix', '')
gn_args['cros_target_extra_ldflags'] = ld_flags
# For SimpleChrome, we use the binutils that comes bundled within Chrome.
# We should not use the binutils from the host system.
gn_args['linux_use_bundled_binutils'] = True
# Need to reset these after the env vars have been fixed by
# _SetupTCEnvironment.
gn_args['cros_host_is_clang'] = True
# v8 snapshot is built on the host, so we need to set this.
# See crosbug/618346.
gn_args['cros_v8_snapshot_is_clang'] = True
target_tc_path = sdk_ctx.key_map[self.sdk.TARGET_TOOLCHAIN_KEY].path
modified_env_cc = self._ModifyPathForGomaBuild(env['CC'], target_tc_path)
modified_env_cxx = self._ModifyPathForGomaBuild(env['CXX'], target_tc_path)
gn_args['cros_target_cc'] = modified_env_cc.split()[0]
gn_args['cros_target_cxx'] = modified_env_cxx.split()[0]
gn_args['cros_target_ld'] = env['LD']
gn_args['cros_target_nm'] = env['NM']
gn_args['cros_target_readelf'] = env['READELF']
gn_args['cros_target_extra_cflags'] = ' '.join(
[env.get('CFLAGS', '')] + modified_env_cc.split()[1:])
gn_args['cros_target_extra_cxxflags'] = ' '.join(
[env.get('CXXFLAGS', '')] + modified_env_cxx.split()[1:])
gn_args['cros_host_cc'] = self._ModifyPathForGomaBuild(env['CC_host'])
gn_args['cros_host_cxx'] = self._ModifyPathForGomaBuild(env['CXX_host'])
gn_args['cros_host_ld'] = env['LD_host']
gn_args['cros_host_nm'] = env['NM_host']
gn_args['cros_host_ar'] = env['AR_host']
gn_args['cros_host_readelf'] = env['READELF_host']
gn_args['cros_v8_snapshot_cc'] = self._ModifyPathForGomaBuild(
gn_args['cros_v8_snapshot_cxx'] = self._ModifyPathForGomaBuild(
gn_args['cros_v8_snapshot_ld'] = env['LD_host']
gn_args['cros_v8_snapshot_nm'] = env['NM_host']
gn_args['cros_v8_snapshot_ar'] = env['AR_host']
gn_args['cros_v8_snapshot_readelf'] = env['READELF_host']
# No need to adjust CFLAGS and CXXFLAGS for GN since the only
# adjustment made in _SetupTCEnvironment is for split debug which
# is done with 'use_debug_fission'.
# Enable goma if requested.
if not options.goma:
# If --nogoma option is explicitly set, disable goma, even if it is
# used in the original GN_ARGS.
gn_args['use_goma'] = False
elif goma_dir:
gn_args['use_goma'] = True
# Disable automatic gomacc handling in gn since we handle it ourselves.
gn_args['has_gomacc_path'] = True
env['GOMACC_PATH'] = os.path.join(goma_dir, 'gomacc')
# This is used to invoke host compiler via gomacc (e.g. v8_snapshot).
gn_args['goma_dir'] = goma_dir
gn_args.pop('internal_khronos_glcts_tests', None) #
# Disable ThinLTO and CFI for simplechrome. Tryjob machines do not have
# enough file descriptors to use.
if not options.thinlto and 'use_thin_lto' in gn_args:
gn_args['use_thin_lto'] = False
if not options.cfi and 'is_cfi' in gn_args:
gn_args['is_cfi'] = False
gn_args['use_cfi_cast'] = False
# We need to remove the flag -Wl,-plugin-opt,-import-instr-limit=$num
# from cros_target_extra_ldflags if options.thinlto is not set.
# The format of ld flags is something like
# '-Wl,-O1 -Wl,-O2 -Wl,--as-needed -stdlib=libc++'
if not options.thinlto:
extra_ldflags = gn_args.get('cros_target_extra_ldflags', '')
ld_flags_list = extra_ldflags.split()
ld_flags_list = (
[f for f in ld_flags_list
if not f.startswith('-Wl,-plugin-opt,-import-instr-limit')])
if extra_ldflags:
gn_args['cros_target_extra_ldflags'] = ' '.join(ld_flags_list)
# We removed webcore debug symbols on release builds on arm.
# See However, we want to keep the symbols
# for simplechrome builds.
# TODO: remove the 'remove_webcore_debug_symbols' once we
# change the ebuild file.
gn_args['remove_webcore_debug_symbols'] = False
gn_args['blink_symbol_level'] = -1
# Remove symbol_level specified in the ebuild to use the default.
# Currently that is 1 when is_debug=false, instead of 2 specified by the
# ebuild. This results in faster builds in Simple Chrome.
if 'symbol_level' in gn_args:
symbol_level = gn_args.pop('symbol_level')'Removing symbol_level = %d from gn args, use '
'--gn-extra-args to specify a non default value.',
if options.gn_extra_args:
gn_args_env = gn_helpers.ToGNString(gn_args)
env['GN_ARGS'] = gn_args_env
# PS1 sets the command line prompt and xterm window caption.
full_version = sdk_ctx.version
if full_version != CUSTOM_VERSION:
full_version = self.sdk.GetFullVersion(sdk_ctx.version)
env['PS1'] = self._CreatePS1(self.board, full_version,
# Set the useful part of PS1 for users with a custom PROMPT_COMMAND.
env['CROS_PS1_PREFIX'] = self._PS1Prefix(self.board, full_version,
out_dir = 'out_%s' % self.board
env['builddir_name'] = out_dir
build_label = 'Release'
# This is used by to prevent collisions when building both
# chromeos and android from shared source.
# For context, see
env['CHROMIUM_OUT_DIR'] = os.path.join(options.chrome_src, out_dir)
self._UpdateGnArgsIfStale(out_dir, build_label, gn_args, env['SDK_BOARD'])
return env
def _VerifyGoma(user_rc):
"""Verify that the user has no goma installations set up in user_rc.
If the user does have a goma installation set up, verify that it's for
user_rc: User-supplied rc file.
user_env = osutils.SourceEnvironment(user_rc, ['PATH'])
goma_ctl = osutils.Which('', user_env.get('PATH'))
if goma_ctl is not None:
'%s is adding Goma to the PATH. Using that Goma instead of the '
'managed Goma install.', user_rc)
def _VerifyChromiteBin(user_rc):
"""Verify that the user has not set a chromite bin/ dir in user_rc.
user_rc: User-supplied rc file.
user_env = osutils.SourceEnvironment(user_rc, ['PATH'])
chromite_bin = osutils.Which('parallel_emerge', user_env.get('PATH'))
if chromite_bin is not None:
'%s is adding chromite/bin to the PATH. Remove it from the PATH to '
'use the the default Chromite.', user_rc)
def _GetRCFile(self, env, user_rc):
"""Returns path to dynamically created bashrc file.
The bashrc file sets the environment variables contained in |env|, as well
as sources the user-editable chrome_sdk.bashrc file in the user's home
directory. That rc file is created if it doesn't already exist.
env: A dictionary of environment variables that will be set by the rc
user_rc: User-supplied rc file.
if not os.path.exists(user_rc):
osutils.Touch(user_rc, makedirs=True)
# We need a temporary rc file to 'wrap' the user configuration file,
# because running with '--rcfile' causes bash to ignore bash special
# variables passed through subprocess.Popen, such as PS1. So we set them
# here.
# Having a wrapper rc file will also allow us to inject bash functions into
# the environment, not just variables.
with osutils.TempDir() as tempdir:
# Only source the user's ~/.bashrc if running in interactive mode.
contents = [
'[[ -e ~/.bashrc && $- == *i* ]] && . ~/.bashrc\n',
for key, value in env.iteritems():
contents.append("export %s='%s'\n" % (key, value))
contents.append('. "%s"\n' % user_rc)
rc_file = os.path.join(tempdir, 'rcfile')
osutils.WriteFile(rc_file, contents)
yield rc_file
def _GomaPort(self, goma_dir):
"""Returns current active Goma port."""
port = cros_build_lib.RunCommand(
self.GOMACC_PORT_CMD, cwd=goma_dir, debug_level=logging.DEBUG,
error_code_ok=True, capture_output=True).output.strip()
return port
def _FetchGoma(self):
"""Fetch, install, and start Goma, using cached version if it exists.
A tuple (dir, port) containing the path to the cached goma/ dir and the
Goma port.
common_path = os.path.join(self.options.cache_dir, constants.COMMON_CACHE)
common_cache = cache.DiskCache(common_path)
goma_dir = self.options.gomadir
if not goma_dir:
ref = common_cache.Lookup(('goma', '2'))
if not ref.Exists():
Log('Installing Goma.', silent=self.silent)
with osutils.TempDir() as tempdir:
goma_dir = os.path.join(tempdir, 'goma')
with osutils.ChdirContext(tempdir):
result = retry_util.RunCurl(['--fail', self._GOMA_DOWNLOAD_URL],
if result.returncode:
raise GomaError('Failed to fetch Goma Download URL')
download_url = result.output.strip()
result = retry_util.RunCurl(
['--fail', '%s/%s' % (download_url, self._GOMA_TGZ),
'--output', self._GOMA_TGZ])
if result.returncode:
raise GomaError('Failed to fetch Goma')
except retry_util.DownloadError:
raise GomaError('Failed to fetch Goma')
result = cros_build_lib.DebugRunCommand(
['tar', 'xf', self._GOMA_TGZ,
'--strip=1', '-C', goma_dir])
if result.returncode:
raise GomaError('Failed to extract Goma')
result = cros_build_lib.DebugRunCommand(
[os.path.join(goma_dir, ''), 'update'],
extra_env={'PLATFORM': 'goobuntu'})
if result.returncode:
raise GomaError('Failed to install Goma')
goma_dir = ref.path
port = None
if self.options.start_goma:
Log('Starting Goma.', silent=self.silent)
[os.path.join(goma_dir, ''), 'ensure_start'])
port = self._GomaPort(goma_dir)
Log('Goma is started on port %s', port, silent=self.silent)
if not port:
raise GomaError('No Goma port detected')
return goma_dir, port
def Run(self):
"""Perform the command."""
if os.environ.get(SDKFetcher.SDK_VERSION_ENV) is not None:
cros_build_lib.Die('Already in an SDK shell.')
src_path = self.options.chrome_src or os.getcwd()
checkout = path_util.DetermineCheckout(src_path)
if not checkout.chrome_src_dir:
cros_build_lib.Die('Chrome checkout not found at %s', src_path)
self.options.chrome_src = checkout.chrome_src_dir
if self.options.clang and not self.options.chrome_src:
cros_build_lib.Die('--clang requires --chrome-src to be set.')
if self.options.version and self.options.sdk_path:
cros_build_lib.Die('Cannot specify both --version and --sdk-path.')
if self.options.cfi and not self.options.thinlto:
cros_build_lib.Die('CFI requires ThinLTO.')
self.silent = bool(self.options.cmd)
# Lazy initialize because SDKFetcher creates a GSContext() object in its
# constructor, which may block on user input.
self.sdk = SDKFetcher(
self.options.cache_dir, self.options.board,
prepare_version = self.options.version
if not prepare_version and not self.options.sdk_path:
prepare_version, _ = self.sdk.UpdateDefaultVersion()
components = [self.sdk.TARGET_TOOLCHAIN_KEY, constants.CHROME_ENV_TAR]
if not self.options.chroot:
if self.options.download_vm:
goma_dir = None
goma_port = None
if self.options.goma:
goma_dir, goma_port = self._FetchGoma()
except GomaError as e:
logging.error('Goma: %s. Bypass by running with --nogoma.', e)
with self.sdk.Prepare(components, version=prepare_version,
toolchain_url=self.options.toolchain_url) as ctx:
env = self._SetupEnvironment(self.options.board, ctx, self.options,
goma_dir=goma_dir, goma_port=goma_port)
with self._GetRCFile(env, self.options.bashrc) as rcfile:
bash_cmd = ['/bin/bash']
extra_env = None
if not self.options.cmd:
bash_cmd.extend(['--rcfile', rcfile, '-i'])
# The '"$@"' expands out to the properly quoted positional args
# coming after the '--'.
bash_cmd.extend(['-c', '"$@"', '--'])
# When run in noninteractive mode, bash sources the rc file set in
# BASH_ENV, and ignores the --rcfile flag.
extra_env = {'BASH_ENV': rcfile}
# Bash behaves differently when it detects that it's being launched by
# sshd - it ignores the BASH_ENV variable. So prevent ssh-related
# environment variables from being passed through.
os.environ.pop('SSH_CLIENT', None)
os.environ.pop('SSH_CONNECTION', None)
os.environ.pop('SSH_TTY', None)
cmd_result = cros_build_lib.RunCommand(
bash_cmd, print_cmd=False, debug_level=logging.CRITICAL,
error_code_ok=True, extra_env=extra_env, cwd=self.options.cwd)
if self.options.cmd:
return cmd_result.returncode