blob: 904fd524682174779eaaa3d738ed8c72f1a1d384 [file] [log] [blame]
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Wrapper for building the Chromium OS platform.
Takes care of running GN/ninja/etc... with all the right values.
from __future__ import print_function
import collections
import glob
import json
import os
import six
from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import osutils
from chromite.lib import portage_util
import common_utils
import ebuild_function
# USE flags used in should be listed in _IUSE or _IUSE_TRUE.
# USE flags whose default values are false.
_IUSE = [
# USE flags whose default values are true.
class Platform2(object):
"""Main builder logic for platform2"""
def __init__(self, use_flags=None, board=None, host=False, libdir=None,
incremental=True, verbose=False, enable_tests=False,
cache_dir=None, jobs=None, platform_subdir=None):
self.board = board = host
self.incremental = incremental = jobs
self.verbose = verbose
self.platform_subdir = platform_subdir
if use_flags is not None:
self.use_flags = use_flags
self.use_flags = portage_util.GetBoardUseFlags(self.board)
if enable_tests:
self.sysroot = '/'
board_vars = self.get_portageq_envvars(['SYSROOT'], board=board)
self.sysroot = board_vars['SYSROOT']
if libdir:
self.libdir = libdir
self.libdir = '/usr/lib'
if cache_dir:
self.cache_dir = cache_dir
self.cache_dir = os.path.join(self.sysroot,
self.libbase_ver = os.environ.get('BASE_VER', '')
if not self.libbase_ver:
# If BASE_VER variable not set, read the content of
# $SYSROOT/usr/share/libchrome/BASE_VER
# file which contains the default libchrome revision number.
base_ver_file = os.path.join(self.sysroot,
self.libbase_ver = osutils.ReadFile(base_ver_file).strip()
except FileNotFoundError:
# Software not depending on libchrome still uses, Instead
# of asserting here. Provide a human readable bad value that is not
# supposed to be used.
self.libbase_ver = 'NOT-INSTALLED'
def get_src_dir(self):
"""Return the path to build tools and common GN files"""
return os.path.realpath(os.path.dirname(__file__))
def get_platform2_root(self):
"""Return the path to src/platform2"""
return os.path.dirname(self.get_src_dir())
def get_buildroot(self):
"""Return the path to the folder where build artifacts are located."""
if not self.incremental:
workdir = os.environ.get('WORKDIR')
if workdir:
# Matches $(cros-workon_get_build_dir) behavior.
return os.path.join(workdir, 'build')
return os.getcwd()
return self.cache_dir
def get_products_path(self):
"""Return the path to the folder where build product are located."""
return os.path.join(self.get_buildroot(), 'out/Default')
def get_portageq_envvars(self, varnames, board=None):
"""Returns the values of a given set of variables using portageq."""
# See if the env already has these settings. If so, grab them directly.
# This avoids the need to specify --board at all most of the time.
board_vars = {}
for varname in varnames:
board_vars[varname] = os.environ[varname]
return board_vars
except KeyError:
if board is None and not
board = self.board
# Portage will set this to an incomplete list which breaks portageq
# walking all of the repos. Clear it and let the value be repopulated.
os.environ.pop('PORTDIR_OVERLAY', None)
return portage_util.PortageqEnvvars(varnames, board=board,
def get_build_environment(self):
"""Returns a dict containing environment variables we will use to run GN.
We do this to set the various toolchain names for the target board.
varnames = ['CHOST', 'AR', 'CC', 'CXX', 'PKG_CONFIG']
if not and not self.board:
for v in varnames:
os.environ.setdefault(v, '')
board_env = self.get_portageq_envvars(varnames)
tool_names = {
'AR': 'ar',
'CC': 'gcc',
'CXX': 'g++',
'PKG_CONFIG': 'pkg-config',
env = os.environ.copy()
for var, tool in tool_names.items():
env['%s_target' % var] = (board_env[var] if board_env[var] else \
'%s-%s' % (board_env['CHOST'], tool))
return env
def get_components_glob(self):
"""Return a glob of marker files for components/projects that were built.
Each project spits out a file whilst building: we return a glob of them
so we can install/test those projects or reset between compiles to ensure
components that are no longer part of the build don't get installed.
return glob.glob(os.path.join(self.get_products_path(),
def can_use_gn(self):
"""Returns true if GN can be used on configure.
All packages in platform2/ should be configured by GN.
build_gn = os.path.join(self.get_platform2_root(), self.platform_subdir,
return os.path.isfile(build_gn)
def configure(self, args):
"""Runs the configure step of the Platform2 build.
Creates the build root if it doesn't already exists. Then runs the
appropriate configure tool. Currenty only GN is supported.
assert self.can_use_gn()
# The args was used only for gyp.
# TODO( remove code for handling args.
# There is a logic to in the platform eclass file, which detects a .gyp
# file under project root and passes it to here an arg.
if args:
print('Warning: Args for GYP was given. We will no longer use GYP. '
'Ignoring it and continuing configuration with GN.')
if not os.path.isdir(self.get_buildroot()):
if not self.incremental:
osutils.RmDir(self.get_products_path(), ignore_missing=True)
def gen_common_args(self, should_parse_shell_string):
"""Generates common arguments for the tools to configure as a dict.
Returned value types are str, bool or list of strs.
Lists are returned only when should_parse_shell_string is set to True.
def flags(s):
if should_parse_shell_string:
return common_utils.parse_shell_args(s)
return s
args = {
'OS': 'linux',
'sysroot': self.sysroot,
'libdir': self.libdir,
'build_root': self.get_buildroot(),
'platform2_root': self.get_platform2_root(),
'libbase_ver': self.libbase_ver,
'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1',
'external_cflags': flags(os.environ.get('CFLAGS', '')),
'external_cxxflags': flags(os.environ.get('CXXFLAGS', '')),
'external_cppflags': flags(os.environ.get('CPPFLAGS', '')),
'external_ldflags': flags(os.environ.get('LDFLAGS', '')),
return args
def configure_gn_args(self):
"""Configure with GN.
Generates flags to run GN with, and then runs GN.
def to_gn_string(s):
return '"%s"' % s.replace('"', '\\"')
def to_gn_list(strs):
return '[%s]' % ','.join([to_gn_string(s) for s in strs])
def to_gn_args_args(gn_args):
for k, v in gn_args.items():
if isinstance(v, bool):
v = str(v).lower()
elif isinstance(v, list):
v = to_gn_list(v)
elif isinstance(v, six.string_types):
v = to_gn_string(v)
raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v))
yield '%s=%s' % (k.replace('-', '_'), v)
buildenv = self.get_build_environment()
gn_args = {
'platform_subdir': self.platform_subdir,
'cc': buildenv.get('CC_target', buildenv.get('CC', '')),
'cxx': buildenv.get('CXX_target', buildenv.get('CXX', '')),
'ar': buildenv.get('AR_target', buildenv.get('AR', '')),
'pkg-config': buildenv.get('PKG_CONFIG_target',
buildenv.get('PKG_CONFIG', '')),
gn_args['clang_cc'] = 'clang' in gn_args['cc']
gn_args['clang_cxx'] = 'clang' in gn_args['cxx']
gn_args_args = list(to_gn_args_args(gn_args))
# Set use flags as a scope.
uses = {}
for flag in _IUSE:
uses[flag] = False
for flag in _IUSE_TRUE:
uses[flag] = True
for x in self.use_flags:
uses[x.replace('-', '_')] = True
use_args = ['%s=%s' % (x, str(uses[x]).lower()) for x in uses]
gn_args_args += ['use={%s}' % (' '.join(use_args))]
return gn_args_args
def configure_gn(self):
"""Configure with GN.
Runs gn gen with generated flags.
gn_args_args = self.configure_gn_args()
gn_args = ['gn', 'gen']
if self.verbose:
gn_args += ['-v']
gn_args += [
'--root=%s' % self.get_platform2_root(),
'--args=%s' % ' '.join(gn_args_args),
], env=self.get_build_environment(),
def gn_desc(self, *args):
Runs gn desc with generated flags.
gn_args_args = self.configure_gn_args()
cmd = [
'gn', 'desc',
'//%s/*' % self.platform_subdir,
'--root=%s' % self.get_platform2_root(),
'--args=%s' % ' '.join(gn_args_args),
cmd += args
result =, env=self.get_build_environment(),
stdout=True, encoding='utf-8')
return json.loads(result.output)
def compile(self, args):
"""Runs the compile step of the Platform2 build.
Removes any existing component markers that may exist (so we don't run
tests/install for projects that have been disabled since the last
build). Builds arguments for running Ninja and then runs Ninja.
for component in self.get_components_glob():
args = ['%s:%s' % (self.platform_subdir, x) for x in args]
ninja_args = ['ninja', '-C', self.get_products_path()]
ninja_args += ['-j', str(]
ninja_args += args
if self.verbose:
if os.environ.get('NINJA_ARGS'):
def deviterate(self, args):
"""Runs the configure and compile steps of the Platform2 build.
This is the default action, to allow easy iterative testing of changes
as a developer.
def configure_test(self):
"""Generates test options from GN."""
def to_options(options):
"""Convert dict to shell string."""
result = []
for key, value in options.items():
if isinstance(value, bool):
if value:
result.append('--%s' % key)
if key == 'raw':
result.append('--%s=%s' % (key, value))
return result
conf = self.gn_desc('--all', '--type=executable')
group_all = conf.get('//%s:all' % self.platform_subdir, {})
group_all_deps = group_all.get('deps', [])
options_list = []
for target_name in group_all_deps:
test_target = conf.get(target_name)
outputs = test_target.get('outputs', [])
if len(outputs) != 1:
output = outputs[0]
metadata = test_target.get('metadata', {})
run_test = unwrap_value(metadata, '_run_test', False)
if not run_test:
test_config = unwrap_value(metadata, '_test_config', {})
p2_test_py = os.path.join(self.get_src_dir(), '')
options = [
'--sysroot=%s' % self.sysroot,
options += ['--host']
p2_test_filter = os.environ.get('P2_TEST_FILTER')
if p2_test_filter:
options += ['--user_gtest_filter=%s' % p2_test_filter]
options += to_options(test_config)
options += ['--', output]
return options_list
def test_all(self, _args):
"""Runs all tests described from GN."""
test_options_list = self.configure_test()
for test_options in test_options_list:, encoding='utf-8')
def configure_install(self):
"""Generates installation commands of ebuild."""
conf = self.gn_desc('--all')
group_all = conf.get('//%s:all' % self.platform_subdir, {})
group_all_deps = group_all.get('deps', [])
config_group = collections.defaultdict(list)
for target_name in group_all_deps:
target_conf = conf.get(target_name, {})
metadata = target_conf.get('metadata', {})
install_config = unwrap_value(metadata, '_install_config')
if not install_config:
sources = install_config.get('sources')
if not sources:
install_path = install_config.get('install_path')
outputs = install_config.get('outputs')
symlinks = install_config.get('symlinks')
recursive = install_config.get('recursive')
options = install_config.get('options')
command_type = install_config.get('type')
config_key = (install_path, recursive, options, command_type)
config_group[config_key].append((sources, outputs, symlinks))
cmd_list = []
for install_config, install_args in config_group.items():
args = []
# Commands to install sources without explicit outputs nor symlinks can be
# merged into one. Concat all such sources.
sources = sum([sources for sources, outputs, symlinks in install_args
if not outputs and not symlinks], [])
if sources:
args.append((sources, None, None))
# Append all remaining sources/outputs/symlinks.
args += [(sources, outputs, symlinks) for
sources, outputs, symlinks in install_args
if outputs or symlinks]
# Generate the command line.
install_path, recursive, options, command_type = install_config
for sources, outputs, symlinks in args:
cmd_list += ebuild_function.generate(sources=sources,
return cmd_list
def install(self, _args):
"""Outputs the installation commands of ebuild as a standard output."""
install_cmd_list = self.configure_install()
for install_cmd in install_cmd_list:
# An error occurs at six.moves.shlex_quote when running pylint.
# pylint: disable=too-many-function-args
print(' '.join(six.moves.shlex_quote(arg) for arg in install_cmd))
def unwrap_value(metadata, attr, default=None):
"""Gets a value like dict.get() with unwrapping it."""
data = metadata.get(attr)
if data is None:
return default
return data[0]
def GetParser():
"""Return a command line parser."""
actions = ['configure', 'compile', 'deviterate', 'test_all', 'install']
parser = commandline.ArgumentParser(description=__doc__)
parser.add_argument('--action', default='deviterate',
choices=actions, help='action to run')
help='board to build for')
help='directory to use as cache for incremental build')
parser.add_argument('--disable_incremental', action='store_false',
dest='incremental', help='disable incremental build')
parser.add_argument('--enable_tests', action='store_true',
help='build and run tests')
parser.add_argument('--host', action='store_true',
help="specify that we're building for the host")
help='the libdir for the specific board, eg /usr/lib64')
action='split_extend', help='USE flags to enable')
parser.add_argument('-j', '--jobs', type=int, default=None,
help='number of jobs to run in parallel')
parser.add_argument('--platform_subdir', required=True,
help='subdir in platform2 where the package is located')
parser.add_argument('args', nargs='*')
return parser
def main(argv):
parser = GetParser()
# Temporary measure. Moving verbose argument, but can't do it all in one
# sweep due to CROS_WORKON_BLACKLISTed packages. Use parse_known_args and
# manually handle verbose parsing to maintain compatibility.
options, unknown = parser.parse_known_args(argv)
if not hasattr(options, 'verbose'):
options.verbose = '--verbose' in unknown
if '--verbose' in unknown:
if unknown:
parser.error('Unrecognized arguments: %s' % unknown)
if and options.board:
raise AssertionError('You must provide only one of --board or --host')
if not options.verbose:
# Should convert to cros_build_lib.BooleanShellValue.
options.verbose = (os.environ.get('VERBOSE', '0') == '1')
p2 = Platform2(options.use_flags, options.board,,
options.libdir, options.incremental, options.verbose,
options.enable_tests, options.cache_dir,,
getattr(p2, options.action)(options.args)
if __name__ == '__main__':
commandline.ScriptWrapperMain(lambda _: main)