SysrootService: use package_indexes list when present.
If package_indexes is provided, then that is used instead of the
BINHOST.conf from the overlays.
BUG=chromium:1088059
TEST=unit tests pass.
Change-Id: Ic2041e7bcc93242403d01e1f12d93fbd44539d13
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/2341833
Tested-by: LaMont Jones <lamontjones@chromium.org>
Reviewed-by: Alex Klein <saklein@chromium.org>
Commit-Queue: LaMont Jones <lamontjones@chromium.org>
diff --git a/api/controller/sysroot.py b/api/controller/sysroot.py
index b0901ab..d632f06 100644
--- a/api/controller/sysroot.py
+++ b/api/controller/sysroot.py
@@ -15,6 +15,7 @@
from chromite.api import validate
from chromite.api.controller import controller_util
from chromite.api.metrics import deserialize_metrics_log
+from chromite.lib import binpkg
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import goma_lib
@@ -41,8 +42,13 @@
build_target = controller_util.ParseBuildTarget(input_proto.build_target,
input_proto.profile)
+ package_indexes = [
+ binpkg.PackageIndexInfo.from_protobuf(x)
+ for x in input_proto.package_indexes
+ ]
run_configs = sysroot.SetupBoardRunConfig(
- force=replace_sysroot, upgrade_chroot=update_chroot)
+ force=replace_sysroot, upgrade_chroot=update_chroot,
+ package_indexes=package_indexes)
try:
created = sysroot.Create(build_target, run_configs,
@@ -158,6 +164,10 @@
input_proto.sysroot.build_target)
packages = [controller_util.PackageInfoToString(x)
for x in input_proto.packages]
+ package_indexes = [
+ binpkg.PackageIndexInfo.from_protobuf(x)
+ for x in input_proto.package_indexes
+ ]
if not target_sysroot.IsToolchainInstalled():
cros_build_lib.Die('Toolchain must first be installed.')
@@ -169,6 +179,7 @@
usepkg=not compile_source,
install_debug_symbols=True,
packages=packages,
+ package_indexes=package_indexes,
use_flags=use_flags,
use_goma=use_goma,
incremental_build=False)
diff --git a/api/controller/sysroot_unittest.py b/api/controller/sysroot_unittest.py
index 5aad793..f852abc 100644
--- a/api/controller/sysroot_unittest.py
+++ b/api/controller/sysroot_unittest.py
@@ -121,7 +121,8 @@
sysroot_controller.Create(in_proto, out_proto, self.api_config)
# Default value checks.
- rc_patch.assert_called_with(force=force, upgrade_chroot=upgrade_chroot)
+ rc_patch.assert_called_with(force=force, upgrade_chroot=upgrade_chroot,
+ package_indexes=[])
self.assertEqual(board, out_proto.sysroot.build_target.name)
self.assertEqual(sysroot_path, out_proto.sysroot.path)
@@ -137,7 +138,8 @@
sysroot_controller.Create(in_proto, out_proto, self.api_config)
# Not default value checks.
- rc_patch.assert_called_with(force=force, upgrade_chroot=upgrade_chroot)
+ rc_patch.assert_called_with(force=force, upgrade_chroot=upgrade_chroot,
+ package_indexes=[])
self.assertEqual(board, out_proto.sysroot.build_target.name)
self.assertEqual(sysroot_path, out_proto.sysroot.path)
diff --git a/lib/binpkg.py b/lib/binpkg.py
index e48cf22..ffee91d 100644
--- a/lib/binpkg.py
+++ b/lib/binpkg.py
@@ -22,11 +22,14 @@
from six.moves import urllib
+from chromite.api.gen.chromiumos import common_pb2
+from chromite.lib import build_target_lib
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
from chromite.lib import gs
from chromite.lib import osutils
from chromite.lib import parallel
+from chromite.lib import sysroot_lib
assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'
@@ -319,6 +322,54 @@
self.modified = False
+class PackageIndexInfo(object):
+ """A parser for PackageIndex metadata.
+
+ Attributes:
+ snapshot_sha (str): The git SHA of the manifest snapshot.
+ snapshot_number (int): The snapshot number.
+ build_target (build_target_lib.BuildTarget): The build_target.
+ profile (Profile): The build_target.
+ location (str): The GS path for the prebuilts directory.
+ """
+
+ def __init__(self, snapshot_sha='', snapshot_number=0,
+ build_target=None, profile=None, location=''):
+ self.snapshot_sha = snapshot_sha
+ self.snapshot_number = snapshot_number
+ self.build_target = build_target or build_target_lib.BuildTarget(name='')
+ self.profile = profile or sysroot_lib.Profile()
+ self.location = location
+
+ @property
+ def as_protobuf(self):
+ """Return a chromiumos.PackageIndexInfo protobuf."""
+ return common_pb2.PackageIndexInfo(
+ snapshot_sha=self.snapshot_sha,
+ snapshot_number=self.snapshot_number,
+ build_target=self.build_target.as_protobuf,
+ profile=self.profile.as_protobuf,
+ location=self.location)
+
+ @classmethod
+ def from_protobuf(cls, message):
+ """Return a PackageIndexInfo object for the given PackageIndexInfo protobuf.
+
+ Args:
+ message (chromiumos.PackageIndexInfo): The protobuf to parse
+
+ Returns:
+ (PackageIndexInfo) the parsed instance.
+ """
+ return cls(
+ snapshot_sha=message.snapshot_sha,
+ snapshot_number=message.snapshot_number,
+ build_target=build_target_lib.BuildTarget.from_protobuf(
+ message.build_target),
+ profile=sysroot_lib.Profile.from_protobuf(message.profile),
+ location=message.location)
+
+
def _RetryUrlOpen(url, tries=3):
"""Open the specified url, retrying if we run into temporary errors.
diff --git a/lib/build_target_lib.py b/lib/build_target_lib.py
index d6bee95..828e3bd 100644
--- a/lib/build_target_lib.py
+++ b/lib/build_target_lib.py
@@ -10,6 +10,8 @@
import os
import re
+from chromite.api.gen.chromiumos import common_pb2
+
class Error(Exception):
"""Base module error class."""
@@ -59,6 +61,18 @@
def name(self):
return self._name
+ @property
+ def as_protobuf(self):
+ return common_pb2.BuildTarget(name=self.name)
+
+ @classmethod
+ def from_protobuf(cls, message):
+ return cls(name=message.name)
+
+ @property
+ def profile_protobuf(self):
+ return common_pb2.Profile(name=self.profile)
+
def full_path(self, *args):
"""Turn a sysroot-relative path into an absolute path."""
return os.path.join(self.root, *[part.lstrip(os.sep) for part in args])
diff --git a/lib/sysroot_lib.py b/lib/sysroot_lib.py
index c9f2bfd..3ed8b52 100644
--- a/lib/sysroot_lib.py
+++ b/lib/sysroot_lib.py
@@ -12,6 +12,7 @@
import os
import sys
+from chromite.api.gen.chromiumos import common_pb2
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import cros_logging as logging
@@ -239,6 +240,28 @@
return '/%s' % _MAKE_CONF_USER
+class Profile(object):
+ """Class that encapsulates the profile name for a sysroot."""
+
+ def __init__(self, name=''):
+ self._name = name
+
+ @property
+ def name(self):
+ return self._name
+
+ def __eq__(self, other):
+ return self.name == other.name
+
+ @property
+ def as_protobuf(self):
+ return common_pb2.Profile(name=self._name)
+
+ @classmethod
+ def from_protobuf(cls, message):
+ return cls(name=message.name)
+
+
class Sysroot(object):
"""Class that encapsulate the interaction with sysroots."""
@@ -412,12 +435,15 @@
config_file = _GetMakeConfGenericPath()
osutils.SafeSymlink(config_file, self._Path(_MAKE_CONF), sudo=True)
- def InstallMakeConfBoard(self, accepted_licenses=None, local_only=False):
+ def InstallMakeConfBoard(self, accepted_licenses=None, local_only=False,
+ package_indexes=None):
"""Make sure the make.conf.board file exists and is up to date.
Args:
accepted_licenses (str): Any additional accepted licenses.
local_only (bool): Whether prebuilts can be fetched from remote sources.
+ package_indexes (list[PackageIndexInfo]): List of information about
+ available prebuilts, youngest first, or None.
"""
board_conf = self.GenerateBoardMakeConf(accepted_licenses=accepted_licenses)
make_conf_path = self._Path(_MAKE_CONF_BOARD)
@@ -426,7 +452,8 @@
# Once make.conf.board has been generated, generate the binhost config.
# We need to do this in two steps as the binhost generation step needs
# portageq to be available.
- binhost_conf = self.GenerateBinhostConf(local_only=local_only)
+ binhost_conf = self.GenerateBinhostConf(local_only=local_only,
+ package_indexes=package_indexes)
osutils.WriteFile(make_conf_path, '%s\n%s\n' % (board_conf, binhost_conf),
sudo=True)
@@ -554,11 +581,13 @@
return '\n'.join(config)
- def GenerateBinhostConf(self, local_only=False):
+ def GenerateBinhostConf(self, local_only=False, package_indexes=None):
"""Returns the binhost configuration.
Args:
local_only (bool): If True, use binary packages from local boards only.
+ package_indexes (list[PackageIndexInfo]): List of information about
+ available prebuilts, youngest first, or None.
Returns:
str - The config contents.
@@ -577,6 +606,17 @@
'PORTAGE_BINHOST="$LOCAL_BINHOST"'])
config = []
+ if package_indexes:
+ # TODO(crbug/1088059): Drop all use of overlay commits, once the solution
+ # is in place for non-snapshot checkouts.
+ # If present, this defines PORTAGE_BINHOST. These are independent of the
+ # overlay commits.
+ config.append('# This is the list of binhosts provided by the API.')
+ config.append('PASSED_BINHOST="%s"' % ' '.join(
+ x.location for x in reversed(package_indexes)))
+ config.append('PORTAGE_BINHOST="$PASSED_BINHOST"')
+ return '\n'.join(config)
+
postsubmit_binhost, postsubmit_binhost_internal = self._PostsubmitBinhosts(
board)
diff --git a/service/sysroot.py b/service/sysroot.py
index 2e84ef9..8aaace8 100644
--- a/service/sysroot.py
+++ b/service/sysroot.py
@@ -41,7 +41,7 @@
def __init__(self, set_default=False, force=False, usepkg=True, jobs=None,
regen_configs=False, quiet=False, update_toolchain=True,
upgrade_chroot=True, init_board_pkgs=True, local_build=False,
- toolchain_changed=False):
+ toolchain_changed=False, package_indexes=None):
"""Initialize method.
Args:
@@ -57,6 +57,8 @@
local_build (bool): Bootstrap only from local packages?
toolchain_changed (bool): Has a toolchain change occurred? Implies
'force'.
+ package_indexes (list[PackageIndexInfo]): List of information about
+ available prebuilts, youngest first, or None.
"""
self.set_default = set_default
@@ -69,6 +71,7 @@
self.update_chroot = upgrade_chroot
self.init_board_pkgs = init_board_pkgs
self.local_build = local_build
+ self.package_indexes = package_indexes or []
def GetUpdateChrootArgs(self):
"""Create a list containing the relevant update_chroot arguments.
@@ -96,7 +99,7 @@
def __init__(self, usepkg=True, install_debug_symbols=False,
packages=None, use_flags=None, use_goma=False,
- incremental_build=True):
+ incremental_build=True, package_indexes=None):
"""Init method.
Args:
@@ -112,6 +115,8 @@
build or a fresh build. Always treating it as an incremental build is
safe, but certain operations can be faster when we know we are doing
a fresh build.
+ package_indexes (list[PackageIndexInfo]): List of information about
+ available prebuilts, youngest first, or None.
"""
self.usepkg = usepkg
self.install_debug_symbols = install_debug_symbols
@@ -119,6 +124,7 @@
self.use_flags = use_flags
self.use_goma = use_goma
self.is_incremental = incremental_build
+ self.package_indexes = package_indexes or []
def GetBuildPackagesArgs(self):
"""Get the build_packages script arguments."""
@@ -177,6 +183,10 @@
if self.use_goma:
env['USE_GOMA'] = 'true'
+ if self.package_indexes:
+ env['PORTAGE_BINHOST'] = ' '.join(
+ x.location for x in reversed(self.package_indexes))
+
return env
@@ -269,7 +279,8 @@
# Refresh the workon symlinks to compensate for crbug.com/679831.
logging.info('Setting up portage in the sysroot.')
_InstallPortageConfigs(sysroot, target, accept_licenses,
- run_configs.local_build)
+ run_configs.local_build,
+ package_indexes=run_configs.package_indexes)
# Developer Experience Step: Set default board (if requested) to allow
# running later commands without needing to pass the --board argument.
@@ -407,7 +418,8 @@
sysroot.InstallMakeConfUser()
-def _InstallPortageConfigs(sysroot, target, accept_licenses, local_build):
+def _InstallPortageConfigs(sysroot, target, accept_licenses, local_build,
+ package_indexes=None):
"""Install portage wrappers and configurations.
Dependencies: make.conf.board_setup (InstallConfigs).
@@ -420,13 +432,16 @@
in the sysroot.
accept_licenses (str): Additional accepted licenses as a string.
local_build (bool): If the build is a local only build.
+ package_indexes (list[PackageIndexInfo]): List of information about
+ available prebuilts, youngest first, or None.
"""
sysroot.CreateAllWrappers(friendly_name=target.name)
_ChooseProfile(target, sysroot)
_RefreshWorkonSymlinks(target.name, sysroot)
# Must be done after the profile is chosen or binhosts may be incomplete.
sysroot.InstallMakeConfBoard(accepted_licenses=accept_licenses,
- local_only=local_build)
+ local_only=local_build,
+ package_indexes=package_indexes)
def _InstallToolchain(sysroot, target, local_init=True):
