scripts/build_sdk_subtools.py - third_party/chromite - Git at Google

 # Copyright 2023 The ChromiumOS Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """build_sdk_subtools rebuilds binary packages exported by the subtools builder.

 The build_sdk_subtools process takes (copies) an amd64-host base SDK, compiles
 and installs additional packages needed by subtools, then creates relocatable
 binary subtool bundles that can be consumed by other build hosts and developer
 machines.

 If build_sdk_subtools has already been invoked for the provided chroot, all
 non-toolchain packages in the subtools deptree that have updated revisions or
 changed USE flags will be rebuilt, along with reverse dependencies.

 Packages (e.g. an ebuild) provide manifests that describes how files, once
 installed, are to be bundled and exported.

 If packages are specified in the command line, only consider the deptree from
 those specific packages rather than all of virtual/target-sdk-subtools.
 """

 import argparse
 import os
 from pathlib import Path
 import sys
 from typing import List, Optional, Protocol

 import chromite
 from chromite.lib import build_target_lib
 from chromite.lib import commandline
 from chromite.lib import constants
 from chromite.lib import cros_build_lib
 from chromite.lib import osutils
 from chromite.lib import sysroot_lib
 from chromite.service import sdk_subtools


 assert sys.version_info >= (3, 8), "build_sdk_subtools uses Python 3.8 features"

 logger = chromite.ChromiteLogger.getLogger(__name__)


 # Affects where building occurs (e.g. /build/amd64-subtools-host) if not
 # overridden by --output-dir. Note this will be a chroot.
 SUBTOOLS_OUTPUT_DIR = "amd64-subtools-host"

 # Flag passed to subprocesses in chroots that might not yet be set up as a
 # subtools chroot.
 _RELAUNCH_FOR_SETUP_FLAG = "--relaunch-for-setup"

 # Used to populate a test manifest in /etc/cros/standalone-packages.d/.
 # Ebuilds will later be updated to provide these files instead.
 _TEST_PACKAGE = (
     sdk_subtools.SUBTOOLS_EXPORTS_CONFIG_DIR / "shellcheck.textproto"
 )
 _TEST_PACKAGE_CONTENTS = """\
 # proto-file: chromiumos/build/api/subtools.proto
 # proto-message: chromiumos.build.api.SubtoolPackage
 name: "shellcheck"
 type: EXPORT_CIPD
 max_files: 2
 paths: [{
     input: "/usr/bin/shellcheck"
 }]
 """


 class Options(Protocol):
     """Protocol to formalize commandline arguments."""

     clean: bool
     setup_chroot: bool
     update_packages: bool
     production: bool
     relaunch_for_setup: bool
     output_dir: Path
     packages: List[str]
     export: List[str]
     jobs: int

     def Freeze(self) -> None:
         pass


 def get_parser() -> commandline.ArgumentParser:
     """Returns the cmdline argparser, populates the options and descriptions."""
     parser = commandline.ArgumentParser(description=__doc__)

     parser.add_bool_argument(
         "--clean",
         False,
         "Remove the subtools chroot and re-extract the SDK.",
         "Re-use an existing subtools chroot.",
     )

     parser.add_bool_argument(
         "--setup-chroot",
         True,
         "Look for a newer base SDK and set it up as a subtools SDK.",
         "Don't look for a newer base SDK and assume the chroot is setup.",
     )

     parser.add_bool_argument(
         "--update-packages",
         True,
         "Update and install packages before looking for things to export.",
         "Only export packages already installed in the subtools SDK.",
     )

     parser.add_bool_argument(
         "--production",
         False,
         "Use production environments for subtool exports.",
         "Use staging environments for subtool exports.",
     )

     parser.add_argument(
         "--output-dir",
         type=osutils.ExpandPath,
         metavar="PATH",
         help=f"Extract SDK and build in chroot (e.g. {SUBTOOLS_OUTPUT_DIR}).",
     )

     parser.add_argument(
         "--export",
         nargs="+",
         default=[],
         metavar="BUNDLE",
         help="Packages to export (e.g. to CIPD). May require auth.",
     )

     parser.add_argument(
         "packages",
         nargs="*",
         default=["virtual/target-sdk-subtools"],
         help="Packages to build before looking for export candidates.",
     )

     parser.add_argument(
         "--jobs",
         "-j",
         type=int,
         default=os.cpu_count(),
         help="Number of packages to build in parallel. (Default: %(default)s)",
     )

     parser.add_argument(
         _RELAUNCH_FOR_SETUP_FLAG,
         action="store_true",
         default=False,
         help=argparse.SUPPRESS,
     )

     # TODO(b/277992359): Consider possibly relevant flags from build_packages:
     #  * --rebuild_revdeps=no: don't rebuild reverse dependencies.
     #  * --skip-toolchain-update? Likely no - the SDK is our toolchain.
     #  * --withdebugsymbols
     #  * --rebuild=no "Automatically rebuild dependencies"
     #  * --backtrack
     #  * --bazel  "Use Bazel to build packages"

     return parser


 def parse_args(argv: Optional[List[str]]) -> Options:
     """Parse and validate CLI arguments."""

     parser = get_parser()
     opts: Options = parser.parse_args(argv)
     opts.Freeze()
     return opts


 def _setup_base_sdk(
     build_target: build_target_lib.BuildTarget,
     setup_chroot: bool,
 ) -> None:
     """SetupBoard workalike that converts a regular SDK into a subtools chroot.

     Runs inside the /build/amd64-subtools-host subtools SDK chroot.
     """
     cros_build_lib.AssertInsideChroot()
     cros_build_lib.AssertRootUser()

     sdk_subtools.setup_base_sdk(build_target, setup_chroot)

     if setup_chroot:
         _TEST_PACKAGE.write_text(_TEST_PACKAGE_CONTENTS, encoding="utf-8")


 def _run_inside_subtools_chroot(opts: Options) -> None:
     """Steps that build_sdk_subtools performs once it is in its chroot."""
     if opts.update_packages:
         try:
             sdk_subtools.update_packages(opts.packages, opts.jobs)
         except sysroot_lib.PackageInstallError as e:
             cros_build_lib.Die(e)

     installed = sdk_subtools.bundle_and_export(opts.production, opts.export)
     if not installed.subtools:
         logger.warn("No subtools available.")
     elif not opts.export:
         logger.notice(
             "Use --export to export a package. Available:%s",
             "".join(f"\n\t{x.summary}" for x in installed.subtools),
         )


 def main(argv: Optional[List[str]] = None) -> Optional[int]:
     opts = parse_args(argv)
     return build_sdk_subtools(opts, argv if argv else [])


 def build_sdk_subtools(opts: Options, argv: List[str]) -> int:
     """Executes SDK subtools builder steps according to `opts`."""
     # BuildTarget needs a str, but opts.output_dir is osutils.ExpandPath.
     custom_output_dir = str(opts.output_dir) if opts.output_dir else None
     build_target = build_target_lib.BuildTarget(
         name=SUBTOOLS_OUTPUT_DIR, build_root=custom_output_dir
     )

     # If the process is in the subtools chroot, we must assume it's already set
     # up (we are in it). So start building.
     if sdk_subtools.is_inside_subtools_chroot() and not opts.relaunch_for_setup:
         _run_inside_subtools_chroot(opts)
         return 0

     # Otherwise, we have the option to set it up. Then restart inside it. The
     # setup runs `cros_sdk` to get a base SDK, creates an SDK subprocess to set
     # it up as a subtools SDK, then restarts inside the subtools SDK.
     if cros_build_lib.IsInsideChroot():
         if opts.relaunch_for_setup:
             # This is the subprocess of the not-in-chroot path used to convert
             # the base SDK to a subtools SDK (within the chroot).
             _setup_base_sdk(build_target, opts.setup_chroot)
             return 0
         else:
             cros_build_lib.Die(
                 "build_sdk_subtools must be run outside the chroot."
             )

     subtools_chroot = constants.DEFAULT_OUT_PATH / build_target.root.lstrip("/")
     chroot_args = ["--chroot", subtools_chroot]
     logger.info("Initializing subtools builder in %s", subtools_chroot)

     if opts.setup_chroot:
         # Get an SDK. TODO(b/277992359):
         #   - Fetch an SDK version pinned by pupr rather than the default.
         #   - Should this use cros_sdk_lib directly?

         # Ensure `cros_sdk` can check for a lock file on first use.
         osutils.SafeMakedirs(subtools_chroot.parent)

         # Pass "--skip-chroot-upgrade": the SDK should initially be used
         # "as-is", but later steps may upgrade packages in the subtools deptree.
         cros_sdk_args = ["--create", "--skip-chroot-upgrade"]
         if opts.clean:
             # Subtools SDKs go under out/build, so if --no-delete-out-dir isn't
             # used, cros_sdk will try to delete the SDK it's also making.
             cros_sdk_args += ["--delete", "--no-delete-out-dir"]

         cros_sdk = cros_build_lib.run(
             ["cros_sdk"] + chroot_args + cros_sdk_args,
             check=False,
             cwd=constants.SOURCE_ROOT,
         )
         if cros_sdk.returncode != 0:
             return cros_sdk.returncode

         # Invoke `_setup_base_sdk()` inside the SDK.
         setup_base_sdk = cros_build_lib.sudo_run(
             ["build_sdk_subtools"] + argv + [_RELAUNCH_FOR_SETUP_FLAG],
             check=False,
             enter_chroot=True,
             chroot_args=chroot_args,
             cwd=constants.SOURCE_ROOT,
         )
         if setup_base_sdk.returncode != 0:
             return setup_base_sdk.returncode

     raise commandline.ChrootRequiredError(
         ["build_sdk_subtools"] + argv, chroot_args=chroot_args
     )
	# Copyright 2023 The ChromiumOS Authors
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""build_sdk_subtools rebuilds binary packages exported by the subtools builder.

	The build_sdk_subtools process takes (copies) an amd64-host base SDK, compiles
	and installs additional packages needed by subtools, then creates relocatable
	binary subtool bundles that can be consumed by other build hosts and developer
	machines.

	If build_sdk_subtools has already been invoked for the provided chroot, all
	non-toolchain packages in the subtools deptree that have updated revisions or
	changed USE flags will be rebuilt, along with reverse dependencies.

	Packages (e.g. an ebuild) provide manifests that describes how files, once
	installed, are to be bundled and exported.

	If packages are specified in the command line, only consider the deptree from
	those specific packages rather than all of virtual/target-sdk-subtools.
	"""

	import argparse
	import os
	from pathlib import Path
	import sys
	from typing import List, Optional, Protocol

	import chromite
	from chromite.lib import build_target_lib
	from chromite.lib import commandline
	from chromite.lib import constants
	from chromite.lib import cros_build_lib
	from chromite.lib import osutils
	from chromite.lib import sysroot_lib
	from chromite.service import sdk_subtools


	assert sys.version_info >= (3, 8), "build_sdk_subtools uses Python 3.8 features"

	logger = chromite.ChromiteLogger.getLogger(__name__)


	# Affects where building occurs (e.g. /build/amd64-subtools-host) if not
	# overridden by --output-dir. Note this will be a chroot.
	SUBTOOLS_OUTPUT_DIR = "amd64-subtools-host"

	# Flag passed to subprocesses in chroots that might not yet be set up as a
	# subtools chroot.
	_RELAUNCH_FOR_SETUP_FLAG = "--relaunch-for-setup"

	# Used to populate a test manifest in /etc/cros/standalone-packages.d/.
	# Ebuilds will later be updated to provide these files instead.
	_TEST_PACKAGE = (
	sdk_subtools.SUBTOOLS_EXPORTS_CONFIG_DIR / "shellcheck.textproto"
	)
	_TEST_PACKAGE_CONTENTS = """\
	# proto-file: chromiumos/build/api/subtools.proto
	# proto-message: chromiumos.build.api.SubtoolPackage
	name: "shellcheck"
	type: EXPORT_CIPD
	max_files: 2
	paths: [{
	input: "/usr/bin/shellcheck"
	}]
	"""


	class Options(Protocol):
	"""Protocol to formalize commandline arguments."""

	clean: bool
	setup_chroot: bool
	update_packages: bool
	production: bool
	relaunch_for_setup: bool
	output_dir: Path
	packages: List[str]
	export: List[str]
	jobs: int

	def Freeze(self) -> None:
	pass


	def get_parser() -> commandline.ArgumentParser:
	"""Returns the cmdline argparser, populates the options and descriptions."""
	parser = commandline.ArgumentParser(description=__doc__)

	parser.add_bool_argument(
	"--clean",
	False,
	"Remove the subtools chroot and re-extract the SDK.",
	"Re-use an existing subtools chroot.",
	)

	parser.add_bool_argument(
	"--setup-chroot",
	True,
	"Look for a newer base SDK and set it up as a subtools SDK.",
	"Don't look for a newer base SDK and assume the chroot is setup.",
	)

	parser.add_bool_argument(
	"--update-packages",
	True,
	"Update and install packages before looking for things to export.",
	"Only export packages already installed in the subtools SDK.",
	)

	parser.add_bool_argument(
	"--production",
	False,
	"Use production environments for subtool exports.",
	"Use staging environments for subtool exports.",
	)

	parser.add_argument(
	"--output-dir",
	type=osutils.ExpandPath,
	metavar="PATH",
	help=f"Extract SDK and build in chroot (e.g. {SUBTOOLS_OUTPUT_DIR}).",
	)

	parser.add_argument(
	"--export",
	nargs="+",
	default=[],
	metavar="BUNDLE",
	help="Packages to export (e.g. to CIPD). May require auth.",
	)

	parser.add_argument(
	"packages",
	nargs="*",
	default=["virtual/target-sdk-subtools"],
	help="Packages to build before looking for export candidates.",
	)

	parser.add_argument(
	"--jobs",
	"-j",
	type=int,
	default=os.cpu_count(),
	help="Number of packages to build in parallel. (Default: %(default)s)",
	)

	parser.add_argument(
	_RELAUNCH_FOR_SETUP_FLAG,
	action="store_true",
	default=False,
	help=argparse.SUPPRESS,
	)

	# TODO(b/277992359): Consider possibly relevant flags from build_packages:
	# * --rebuild_revdeps=no: don't rebuild reverse dependencies.
	# * --skip-toolchain-update? Likely no - the SDK is our toolchain.
	# * --withdebugsymbols
	# * --rebuild=no "Automatically rebuild dependencies"
	# * --backtrack
	# * --bazel "Use Bazel to build packages"

	return parser


	def parse_args(argv: Optional[List[str]]) -> Options:
	"""Parse and validate CLI arguments."""

	parser = get_parser()
	opts: Options = parser.parse_args(argv)
	opts.Freeze()
	return opts


	def _setup_base_sdk(
	build_target: build_target_lib.BuildTarget,
	setup_chroot: bool,
	) -> None:
	"""SetupBoard workalike that converts a regular SDK into a subtools chroot.

	Runs inside the /build/amd64-subtools-host subtools SDK chroot.
	"""
	cros_build_lib.AssertInsideChroot()
	cros_build_lib.AssertRootUser()

	sdk_subtools.setup_base_sdk(build_target, setup_chroot)

	if setup_chroot:
	_TEST_PACKAGE.write_text(_TEST_PACKAGE_CONTENTS, encoding="utf-8")


	def _run_inside_subtools_chroot(opts: Options) -> None:
	"""Steps that build_sdk_subtools performs once it is in its chroot."""
	if opts.update_packages:
	try:
	sdk_subtools.update_packages(opts.packages, opts.jobs)
	except sysroot_lib.PackageInstallError as e:
	cros_build_lib.Die(e)

	installed = sdk_subtools.bundle_and_export(opts.production, opts.export)
	if not installed.subtools:
	logger.warn("No subtools available.")
	elif not opts.export:
	logger.notice(
	"Use --export to export a package. Available:%s",
	"".join(f"\n\t{x.summary}" for x in installed.subtools),
	)


	def main(argv: Optional[List[str]] = None) -> Optional[int]:
	opts = parse_args(argv)
	return build_sdk_subtools(opts, argv if argv else [])


	def build_sdk_subtools(opts: Options, argv: List[str]) -> int:
	"""Executes SDK subtools builder steps according to `opts`."""
	# BuildTarget needs a str, but opts.output_dir is osutils.ExpandPath.
	custom_output_dir = str(opts.output_dir) if opts.output_dir else None
	build_target = build_target_lib.BuildTarget(
	name=SUBTOOLS_OUTPUT_DIR, build_root=custom_output_dir
	)

	# If the process is in the subtools chroot, we must assume it's already set
	# up (we are in it). So start building.
	if sdk_subtools.is_inside_subtools_chroot() and not opts.relaunch_for_setup:
	_run_inside_subtools_chroot(opts)
	return 0

	# Otherwise, we have the option to set it up. Then restart inside it. The
	# setup runs `cros_sdk` to get a base SDK, creates an SDK subprocess to set
	# it up as a subtools SDK, then restarts inside the subtools SDK.
	if cros_build_lib.IsInsideChroot():
	if opts.relaunch_for_setup:
	# This is the subprocess of the not-in-chroot path used to convert
	# the base SDK to a subtools SDK (within the chroot).
	_setup_base_sdk(build_target, opts.setup_chroot)
	return 0
	else:
	cros_build_lib.Die(
	"build_sdk_subtools must be run outside the chroot."
	)

	subtools_chroot = constants.DEFAULT_OUT_PATH / build_target.root.lstrip("/")
	chroot_args = ["--chroot", subtools_chroot]
	logger.info("Initializing subtools builder in %s", subtools_chroot)

	if opts.setup_chroot:
	# Get an SDK. TODO(b/277992359):
	# - Fetch an SDK version pinned by pupr rather than the default.
	# - Should this use cros_sdk_lib directly?

	# Ensure `cros_sdk` can check for a lock file on first use.
	osutils.SafeMakedirs(subtools_chroot.parent)

	# Pass "--skip-chroot-upgrade": the SDK should initially be used
	# "as-is", but later steps may upgrade packages in the subtools deptree.
	cros_sdk_args = ["--create", "--skip-chroot-upgrade"]
	if opts.clean:
	# Subtools SDKs go under out/build, so if --no-delete-out-dir isn't
	# used, cros_sdk will try to delete the SDK it's also making.
	cros_sdk_args += ["--delete", "--no-delete-out-dir"]

	cros_sdk = cros_build_lib.run(
	["cros_sdk"] + chroot_args + cros_sdk_args,
	check=False,
	cwd=constants.SOURCE_ROOT,
	)
	if cros_sdk.returncode != 0:
	return cros_sdk.returncode

	# Invoke `_setup_base_sdk()` inside the SDK.
	setup_base_sdk = cros_build_lib.sudo_run(
	["build_sdk_subtools"] + argv + [_RELAUNCH_FOR_SETUP_FLAG],
	check=False,
	enter_chroot=True,
	chroot_args=chroot_args,
	cwd=constants.SOURCE_ROOT,
	)
	if setup_base_sdk.returncode != 0:
	return setup_base_sdk.returncode

	raise commandline.ChrootRequiredError(
	["build_sdk_subtools"] + argv, chroot_args=chroot_args
	)