dev-embedded/ti50-sdk/files/build_rust_toolchain.py - third_party/overlays/chromiumos-overlay - Git at Google

 #!/usr/bin/env python3
 # Copyright 2021 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Builds the Rust toolchain used to compile and run TockOS."""

 import argparse
 import logging
 import os
 from pathlib import Path
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 from typing import Iterable, List, NamedTuple, Optional

 # The target triples that we want this toolchain to be able to build binaries
 # for. ATM this consists of the host and one target. If you update this,
 # remember to update |write_config_toml| with info about the compilers to use.
 #
 # _A_ rust compiler is needed for the host because some packages have build
 # scripts (e.g., build.rs), and Ti50 has previously expressed interest in
 # running _some_ tests in a test environment that runs natively on the host.
 # It's unclear how difficult it'd be to use the CrOS Rust toolchain for both of
 # these, given that said toolchain is locked to the stable channel, and is
 # likely to be a very different version of Rust than Ti50's toolchain.
 TARGET_TRIPLES_TO_SHIP = (
     'riscv32imc-unknown-none-elf',
     'x86_64-unknown-linux-gnu',
 )


 class RustComponent(NamedTuple):
   """A component of Rust that we want to build and install."""
   # The name of the component that |./x.py dist| understands.
   dist_build_name: str
   # A regex pattern that matches the tarball produced for this component under
   # dist/.
   dist_tarball_pattern: str
   # Whether the component's dist tarballs are per-triple.
   dist_tarballs_are_per_triple: bool


 # Components we ship as a part of our toolchain.
 COMPONENTS_TO_SHIP = (
     # cargo, which is shipped with |rustup| toolchains by default.
     RustComponent(
         dist_build_name='cargo',
         dist_tarball_pattern=r'^cargo-',
         dist_tarballs_are_per_triple=False,
     ),
     # clippy, which was requested by TockOS.
     RustComponent(
         dist_build_name='clippy',
         dist_tarball_pattern=r'^clippy-',
         dist_tarballs_are_per_triple=False,
     ),
     # llvm-tools-preview, which is apparently required by TockOS itself.
     RustComponent(
         dist_build_name='llvm-tools',
         dist_tarball_pattern=r'^llvm-tools-',
         dist_tarballs_are_per_triple=False,
     ),
     # src, which is required when cross-compiling in some contexts.
     RustComponent(
         dist_build_name='src',
         dist_tarball_pattern=r'^rust-src-',
         dist_tarballs_are_per_triple=False,
     ),
     # src/librustc, which is the actual |rustc| compiler.
     RustComponent(
         dist_build_name='src/librustc',
         dist_tarball_pattern=r'^rustc-[^-]+-dev-(?!src)',
         dist_tarballs_are_per_triple=False,
     ),
     # rustfmt, the standard formatter for rust code.
     RustComponent(
         dist_build_name='rustfmt',
         dist_tarball_pattern=r'^rustfmt-',
         dist_tarballs_are_per_triple=False,
     ),
     # library/std, which consists of stdlib |rlib|s.
     RustComponent(
         dist_build_name='library/std',
         dist_tarball_pattern=r'^rust-std-',
         dist_tarballs_are_per_triple=True,
     ),
 )


 def must_getenv(key: str) -> str:
   """Gets the value of an environment variable; raises if unset or empty."""
   x = os.getenv(key)
   if not x:
     raise ValueError(f'No value found for {key}; this env var is required')
   return x


 def write_config_toml(rust_src: Path, rv_clang_bin: Path, install_prefix: Path,
                       rustc: Optional[Path], cargo: Optional[Path],
                       rustfmt: Optional[Path]) -> None:
   """Writes Rust's config.toml with the given parameters."""
   # We need a Pythony list, but formatted with double-quotes.
   assert not any('"' in x or '\\' in x
                  for x in TARGET_TRIPLES_TO_SHIP), TARGET_TRIPLES_TO_SHIP
   target_triples_str = ', '.join(f'"{x}"' for x in TARGET_TRIPLES_TO_SHIP)

   component_locations = (
       ('rustc', rustc),
       ('cargo', cargo),
       ('rustfmt', rustfmt),
   )
   optional_bootstrap_configuration = '\n'.join(
       f'{key} = "{exe}"' for key, exe in component_locations if exe is not None)

   toml_contents = f"""
 [build]
 target = [{target_triples_str}]
 extended = true
 vendor = true
 python = "{must_getenv('EPYTHON')}"
 submodules = false
 profiler = true
 {optional_bootstrap_configuration}

 [llvm]
 use-libcxx = true
 ninja = true

 [install]
 prefix = "{install_prefix}"

 [rust]
 # Provide llvm-tools like llvm-objcopy for the llvm-tools package.
 llvm-tools = true
 codegen-units = 0
 codegen-tests = false

 lld = true
 default-linker = "{must_getenv('CBUILD')}-clang"
 use-lld = false

 [target.riscv32imc-unknown-none-elf]
 cc = "{rv_clang_bin}/clang"
 cxx = "{rv_clang_bin}/clang++"
 linker = "{rv_clang_bin}/clang++"

 [target.x86_64-unknown-linux-gnu]
 cc = "x86_64-pc-linux-gnu-clang"
 cxx = "x86_64-pc-linux-gnu-clang++"
 linker = "x86_64-pc-linux-gnu-clang++"
 """

   (rust_src / 'config.toml').write_text(toml_contents, encoding='utf-8')


 def compile_rust_src(rust_src: Path) -> None:
   """Compiles all targets specified by COMPONENTS_TO_SHIP."""
   targets = [x.dist_build_name for x in COMPONENTS_TO_SHIP]
   subprocess.check_call(['./x.py', 'dist'] + targets, cwd=rust_src)


 def copy_tree(input_path: Path, output_path: Path,
               ignore_file_names: Iterable[str]):
   """shutil.copytree with dirs_exist_ok=True.

   dirs_exist_ok=True is only available in py3.8+.
   """
   output_path.mkdir(parents=True, exist_ok=True)
   for root, _, files in os.walk(input_path):
     root = Path(root).relative_to(input_path)
     input_root = input_path / root
     output_root = output_path / root
     output_root.mkdir(exist_ok=True)
     for f in files:
       if f in ignore_file_names:
         continue

       output_file = output_root / f
       if output_file.exists():
         raise ValueError(f'File at {output_file} already exists; refusing to '
                          'overwrite')
       logging.debug('Copying %s => %s', input_root / f, output_root / f)
       shutil.copy2(input_root / f, output_root / f)


 def package_rust_src(rust_src: Path, install_dir: Path):
   """Installs all components built in |rust_src| into |install_dir|."""
   dist_path = rust_src / 'build' / 'dist'
   all_tarballs = [x for x in dist_path.iterdir() if x.name.endswith('.tar.gz')]
   temp_dir = Path(tempfile.mkdtemp(prefix='build_rust_package'))

   def install_component(tarball: Path) -> None:
     logging.info('Untarring %s into %s', tarball, temp_dir)

     # All tarballs unpack to a single directory, which is the tarball without
     # the |.tar.${comp}| suffix.
     subprocess.check_call(['tar', 'xaf', str(tarball)], cwd=temp_dir)

     subdir_name = tarball.stem
     tar_suffix = '.tar'
     assert tarball.stem.endswith(tar_suffix), subdir_name
     subdir_name = subdir_name[:-len(tar_suffix)]
     base_dir = temp_dir / subdir_name

     # ...And this directory itself should contain a |components| file
     # describing what components this tarball has.
     components = [
         x.strip()
         for x in (base_dir /
                   'components').read_text(encoding='utf-8').splitlines()
     ]

     for component in components:
       input_path = base_dir / component
       logging.info('Installing component at %s into %s', input_path,
                    install_dir)
       copy_tree(
           input_path,
           install_dir,
           ignore_file_names=('manifest.in',),
       )

       # All inputs have a manifest.in at their root, which describes the files
       # installed by the given component. |rustup| likes to include those in
       # the final installation directory, and that seems like a generally okay
       # idea. These are copied _purely_ for informational purposes, rather than
       # serving some functional goal.
       manifest_dir = install_dir / 'lib' / 'rustlib'
       manifest_dir.mkdir(parents=True, exist_ok=True)
       manifest_name = f'manifest-{input_path.name}'
       shutil.copyfile(input_path / 'manifest.in', manifest_dir / manifest_name)

   try:
     install_dir.mkdir(parents=True, exist_ok=True)

     for component in COMPONENTS_TO_SHIP:
       regex = re.compile(component.dist_tarball_pattern)
       targets = [x for x in all_tarballs if regex.search(x.name)]
       if component.dist_tarballs_are_per_triple:
         for target_triple in TARGET_TRIPLES_TO_SHIP:
           ts = [x for x in targets if target_triple in x.name]
           if len(ts) != 1:
             raise ValueError(f'Expected exactly one match for {component} with '
                              f'triple {target_triple}; got {ts}')
           install_component(ts[0])
       else:
         if len(targets) != 1:
           raise ValueError(f'Expected exactly one match for {component}; '
                            f'got {targets}')
         install_component(targets[0])
   except:
     logging.error('Packaging failed; leaving tempdir around at %s', temp_dir)
     raise
   else:
     logging.info('Packaging succeeded; cleaning up tempdir. Results are in %s',
                  install_dir)
     shutil.rmtree(temp_dir)


 def get_parser():
   """Creates a parser for commandline args."""
   parser = argparse.ArgumentParser(
       description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
   parser.add_argument(
       '--rust-src',
       required=True,
       type=Path,
       help="Path to rust's source code",
   )
   parser.add_argument(
       '--install-dir',
       required=True,
       type=Path,
       help='Path to the directory in which we should put installation '
       'artifacts',
   )
   parser.add_argument(
       '--install-prefix',
       required=True,
       type=Path,
       help='Prefix under which installed artifacts will sit on the system',
   )
   parser.add_argument(
       '--rv-clang-bin',
       required=True,
       type=Path,
       help='Path the bin/ directory of a riscv-enabled clang',
   )
   parser.add_argument(
       '--rustc',
       type=Path,
       help='Optional path to the rustc we should use to bootstrap the '
       'compiler',
   )
   parser.add_argument(
       '--cargo',
       type=Path,
       help='Optional path to the cargo we should use to bootstrap the '
       'compiler',
   )
   parser.add_argument(
       '--rustfmt',
       type=Path,
       help='Optional path to the rustfmt we should use to bootstrap the '
       'compiler',
   )
   return parser


 def main(argv: List[str]):
   logging.basicConfig(
       format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
       level=logging.INFO,
   )

   parser = get_parser()
   opts = parser.parse_args(argv)

   install_dir = opts.install_dir.resolve()
   rust_src = opts.rust_src.resolve()
   rv_clang_bin = opts.rv_clang_bin.resolve()
   # This might not exist yet, so we don't want to resolve() it.
   install_prefix = opts.install_prefix
   if not install_prefix.is_absolute():
     parser.error('--install_prefix should be an absolute path')

   cargo = Path(opts.cargo).resolve() if opts.cargo else None
   rustc = Path(opts.rustc).resolve() if opts.rustc else None
   rustfmt = Path(opts.rustfmt).resolve() if opts.rustfmt else None

   write_config_toml(rust_src, rv_clang_bin, install_prefix, rustc, cargo,
                     rustfmt)
   compile_rust_src(rust_src)
   package_rust_src(rust_src, install_dir)


 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))
	#!/usr/bin/env python3
	# Copyright 2021 The Chromium OS Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.
	"""Builds the Rust toolchain used to compile and run TockOS."""

	import argparse
	import logging
	import os
	from pathlib import Path
	import re
	import shutil
	import subprocess
	import sys
	import tempfile
	from typing import Iterable, List, NamedTuple, Optional

	# The target triples that we want this toolchain to be able to build binaries
	# for. ATM this consists of the host and one target. If you update this,
	# remember to update \|write_config_toml\| with info about the compilers to use.
	#
	# _A_ rust compiler is needed for the host because some packages have build
	# scripts (e.g., build.rs), and Ti50 has previously expressed interest in
	# running _some_ tests in a test environment that runs natively on the host.
	# It's unclear how difficult it'd be to use the CrOS Rust toolchain for both of
	# these, given that said toolchain is locked to the stable channel, and is
	# likely to be a very different version of Rust than Ti50's toolchain.
	TARGET_TRIPLES_TO_SHIP = (
	'riscv32imc-unknown-none-elf',
	'x86_64-unknown-linux-gnu',
	)


	class RustComponent(NamedTuple):
	"""A component of Rust that we want to build and install."""
	# The name of the component that \|./x.py dist\| understands.
	dist_build_name: str
	# A regex pattern that matches the tarball produced for this component under
	# dist/.
	dist_tarball_pattern: str
	# Whether the component's dist tarballs are per-triple.
	dist_tarballs_are_per_triple: bool


	# Components we ship as a part of our toolchain.
	COMPONENTS_TO_SHIP = (
	# cargo, which is shipped with \|rustup\| toolchains by default.
	RustComponent(
	dist_build_name='cargo',
	dist_tarball_pattern=r'^cargo-',
	dist_tarballs_are_per_triple=False,
	),
	# clippy, which was requested by TockOS.
	RustComponent(
	dist_build_name='clippy',
	dist_tarball_pattern=r'^clippy-',
	dist_tarballs_are_per_triple=False,
	),
	# llvm-tools-preview, which is apparently required by TockOS itself.
	RustComponent(
	dist_build_name='llvm-tools',
	dist_tarball_pattern=r'^llvm-tools-',
	dist_tarballs_are_per_triple=False,
	),
	# src, which is required when cross-compiling in some contexts.
	RustComponent(
	dist_build_name='src',
	dist_tarball_pattern=r'^rust-src-',
	dist_tarballs_are_per_triple=False,
	),
	# src/librustc, which is the actual \|rustc\| compiler.
	RustComponent(
	dist_build_name='src/librustc',
	dist_tarball_pattern=r'^rustc-[^-]+-dev-(?!src)',
	dist_tarballs_are_per_triple=False,
	),
	# rustfmt, the standard formatter for rust code.
	RustComponent(
	dist_build_name='rustfmt',
	dist_tarball_pattern=r'^rustfmt-',
	dist_tarballs_are_per_triple=False,
	),
	# library/std, which consists of stdlib \|rlib\|s.
	RustComponent(
	dist_build_name='library/std',
	dist_tarball_pattern=r'^rust-std-',
	dist_tarballs_are_per_triple=True,
	),
	)


	def must_getenv(key: str) -> str:
	"""Gets the value of an environment variable; raises if unset or empty."""
	x = os.getenv(key)
	if not x:
	raise ValueError(f'No value found for {key}; this env var is required')
	return x


	def write_config_toml(rust_src: Path, rv_clang_bin: Path, install_prefix: Path,
	rustc: Optional[Path], cargo: Optional[Path],
	rustfmt: Optional[Path]) -> None:
	"""Writes Rust's config.toml with the given parameters."""
	# We need a Pythony list, but formatted with double-quotes.
	assert not any('"' in x or '\\' in x
	for x in TARGET_TRIPLES_TO_SHIP), TARGET_TRIPLES_TO_SHIP
	target_triples_str = ', '.join(f'"{x}"' for x in TARGET_TRIPLES_TO_SHIP)

	component_locations = (
	('rustc', rustc),
	('cargo', cargo),
	('rustfmt', rustfmt),
	)
	optional_bootstrap_configuration = '\n'.join(
	f'{key} = "{exe}"' for key, exe in component_locations if exe is not None)

	toml_contents = f"""
	[build]
	target = [{target_triples_str}]
	extended = true
	vendor = true
	python = "{must_getenv('EPYTHON')}"
	submodules = false
	profiler = true
	{optional_bootstrap_configuration}

	[llvm]
	use-libcxx = true
	ninja = true

	[install]
	prefix = "{install_prefix}"

	[rust]
	# Provide llvm-tools like llvm-objcopy for the llvm-tools package.
	llvm-tools = true
	codegen-units = 0
	codegen-tests = false

	lld = true
	default-linker = "{must_getenv('CBUILD')}-clang"
	use-lld = false

	[target.riscv32imc-unknown-none-elf]
	cc = "{rv_clang_bin}/clang"
	cxx = "{rv_clang_bin}/clang++"
	linker = "{rv_clang_bin}/clang++"

	[target.x86_64-unknown-linux-gnu]
	cc = "x86_64-pc-linux-gnu-clang"
	cxx = "x86_64-pc-linux-gnu-clang++"
	linker = "x86_64-pc-linux-gnu-clang++"
	"""

	(rust_src / 'config.toml').write_text(toml_contents, encoding='utf-8')


	def compile_rust_src(rust_src: Path) -> None:
	"""Compiles all targets specified by COMPONENTS_TO_SHIP."""
	targets = [x.dist_build_name for x in COMPONENTS_TO_SHIP]
	subprocess.check_call(['./x.py', 'dist'] + targets, cwd=rust_src)


	def copy_tree(input_path: Path, output_path: Path,
	ignore_file_names: Iterable[str]):
	"""shutil.copytree with dirs_exist_ok=True.

	dirs_exist_ok=True is only available in py3.8+.
	"""
	output_path.mkdir(parents=True, exist_ok=True)
	for root, _, files in os.walk(input_path):
	root = Path(root).relative_to(input_path)
	input_root = input_path / root
	output_root = output_path / root
	output_root.mkdir(exist_ok=True)
	for f in files:
	if f in ignore_file_names:
	continue

	output_file = output_root / f
	if output_file.exists():
	raise ValueError(f'File at {output_file} already exists; refusing to '
	'overwrite')
	logging.debug('Copying %s => %s', input_root / f, output_root / f)
	shutil.copy2(input_root / f, output_root / f)


	def package_rust_src(rust_src: Path, install_dir: Path):
	"""Installs all components built in \|rust_src\| into \|install_dir\|."""
	dist_path = rust_src / 'build' / 'dist'
	all_tarballs = [x for x in dist_path.iterdir() if x.name.endswith('.tar.gz')]
	temp_dir = Path(tempfile.mkdtemp(prefix='build_rust_package'))

	def install_component(tarball: Path) -> None:
	logging.info('Untarring %s into %s', tarball, temp_dir)

	# All tarballs unpack to a single directory, which is the tarball without
	# the \|.tar.${comp}\| suffix.
	subprocess.check_call(['tar', 'xaf', str(tarball)], cwd=temp_dir)

	subdir_name = tarball.stem
	tar_suffix = '.tar'
	assert tarball.stem.endswith(tar_suffix), subdir_name
	subdir_name = subdir_name[:-len(tar_suffix)]
	base_dir = temp_dir / subdir_name

	# ...And this directory itself should contain a \|components\| file
	# describing what components this tarball has.
	components = [
	x.strip()
	for x in (base_dir /
	'components').read_text(encoding='utf-8').splitlines()
	]

	for component in components:
	input_path = base_dir / component
	logging.info('Installing component at %s into %s', input_path,
	install_dir)
	copy_tree(
	input_path,
	install_dir,
	ignore_file_names=('manifest.in',),
	)

	# All inputs have a manifest.in at their root, which describes the files
	# installed by the given component. \|rustup\| likes to include those in
	# the final installation directory, and that seems like a generally okay
	# idea. These are copied _purely_ for informational purposes, rather than
	# serving some functional goal.
	manifest_dir = install_dir / 'lib' / 'rustlib'
	manifest_dir.mkdir(parents=True, exist_ok=True)
	manifest_name = f'manifest-{input_path.name}'
	shutil.copyfile(input_path / 'manifest.in', manifest_dir / manifest_name)

	try:
	install_dir.mkdir(parents=True, exist_ok=True)

	for component in COMPONENTS_TO_SHIP:
	regex = re.compile(component.dist_tarball_pattern)
	targets = [x for x in all_tarballs if regex.search(x.name)]
	if component.dist_tarballs_are_per_triple:
	for target_triple in TARGET_TRIPLES_TO_SHIP:
	ts = [x for x in targets if target_triple in x.name]
	if len(ts) != 1:
	raise ValueError(f'Expected exactly one match for {component} with '
	f'triple {target_triple}; got {ts}')
	install_component(ts[0])
	else:
	if len(targets) != 1:
	raise ValueError(f'Expected exactly one match for {component}; '
	f'got {targets}')
	install_component(targets[0])
	except:
	logging.error('Packaging failed; leaving tempdir around at %s', temp_dir)
	raise
	else:
	logging.info('Packaging succeeded; cleaning up tempdir. Results are in %s',
	install_dir)
	shutil.rmtree(temp_dir)


	def get_parser():
	"""Creates a parser for commandline args."""
	parser = argparse.ArgumentParser(
	description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
	parser.add_argument(
	'--rust-src',
	required=True,
	type=Path,
	help="Path to rust's source code",
	)
	parser.add_argument(
	'--install-dir',
	required=True,
	type=Path,
	help='Path to the directory in which we should put installation '
	'artifacts',
	)
	parser.add_argument(
	'--install-prefix',
	required=True,
	type=Path,
	help='Prefix under which installed artifacts will sit on the system',
	)
	parser.add_argument(
	'--rv-clang-bin',
	required=True,
	type=Path,
	help='Path the bin/ directory of a riscv-enabled clang',
	)
	parser.add_argument(
	'--rustc',
	type=Path,
	help='Optional path to the rustc we should use to bootstrap the '
	'compiler',
	)
	parser.add_argument(
	'--cargo',
	type=Path,
	help='Optional path to the cargo we should use to bootstrap the '
	'compiler',
	)
	parser.add_argument(
	'--rustfmt',
	type=Path,
	help='Optional path to the rustfmt we should use to bootstrap the '
	'compiler',
	)
	return parser


	def main(argv: List[str]):
	logging.basicConfig(
	format='%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: %(message)s',
	level=logging.INFO,
	)

	parser = get_parser()
	opts = parser.parse_args(argv)

	install_dir = opts.install_dir.resolve()
	rust_src = opts.rust_src.resolve()
	rv_clang_bin = opts.rv_clang_bin.resolve()
	# This might not exist yet, so we don't want to resolve() it.
	install_prefix = opts.install_prefix
	if not install_prefix.is_absolute():
	parser.error('--install_prefix should be an absolute path')

	cargo = Path(opts.cargo).resolve() if opts.cargo else None
	rustc = Path(opts.rustc).resolve() if opts.rustc else None
	rustfmt = Path(opts.rustfmt).resolve() if opts.rustfmt else None

	write_config_toml(rust_src, rv_clang_bin, install_prefix, rustc, cargo,
	rustfmt)
	compile_rust_src(rust_src)
	package_rust_src(rust_src, install_dir)


	if __name__ == '__main__':
	sys.exit(main(sys.argv[1:]))