blob: 48df3cfb0f6f06c515dcaed7828d916db908e73a [file] [log] [blame]
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Deps analysis service."""
from __future__ import print_function
import functools
import os
from pathlib import Path
from typing import List, Optional
from chromite.lib import build_target_lib
from chromite.lib import constants
from chromite.lib import cros_build_lib
from chromite.lib import dependency_lib
from chromite.lib import git
from chromite.lib import portage_util
from chromite.scripts import cros_extract_deps
class Error(Exception):
"""Base error class for the module."""
def NormalizeSourcePaths(source_paths):
"""Return the "normalized" form of a list of source paths.
Normalizing includes:
* Sorting the source paths in alphabetical order.
* Remove paths that are sub-path of others in the source paths.
* Ensure all the directory path strings are ended with the trailing '/'.
* Convert all the path from absolute paths to relative path (relative to
the chroot source root).
return dependency_lib.normalize_source_paths(source_paths)
def GenerateSourcePathMapping(packages, sysroot_path, board):
"""Returns a map from each package to the source paths it depends on.
A source path is considered dependency of a package if modifying files in that
path might change the content of the resulting package.
1) This method errs on the side of returning unneeded dependent paths.
i.e: for a given package X, some of its dependency source paths may
contain files which doesn't affect the content of X.
On the other hands, any missing dependency source paths for package X is
considered a bug.
2) This only outputs the direct dependency source paths for a given package
and does not takes include the dependency source paths of dependency
e.g: if package A depends on B (DEPEND=B), then results of computing
dependency source paths of A doesn't include dependency source paths
of B.
packages: The list of packages CPV names (str)
sysroot_path (str): The path to the sysroot. If the packages are board
agnostic, then this should be '/'.
board (str): The name of the board if packages are dependency of board. If
the packages are board agnostic, then this should be None.
Map from each package to the source path (relative to the repo checkout
root, i.e: ~/trunk/ in your cros_sdk) it depends on.
For each source path which is a directory, the string is ended with a
trailing '/'.
return dependency_lib.get_source_path_mapping(packages, sysroot_path, board)
def GetBuildDependency(sysroot_path, board=None, packages=None):
"""Return the build dependency and package -> source path map for |board|.
sysroot_path (str): The path to the sysroot, or None if no sysroot is being
board (str): The name of the board whose artifacts are being created, or
None if no sysroot is being used.
packages (tuple[CPV]): The packages that need to be built, or empty / None
to use the default list.
JSON build dependencies report for the given board which includes:
- Package level deps graph from portage
- Map from each package to the source path
(relative to the repo checkout root, i.e: ~/trunk/ in your cros_sdk) it
depends on
if not sysroot_path:
sysroot_path = cros_build_lib.GetSysroot(board)
results = {
'sysroot_path': sysroot_path,
'target_board': board,
'package_deps': {},
'source_path_mapping': {},
sdk_sysroot = cros_build_lib.GetSysroot(None)
sdk_results = {
'sysroot_path': sdk_sysroot,
'target_board': 'sdk',
'package_deps': {},
'source_path_mapping': {},
if sysroot_path != sdk_sysroot:
board_packages = []
if packages:
board_packages.extend([cpv.cp for cpv in packages])
# Since we don’t have a clear mapping from autotests to git repos
# and/or portage packages, we assume every board run all autotests.
board_packages += ['chromeos-base/autotest-all']
board_deps, board_bdeps = cros_extract_deps.ExtractDeps(
indep_packages = [
'virtual/target-sdk', 'virtual/target-sdk-post-cross',
indep_deps, _ = cros_extract_deps.ExtractDeps(
sysroot=sdk_results['sysroot_path'], package_list=indep_packages)
indep_map = GenerateSourcePathMapping(list(indep_deps), sdk_sysroot, None)
if sysroot_path != sdk_sysroot:
bdep_map = GenerateSourcePathMapping(list(board_bdeps), sdk_sysroot, None)
board_map = GenerateSourcePathMapping(list(board_deps), sysroot_path, board)
return results, sdk_results
def determine_package_relevance(dep_src_paths: List[str],
src_paths: Optional[List[str]] = None) -> bool:
"""Determine if the package is relevant to the given source paths.
A package is relevant if any of its dependent source paths is in the given
list of source paths. If no source paths are given, the default is True.
dep_src_paths: List of source paths the package depends on.
src_paths: List of source paths of interest.
if not src_paths:
return True
for src_path in (Path(x) for x in src_paths):
for dep_src_path in (Path(x) for x in dep_src_paths):
# Will throw an error if src_path isn't under dep_src_path.
return True
except ValueError:
return False
def GetDependencies(sysroot_path: str,
build_target: build_target_lib.BuildTarget,
src_paths: Optional[List[str]] = None,
packages: Optional[List[str]] = None) -> List[str]:
"""Return the packages dependent on the given source paths for |board|.
sysroot_path: The path to the sysroot.
build_target: The build_target whose dependencies are being calculated.
src_paths: List of paths for which to get a list of dependent packages. If
empty / None returns all package dependencies.
packages: The packages that need to be built, or empty / None to use the
default list.
The relevant package dependencies based on the given list of packages and
pkgs = tuple(packages) if packages else None
json_deps, _sdk_json_deps = GetBuildDependency(
sysroot_path,, packages=pkgs)
relevant_packages = set()
for cpv, dep_src_paths in json_deps['source_path_mapping'].items():
if determine_package_relevance(dep_src_paths, src_paths):
return relevant_packages
def DetermineToolchainSourcePaths():
"""Returns a list of all source paths relevant to toolchain packages.
A package is a 'toolchain package' if it is listed as a direct dependency
of virtual/toolchain-packages. This function deliberately does not return
deeper transitive dependencies so that only direct changes to toolchain
packages trigger the expensive full re-compilation required to test toolchain
changes. Eclasses & overlays are not returned as relevant paths for the same
Returned paths are relative to the root of the project checkout.
List[str]: A list of paths considered relevant to toolchain packages.
source_paths = set()
toolchain_pkgs = portage_util.GetFlattenedDepsForPackage(
'virtual/toolchain-packages', depth=1)
toolchain_pkg_ebuilds = portage_util.FindEbuildsForPackages(
toolchain_pkgs, sysroot='/', check=True)
# Include the entire directory containing the toolchain ebuild, as the
# package's FILESDIR and patches also live there.
for ebuild_path in toolchain_pkg_ebuilds.values())
# Source paths which are cros workon source paths.
buildroot = os.path.join(constants.CHROOT_SOURCE_ROOT, 'src')
manifest = git.ManifestCheckout.Cached(buildroot)
for ebuild_path in toolchain_pkg_ebuilds.values():
attrs = portage_util.EBuild.Classify(ebuild_path)
if (not attrs.is_workon or
# Blacklisted ebuild is pinned to a specific git sha1, so change in
# that repo matter to the ebuild.
ebuild = portage_util.EBuild(ebuild_path)
workon_subtrees = ebuild.GetSourceInfo(buildroot, manifest).subtrees
for path in workon_subtrees:
return NormalizeSourcePaths(list(source_paths))