common-mk/check-source-files-included.py - mirrors/cros/chromiumos/platform2 - Git at Google

 #!/usr/bin/env python3
 # Copyright 2023 The ChromiumOS Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """Ensure source files are not missed and included in a gn file.

 Occasionally, a source file, especially a unit test file, is added to the repo
 but is not included in the build. This kind of mistake can often evade code
 review, or is discovered only later in the review stage and wasted time for
 early discovery of errors. This script examines each source file and checks
 whether they are present in a *.gn file in their project directory.

 Additionally, when a build file is changed, occasionally some source files
 (especially test files) are left unbuilt. This script also checks for source
 files that are not included anywhere when a build file is modified.
 """

 import argparse
 import os
 from pathlib import Path
 from pathlib import PurePath
 import sys
 from typing import FrozenSet, Iterable, List, Optional, Tuple, Union


 _HACK_VAR_TO_DISABLE_ISORT = "hack"
 # pylint: disable=wrong-import-position
 import chromite_init  # pylint: disable=unused-import

 from chromite.lib import cros_build_lib
 from chromite.lib import git
 from chromite.lint.linters import gnlint


 TOP_DIR = Path(__file__).resolve().parent.parent
 SOURCE_FILE_SUFFICES = (".c", ".cc", ".cpp", ".cxx")

 # Not all platform2 sources are intended to appear in a cros build (e.g.
 # examples and vm-specific code), so exclude those.
 EXCLUDED_DIRS = (
     "common-mk/example/",
     "vm_tools/cros_im",
     "vm_tools/reference_vm/data/",
     "vm_tools/sommelier/quirks/",
 )


 class ProjectLiterals:
     """Manages project literals."""

     def __init__(self, commit: str):
         # A map that saves project literals.
         self._literals: map = {}
         self._commit: str = commit

     def GetLiterals(self, project: str) -> FrozenSet[PurePath]:
         """Get the literal source files from a project.

         If the literals have not gathered yet, gather them.

         Args:
             project: The project to get literals from.

         Returns:
             Literals from the specified project.
         """
         if project not in self._literals:
             self._literals[project] = self._GatherLiteralsFromProject(project)

         return self._literals[project]

     def _GatherLiteralsFromProject(self, project: str) -> FrozenSet[PurePath]:
         """Gather the literal source files from a project.

         Args:
             project: The project to gather source file literals from.

         Returns:
             A set of all source file literals from a project.
         """

         def _gather() -> Iterable[Tuple[PurePath, PurePath]]:
             """Generate the gn file path and its source file literals."""
             for gn_file in self._FindGnFiles(project):
                 yield from (
                     (gn_file, literal)
                     for literal in ProjectLiterals._GatherLiteralsFromGn(
                         self._ReadFileAtCommit(project, gn_file)
                     )
                 )

         def _resolve(gn_file: PurePath, literal: PurePath) -> PurePath:
             """Resolve a source file literal relative to the top dir."""
             return (
                 (
                     TOP_DIR
                     / project
                     / PurePath(gn_file).parent
                     / gnlint.GetNodeValue(literal)
                 )
                 .resolve()
                 .relative_to(TOP_DIR)
             )

         return frozenset(
             _resolve(gn_file, literal) for gn_file, literal in _gather()
         )

     def _ReadFileAtCommit(self, project: str, file_path: os.PathLike) -> str:
         """Read a file at a commit.

         Args:
             project: The project to read the file from.
             file_path: The path to the file to read. Path is relative to the
             project root directory.

         Returns:
             The content of the file.
         """
         return git.GetObjectAtRev(
             TOP_DIR / project, project / file_path, self._commit
         )

     def _FindFilesEndingWith(
         self, project: str, suffix: Union[str, Tuple[str]]
     ) -> Iterable[PurePath]:
         """Find all gn files in a project.

         Args:
             project: The project to find files in.
             suffix: The suffix that files end with. Can also be a tuple of
                     suffices.

         Returns:
             An iterable of all files ending with the given suffix.
         """
         yield from (
             f.name
             for f in git.LsTree(TOP_DIR / project, self._commit)
             if f.is_file and f.name.name.endswith(suffix)
         )

     def _FindGnFiles(self, project: str) -> Iterable[PurePath]:
         """Find all gn files in a project.

         Args:
             project: The project to find gn files in.

         Returns:
             An iterable of all gn files.
         """
         yield from self._FindFilesEndingWith(project, ".gn")

     def FindSourceFiles(self, project: str) -> Iterable[PurePath]:
         """Find all source files in a project.

         Args:
             project: The project to find source files in.

         Returns:
             An iterable of all source files.
         """
         yield from self._FindFilesEndingWith(project, SOURCE_FILE_SUFFICES)

     @staticmethod
     def _GatherLiteralsFromGn(gn_data: str) -> List[dict]:
         """Gather all source file literals from a gn file.

         Args:
             gn_data: The content of a gn file to gather literals from.

         Returns:
             A list of literal assignments.
         """
         try:
             ast = gnlint.ParseAst(gn_data)
         except cros_build_lib.RunCommandError as e:
             cros_build_lib.Die("Failed to run gn format: %s", e)
         except Exception as e:
             cros_build_lib.Die("Invalid format: %s", e)

         return gnlint.FindAllLiteralAssignments(
             ast, ["sources"], operators=["=", "+="]
         )


 def CheckSourceFileIncludedInBuild(
     commit: str, file_paths: Iterable[Union[str, os.PathLike]]
 ) -> bool:
     """Check that source files are included in builds.

     Args:
         commit: The commit to check in.
         file_paths: Files modified in this commit. Non-source files in this list
                     would be ignored.

     Returns:
         True if source files are included in a *.gn file in the project
         directory. False otherwise.
     """

     ret = True
     project_literals = ProjectLiterals(commit)

     for path in file_paths:
         path = PurePath(path)
         if not path.name.endswith(SOURCE_FILE_SUFFICES):
             # Header files are not checked here because they do not necessarily
             # need to be present in a build file.
             continue

         if str(path).startswith(EXCLUDED_DIRS):
             # List of files that don't need to be checked (e.g. examples).
             continue

         project = path.parts[0]
         if not (Path(project) / "BUILD.gn").exists():
             # This project does not use gn.
             # We are trying to be conservative here, as we don't want to check
             # projects that only uses gn in a subdirectory.
             continue
         if path not in project_literals.GetLiterals(project):
             print(
                 f"{__file__}: {path} is not included in any "
                 f"*.gn files in {project}. "
                 "If you have added the file via an intermediate variable, "
                 "please ensure the source is set via source_set().",
                 file=sys.stderr,
             )
             ret = False

     return ret


 def CheckBuildFileIncludingAllSourceFiles(
     commit: str, file_paths: Iterable[str]
 ) -> bool:
     """Check that BUILD.gn files including all source files.

     Args:
         commit: The commit to check in.
         file_paths: Files modified in this commit. Non-build files in this list
                     would be ignored.
     """
     ret = True
     project_literals = ProjectLiterals(commit)

     # Calling CheckSourceFileIncludedInBuild with all source files in that
     # project as parameter to examine if all source files are included.
     for path in file_paths:
         if not path.endswith(".gn"):
             continue

         project = PurePath(path).parts[0]
         if not CheckSourceFileIncludedInBuild(
             commit,
             (project / f for f in project_literals.FindSourceFiles(project)),
         ):
             ret = False

     return ret


 def get_parser():
     """Return an argument parser."""
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--commit", help="Hash of commit to check in.")
     parser.add_argument("files", nargs="*", help="Files to check.")
     return parser


 def main(argv: Optional[List[str]] = None) -> Optional[int]:
     parser = get_parser()
     opts = parser.parse_args(argv)
     return (
         0
         if (
             CheckSourceFileIncludedInBuild(opts.commit, opts.files)
             and CheckBuildFileIncludingAllSourceFiles(opts.commit, opts.files)
         )
         else 1
     )


 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))
	#!/usr/bin/env python3
	# Copyright 2023 The ChromiumOS Authors
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""Ensure source files are not missed and included in a gn file.

	Occasionally, a source file, especially a unit test file, is added to the repo
	but is not included in the build. This kind of mistake can often evade code
	review, or is discovered only later in the review stage and wasted time for
	early discovery of errors. This script examines each source file and checks
	whether they are present in a *.gn file in their project directory.

	Additionally, when a build file is changed, occasionally some source files
	(especially test files) are left unbuilt. This script also checks for source
	files that are not included anywhere when a build file is modified.
	"""

	import argparse
	import os
	from pathlib import Path
	from pathlib import PurePath
	import sys
	from typing import FrozenSet, Iterable, List, Optional, Tuple, Union


	_HACK_VAR_TO_DISABLE_ISORT = "hack"
	# pylint: disable=wrong-import-position
	import chromite_init # pylint: disable=unused-import

	from chromite.lib import cros_build_lib
	from chromite.lib import git
	from chromite.lint.linters import gnlint


	TOP_DIR = Path(__file__).resolve().parent.parent
	SOURCE_FILE_SUFFICES = (".c", ".cc", ".cpp", ".cxx")

	# Not all platform2 sources are intended to appear in a cros build (e.g.
	# examples and vm-specific code), so exclude those.
	EXCLUDED_DIRS = (
	"common-mk/example/",
	"vm_tools/cros_im",
	"vm_tools/reference_vm/data/",
	"vm_tools/sommelier/quirks/",
	)


	class ProjectLiterals:
	"""Manages project literals."""

	def __init__(self, commit: str):
	# A map that saves project literals.
	self._literals: map = {}
	self._commit: str = commit

	def GetLiterals(self, project: str) -> FrozenSet[PurePath]:
	"""Get the literal source files from a project.

	If the literals have not gathered yet, gather them.

	Args:
	project: The project to get literals from.

	Returns:
	Literals from the specified project.
	"""
	if project not in self._literals:
	self._literals[project] = self._GatherLiteralsFromProject(project)

	return self._literals[project]

	def _GatherLiteralsFromProject(self, project: str) -> FrozenSet[PurePath]:
	"""Gather the literal source files from a project.

	Args:
	project: The project to gather source file literals from.

	Returns:
	A set of all source file literals from a project.
	"""

	def _gather() -> Iterable[Tuple[PurePath, PurePath]]:
	"""Generate the gn file path and its source file literals."""
	for gn_file in self._FindGnFiles(project):
	yield from (
	(gn_file, literal)
	for literal in ProjectLiterals._GatherLiteralsFromGn(
	self._ReadFileAtCommit(project, gn_file)
	)
	)

	def _resolve(gn_file: PurePath, literal: PurePath) -> PurePath:
	"""Resolve a source file literal relative to the top dir."""
	return (
	(
	TOP_DIR
	/ project
	/ PurePath(gn_file).parent
	/ gnlint.GetNodeValue(literal)
	)
	.resolve()
	.relative_to(TOP_DIR)
	)

	return frozenset(
	_resolve(gn_file, literal) for gn_file, literal in _gather()
	)

	def _ReadFileAtCommit(self, project: str, file_path: os.PathLike) -> str:
	"""Read a file at a commit.

	Args:
	project: The project to read the file from.
	file_path: The path to the file to read. Path is relative to the
	project root directory.

	Returns:
	The content of the file.
	"""
	return git.GetObjectAtRev(
	TOP_DIR / project, project / file_path, self._commit
	)

	def _FindFilesEndingWith(
	self, project: str, suffix: Union[str, Tuple[str]]
	) -> Iterable[PurePath]:
	"""Find all gn files in a project.

	Args:
	project: The project to find files in.
	suffix: The suffix that files end with. Can also be a tuple of
	suffices.

	Returns:
	An iterable of all files ending with the given suffix.
	"""
	yield from (
	f.name
	for f in git.LsTree(TOP_DIR / project, self._commit)
	if f.is_file and f.name.name.endswith(suffix)
	)

	def _FindGnFiles(self, project: str) -> Iterable[PurePath]:
	"""Find all gn files in a project.

	Args:
	project: The project to find gn files in.

	Returns:
	An iterable of all gn files.
	"""
	yield from self._FindFilesEndingWith(project, ".gn")

	def FindSourceFiles(self, project: str) -> Iterable[PurePath]:
	"""Find all source files in a project.

	Args:
	project: The project to find source files in.

	Returns:
	An iterable of all source files.
	"""
	yield from self._FindFilesEndingWith(project, SOURCE_FILE_SUFFICES)

	@staticmethod
	def _GatherLiteralsFromGn(gn_data: str) -> List[dict]:
	"""Gather all source file literals from a gn file.

	Args:
	gn_data: The content of a gn file to gather literals from.

	Returns:
	A list of literal assignments.
	"""
	try:
	ast = gnlint.ParseAst(gn_data)
	except cros_build_lib.RunCommandError as e:
	cros_build_lib.Die("Failed to run gn format: %s", e)
	except Exception as e:
	cros_build_lib.Die("Invalid format: %s", e)

	return gnlint.FindAllLiteralAssignments(
	ast, ["sources"], operators=["=", "+="]
	)


	def CheckSourceFileIncludedInBuild(
	commit: str, file_paths: Iterable[Union[str, os.PathLike]]
	) -> bool:
	"""Check that source files are included in builds.

	Args:
	commit: The commit to check in.
	file_paths: Files modified in this commit. Non-source files in this list
	would be ignored.

	Returns:
	True if source files are included in a *.gn file in the project
	directory. False otherwise.
	"""

	ret = True
	project_literals = ProjectLiterals(commit)

	for path in file_paths:
	path = PurePath(path)
	if not path.name.endswith(SOURCE_FILE_SUFFICES):
	# Header files are not checked here because they do not necessarily
	# need to be present in a build file.
	continue

	if str(path).startswith(EXCLUDED_DIRS):
	# List of files that don't need to be checked (e.g. examples).
	continue

	project = path.parts[0]
	if not (Path(project) / "BUILD.gn").exists():
	# This project does not use gn.
	# We are trying to be conservative here, as we don't want to check
	# projects that only uses gn in a subdirectory.
	continue
	if path not in project_literals.GetLiterals(project):
	print(
	f"{__file__}: {path} is not included in any "
	f"*.gn files in {project}. "
	"If you have added the file via an intermediate variable, "
	"please ensure the source is set via source_set().",
	file=sys.stderr,
	)
	ret = False

	return ret


	def CheckBuildFileIncludingAllSourceFiles(
	commit: str, file_paths: Iterable[str]
	) -> bool:
	"""Check that BUILD.gn files including all source files.

	Args:
	commit: The commit to check in.
	file_paths: Files modified in this commit. Non-build files in this list
	would be ignored.
	"""
	ret = True
	project_literals = ProjectLiterals(commit)

	# Calling CheckSourceFileIncludedInBuild with all source files in that
	# project as parameter to examine if all source files are included.
	for path in file_paths:
	if not path.endswith(".gn"):
	continue

	project = PurePath(path).parts[0]
	if not CheckSourceFileIncludedInBuild(
	commit,
	(project / f for f in project_literals.FindSourceFiles(project)),
	):
	ret = False

	return ret


	def get_parser():
	"""Return an argument parser."""
	parser = argparse.ArgumentParser(description=__doc__)
	parser.add_argument("--commit", help="Hash of commit to check in.")
	parser.add_argument("files", nargs="*", help="Files to check.")
	return parser


	def main(argv: Optional[List[str]] = None) -> Optional[int]:
	parser = get_parser()
	opts = parser.parse_args(argv)
	return (
	0
	if (
	CheckSourceFileIncludedInBuild(opts.commit, opts.files)
	and CheckBuildFileIncludingAllSourceFiles(opts.commit, opts.files)
	)
	else 1
	)


	if __name__ == "__main__":
	sys.exit(main(sys.argv[1:]))