toolchain: upload Compiler Rusage Logs
We need to upload the resource usage for compiler invocations
in order to monitor for regressions.
BUG=chromium:1156314
TEST=./run_pytest
Change-Id: I16c8ecc39899cd1dbf6ffe936bf68f64745ddca7
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/2582963
Tested-by: Ryan Beltran <ryanbeltran@chromium.org>
Commit-Queue: Ryan Beltran <ryanbeltran@chromium.org>
Reviewed-by: Tiancong Wang <tcwang@google.com>
Reviewed-by: Alex Klein <saklein@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Reviewed-by: LaMont Jones <lamontjones@chromium.org>
diff --git a/api/controller/toolchain.py b/api/controller/toolchain.py
index bbab194..4007f10 100644
--- a/api/controller/toolchain.py
+++ b/api/controller/toolchain.py
@@ -76,6 +76,9 @@
BuilderConfig.Artifacts.CLANG_CRASH_DIAGNOSES:
_Handlers('ClangCrashDiagnoses', toolchain_util.PrepareForBuild,
toolchain_util.BundleArtifacts),
+ BuilderConfig.Artifacts.COMPILER_RUSAGE_LOG:
+ _Handlers('CompilerRusageLogs', toolchain_util.PrepareForBuild,
+ toolchain_util.BundleArtifacts),
}
diff --git a/lib/toolchain_util.py b/lib/toolchain_util.py
index ca24cf6..5c10686 100644
--- a/lib/toolchain_util.py
+++ b/lib/toolchain_util.py
@@ -203,6 +203,10 @@
"""Error for BundleArtifactsHandler class."""
+class NoArtifactsToBundleError(Error):
+ """Error for bundling empty collection of artifacts."""
+
+
class GenerateChromeOrderfileError(Error):
"""Error for GenerateChromeOrderfile class."""
@@ -2094,6 +2098,12 @@
self._CleanupArtifactDirectory('/tmp/clang_crash_diagnostics')
return PrepareForBuildReturn.UNKNOWN
+ def _PrepareCompilerRusageLogs(self):
+ # We always build this artifact.
+ # Cleanup the temp directory that holds the artifacts
+ self._CleanupArtifactDirectory('/tmp/compiler_rusage')
+ return PrepareForBuildReturn.UNKNOWN
+
class BundleArtifactHandler(_CommonPrepareBundle):
"""Methods for updating ebuilds for toolchain artifacts."""
@@ -2456,42 +2466,87 @@
logging.info('%d files collected', len(output))
return output
+ def _CreateBundle(self, src_dir, tarball, destination, extension=None):
+ """Bundle the files from src_dir into a tar.xz file.
+
+ Args:
+ src_dir: the path to the directory to copy files from.
+ tarball: name of the generated tarballfile (build target, time stamp,
+ and .tar.xz extension will be added automatically)
+ destination: path to create tarball in
+ extension: type of file to search for in src_dir.
+ If extension is None (default), all file types will be allowed.
+
+ Returns:
+ Path to the generated tar.xz file
+ """
+
+ def FilterFile(file_path):
+ return extension is None or file_path.endswith(extension)
+
+ files = self._CollectFiles(
+ src_dir,
+ destination,
+ include_file=FilterFile)
+ if not files:
+ logging.info('No data found for %s, skip bundle artifact', tarball)
+ raise NoArtifactsToBundleError(f'No {extension} files in {src_dir}')
+
+ now = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')
+ name = f'{self.build_target}.{now}.{tarball}.tar.xz'
+ output_compressed = os.path.join(self.output_dir, name)
+ cros_build_lib.CreateTarball(output_compressed, destination, inputs=files)
+
+ return output_compressed
+
def _BundleToolchainWarningLogs(self):
"""Bundle the compiler warnings for upload for werror checker."""
with self.chroot.tempdir() as tempdir:
- warning_files = self._CollectFiles(
- '/tmp/fatal_clang_warnings',
- tempdir,
- include_file=lambda file_path: file_path.endswith('.json'))
-
- if not warning_files:
- logging.info('No fatal-clang-warnings found, skip bundle artifact')
+ try:
+ return [
+ self._CreateBundle(
+ '/tmp/fatal_clang_warnings',
+ 'fatal_clang_warnings',
+ tempdir,
+ '.json')
+ ]
+ except NoArtifactsToBundleError:
return []
- now = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')
- name = f'{self.build_target}.{now}.fatal_clang_warnings.tar.xz'
- output_compressed = os.path.join(self.output_dir, name)
- cros_build_lib.CreateTarball(
- output_compressed, tempdir, inputs=warning_files)
-
- return [output_compressed]
def _BundleClangCrashDiagnoses(self):
- """Bundle all clang crash diagnoses in chroot for uploading."""
- with osutils.TempDir(prefix='clang_crash_diagnoses_tarball') as tempdir:
- diagnoses = self._CollectFiles(
- '/tmp/clang_crash_diagnostics', tempdir, include_file=lambda _: True)
+ """Bundle all clang crash diagnoses in chroot for uploading.
- if not diagnoses:
- logging.info('No clang crashes found, skip bundle artifact')
+ See bugs.chromium.org/p/chromium/issues/detail?id=1056904 for context.
+ """
+ with osutils.TempDir(prefix='clang_crash_diagnoses_tarball') as tempdir:
+ try:
+ return [
+ self._CreateBundle(
+ '/tmp/clang_crash_diagnostics',
+ 'clang_crash_diagnoses',
+ tempdir)
+ ]
+ except NoArtifactsToBundleError:
return []
- now = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')
- output = os.path.join(
- self.output_dir,
- f'{self.build_target}.{now}.clang_crash_diagnoses.tar.xz')
- cros_build_lib.CreateTarball(output, tempdir, inputs=diagnoses)
- return [output]
+ def _BundleCompilerRusageLogs(self):
+ """Bundle the rusage files created by compiler invocations.
+ This is useful for monitoring changes in compiler performance.
+ These files are created when the TOOLCHAIN_RUSAGE_OUTPUT variable
+ is set in the environment for monitoring compiler performance.
+ """
+ with self.chroot.tempdir() as tempdir:
+ try:
+ return [
+ self._CreateBundle(
+ '/tmp/compiler_rusage',
+ 'compiler_rusage_logs',
+ tempdir,
+ '.json')
+ ]
+ except NoArtifactsToBundleError:
+ return []
def PrepareForBuild(artifact_name, chroot, sysroot_path, build_target,
input_artifacts, profile_info):
diff --git a/lib/toolchain_util_unittest.py b/lib/toolchain_util_unittest.py
index 8bdde76..643016d 100644
--- a/lib/toolchain_util_unittest.py
+++ b/lib/toolchain_util_unittest.py
@@ -751,6 +751,8 @@
expected_output_files):
"""Asserts that the given artifact_path is tarred up properly.
+ If no output files are expected, we assert that no tarballs are created.
+
Args:
artifact_path: the path to touch |input_files| in.
tarball_name: the expected name of the tarball we will produce.
@@ -774,20 +776,32 @@
osutils.Touch(p)
tarball = self.obj.Bundle()
- tarball_path = os.path.join(self.outdir, tarball_name)
- self.assertEqual(tarball, [tarball_path])
- create_tarball_mock.assert_called_once()
- output, _tempdir = create_tarball_mock.call_args[0]
- self.assertEqual(output, tarball_path)
- inputs = create_tarball_mock.call_args[1]['inputs']
- self.assertCountEqual(expected_output_files, inputs)
+ if len(expected_output_files) > 0:
+ tarball_path = os.path.join(self.outdir, tarball_name)
+ self.assertEqual(tarball, [tarball_path])
+
+ create_tarball_mock.assert_called_once()
+ output, _tempdir = create_tarball_mock.call_args[0]
+ self.assertEqual(output, tarball_path)
+ inputs = create_tarball_mock.call_args[1]['inputs']
+ self.assertCountEqual(expected_output_files, inputs)
+ else:
+ # Bundlers do not create tarballs when no artifacts are found.
+ self.assertEqual(tarball, [])
def testBundleToolchainWarningLogs(self):
self.SetUpBundle('ToolchainWarningLogs')
+ artifact_path = '/tmp/fatal_clang_warnings'
+ tarball_name = '%s.DATE.fatal_clang_warnings.tar.xz' % self.board
+
+ # Test behaviour when no artifacts are found.
+ self.runToolchainBundleTest(artifact_path, tarball_name, [], [])
+
+ # Test behaviour when artifacts are found.
self.runToolchainBundleTest(
- artifact_path='/tmp/fatal_clang_warnings',
- tarball_name='%s.DATE.fatal_clang_warnings.tar.xz' % self.board,
+ artifact_path,
+ tarball_name,
input_files=('log1.json', 'log2.json', 'log3.notjson', 'log4'),
expected_output_files=(
'log1.json',
@@ -799,9 +813,16 @@
def testBundleClangCrashDiagnoses(self):
self.SetUpBundle('ClangCrashDiagnoses')
+ artifact_path = '/tmp/clang_crash_diagnostics'
+ tarball_name = '%s.DATE.clang_crash_diagnoses.tar.xz' % self.board
+
+ # Test behaviour when no artifacts are found.
+ self.runToolchainBundleTest(artifact_path, tarball_name, [], [])
+
+ # Test behaviour when artifacts are found.
self.runToolchainBundleTest(
- artifact_path='/tmp/clang_crash_diagnostics',
- tarball_name='%s.DATE.clang_crash_diagnoses.tar.xz' % self.board,
+ artifact_path,
+ tarball_name,
input_files=('1.cpp', '1.sh', '2.cc', '2.sh', 'foo/bar.sh'),
expected_output_files=(
'1.cpp',
@@ -817,6 +838,32 @@
),
)
+ def testBundleCompilerRusageLogs(self):
+ self.SetUpBundle('CompilerRusageLogs')
+ artifact_path = '/tmp/compiler_rusage'
+ tarball_name = '%s.DATE.compiler_rusage_logs.tar.xz' % self.board
+
+ # Test behaviour when no artifacts are found.
+ self.runToolchainBundleTest(artifact_path, tarball_name, [], [])
+
+ # Test behaviour when artifacts are found.
+ self.runToolchainBundleTest(
+ artifact_path,
+ tarball_name,
+ input_files=(
+ 'good1.json', 'good2.json', 'good3.json',
+ 'bad1.notjson', 'bad2', 'json',
+ ),
+ expected_output_files=(
+ 'good1.json',
+ 'good2.json',
+ 'good3.json',
+ 'good10.json',
+ 'good20.json',
+ 'good30.json',
+ ),
+ )
+
class UpdateKernelMetadataTest(PrepareBundleTest):
"""Test _UpdateKernelMetadata() function in BundleArtifactHandler."""
