portage_util: Fix HasPrebuilt check.
The prebuilt check had been using -K to avoid needing to parse the
output, but -K does not consider local changes, so eclass deps
changes could cause it to report binpkgs that actually needed to
be rebuilt. Refactor to use the depgraph lib instead.
BUG=chromium:1083435
TEST=manual
Change-Id: Id7a644390af09b90f6f486bdc1be8b4fc590b2b4
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/chromite/+/2207897
Tested-by: Alex Klein <saklein@chromium.org>
Commit-Queue: Alex Klein <saklein@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
diff --git a/lib/depgraph.py b/lib/depgraph.py
index db16574..05728a7 100644
--- a/lib/depgraph.py
+++ b/lib/depgraph.py
@@ -288,6 +288,24 @@
vardb.counter_tick()
vardb.flush_cache()
+ def HasPrebuilt(self, pkg_cpf: str):
+ """Check if the given package cpf has a prebuilt.
+
+ Args:
+ pkg_cpf: The fully qualified category/package-version-revision.
+
+ Returns:
+ bool: True if there is a prebuilt available, False otherwise.
+ """
+ if not self.package_db:
+ self.GenDependencyTree()
+
+ package = self.package_db.get(pkg_cpf)
+ if package:
+ return package.type_name == 'binary'
+
+ return False
+
def GenDependencyTree(self):
"""Get dependency tree info from emerge.
@@ -408,6 +426,7 @@
if isinstance(pkg, Package):
# Save off info about the package
deps_info[str(pkg.cpv)] = {'idx': len(deps_info)}
+ self.package_db[pkg.cpv] = pkg
seconds = time.time() - start
if '--quiet' not in emerge.opts:
diff --git a/lib/portage_util.py b/lib/portage_util.py
index 7d0d639..b7bed46 100644
--- a/lib/portage_util.py
+++ b/lib/portage_util.py
@@ -11,6 +11,7 @@
import errno
import glob
import itertools
+import json
import multiprocessing
import os
import re
@@ -2228,22 +2229,27 @@
def HasPrebuilt(atom, board=None, extra_env=None):
"""Check if the atom's best visible version has a prebuilt available."""
- best = PortageqBestVisible(atom, board=board)
+ cmd = [
+ os.path.join(constants.CHROOT_SOURCE_ROOT, 'chromite', 'scripts',
+ 'has_prebuilt'),
+ ]
+ if board:
+ cmd += ['--build-target', board]
+ cmd += [atom]
+ result = cros_build_lib.run(cmd, enter_chroot=True, extra_env=extra_env,
+ capture_output=True, encoding='utf-8')
- emerge = 'emerge-%s' % board if board else 'emerge'
- # Emerge args: binpkg only, no deps, pretend, quiet. --binpkg-respect-use is
- # disabled by default when you use -K, so turn it back on.
- cmd = [emerge, '-gKOpq', '--binpkg-respect-use=y', '=%s' % best.cpf]
- logging.debug('Checking %s for %s.', board or 'sdk', best.cpf)
- result = cros_build_lib.run(
- cmd,
- print_cmd=True,
- enter_chroot=True,
- extra_env=extra_env,
- check=False,
- debug_level=logging.DEBUG)
+ if result.returncode:
+ logging.warning('Error when checking for prebuilts: %s', result.stderr)
+ return False
- return not result.returncode
+ # Script produces {atom:has-prebuilt} mapping.
+ prebuilts = json.loads(result.stdout)
+ if atom not in prebuilts:
+ logging.warning('%s not found in has_prebuilt output.', atom)
+ return False
+
+ return prebuilts[atom]
class PortageqError(Error):
diff --git a/lib/portage_util_unittest.py b/lib/portage_util_unittest.py
index f262954..4c788c6 100644
--- a/lib/portage_util_unittest.py
+++ b/lib/portage_util_unittest.py
@@ -7,6 +7,7 @@
from __future__ import print_function
+import json
import os
import sys
@@ -1486,34 +1487,21 @@
"""HasPrebuilt tests."""
def setUp(self):
- self.atom = 'chromeos-base/chromeos-chrome'
- self.r1_cpf = 'chromeos-base/chromeos-chrome-78.0.3900.0_rc-r1'
- self.r2_cpf = 'chromeos-base/chromeos-chrome-78.0.3900.0_rc-r2'
- self.r1_cpv = portage_util.SplitCPV(self.r1_cpf)
- self.r2_cpv = portage_util.SplitCPV(self.r2_cpf)
-
- def _check_pkg(cmd, *_args, **_kwargs):
- rc = 0 if '=%s' % self.r1_cpf in cmd else 1
- return cros_build_lib.CommandResult(cmd=cmd, returncode=rc)
- self.rc.SetDefaultCmdResult(side_effect=_check_pkg)
+ self.atom = constants.CHROME_CP
def testHasPrebuilt(self):
"""Test a package with a matching prebuilt."""
- self.PatchObject(portage_util, 'PortageqBestVisible',
- return_value=self.r1_cpv)
- self.rc.SetDefaultCmdResult(returncode=0)
+ self.rc.SetDefaultCmdResult(
+ returncode=0, stdout=json.dumps({self.atom: True}))
self.assertTrue(portage_util.HasPrebuilt(self.atom))
- self.rc.assertCommandContains(('=%s' % self.r1_cpf,))
def testNoPrebuilt(self):
"""Test a package without a matching prebuilt."""
- self.PatchObject(portage_util, 'PortageqBestVisible',
- return_value=self.r2_cpv)
- self.rc.SetDefaultCmdResult(returncode=1)
+ self.rc.SetDefaultCmdResult(returncode=0,
+ stdout=json.dumps({self.atom: False}))
self.assertFalse(portage_util.HasPrebuilt(self.atom))
- self.rc.assertCommandContains(('=%s' % self.r2_cpf,))
class PortageqBestVisibleTest(cros_test_lib.MockTestCase):
diff --git a/scripts/has_prebuilt b/scripts/has_prebuilt
new file mode 120000
index 0000000..b68b744
--- /dev/null
+++ b/scripts/has_prebuilt
@@ -0,0 +1 @@
+wrapper3.py
\ No newline at end of file
diff --git a/scripts/has_prebuilt.py b/scripts/has_prebuilt.py
new file mode 100644
index 0000000..8dbacbe
--- /dev/null
+++ b/scripts/has_prebuilt.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script to check if the package(s) have prebuilts.
+
+The script must be run inside the chroot. The output is a json dict mapping the
+package atoms to a boolean for whether a prebuilt exists.
+"""
+
+from __future__ import print_function
+
+import json
+
+from chromite.lib import commandline
+from chromite.lib import cros_build_lib
+from chromite.lib import portage_util
+
+if cros_build_lib.IsInsideChroot():
+ from chromite.lib import depgraph
+
+
+def GetParser():
+ """Build the argument parser."""
+ parser = commandline.ArgumentParser(description=__doc__)
+
+ parser.add_argument(
+ '-b',
+ '--build-target',
+ type='build_target',
+ help='The build target that is being checked.')
+ parser.add_argument(
+ 'packages',
+ nargs='+',
+ help='The package atoms that are being checked.')
+
+ return parser
+
+
+def _ParseArguments(argv):
+ """Parse and validate arguments."""
+ parser = GetParser()
+ opts = parser.parse_args(argv)
+
+ # Manually parse the packages as CPVs.
+ packages = []
+ for pkg in opts.packages:
+ cpv = portage_util.SplitCPV(pkg, strict=False)
+ if not cpv.category or not cpv.package:
+ parser.error('Invalid package atom: %s' % pkg)
+
+ packages.append(cpv)
+ opts.packages = packages
+
+ opts.Freeze()
+ return opts
+
+
+def main(argv):
+ opts = _ParseArguments(argv)
+ cros_build_lib.AssertInsideChroot()
+
+ board = opts.build_target.name if opts.build_target else None
+ bests = {}
+ for cpv in opts.packages:
+ bests[cpv.cp] = portage_util.PortageqBestVisible(cpv.cp, board=board)
+
+ # Emerge args:
+ # g: use binpkgs (needed to find if we have one)
+ # u: update packages to latest version (want updates to invalidate binpkgs)
+ # D: deep -- consider full tree rather that just immediate deps
+ # (changes in dependencies and transitive deps can invalidate a binpkg)
+ # N: Packages with changed use flags should be considered
+ # (changes in dependencies and transitive deps can invalidate a binpkg)
+ # q: quiet (simplifies output)
+ # p: pretend (don't actually install it)
+ args = ['-guDNqp', '--with-bdeps=y', '--color=n']
+ if board:
+ args.append('--board=%s' % board)
+ args.extend('=%s' % best.cpf for best in bests.values())
+
+ generator = depgraph.DepGraphGenerator()
+ generator.Initialize(args)
+
+ results = {}
+ for atom, best in bests.items():
+ results[atom] = generator.HasPrebuilt(best.cpf)
+
+ print(json.dumps(results))
