| # -*- coding: utf-8 -*- |
| # Copyright 2019 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Toolchain and related functionality.""" |
| |
| from __future__ import print_function |
| |
| import os |
| import re |
| |
| from chromite.lib import constants |
| from chromite.lib import cros_build_lib |
| from chromite.lib import cros_logging as logging |
| from chromite.lib import gs |
| from chromite.lib import path_util |
| from chromite.lib import portage_util |
| |
| ORDERFILE_GS_URL_UNVETTED = \ |
| 'gs://chromeos-prebuilt/afdo-job/orderfiles/unvetted' |
| ORDERFILE_GS_URL_VETTED = \ |
| 'gs://chromeos-prebuilt/afdo-job/orderfiles/vetted' |
| ORDERFILE_COMPRESSION_SUFFIX = '.xz' |
| |
| |
| class Error(Exception): |
| """Base module error class.""" |
| |
| |
| class GenerateChromeOrderfileError(Error): |
| """Error for GenerateChromeOrderfile class.""" |
| |
| |
| class GenerateChromeOrderfile(object): |
| """Class to handle generation of orderfile for Chrome. |
| |
| This class takes orderfile containing symbols ordered by Call-Chain |
| Clustering (C3), produced when linking Chrome, and uses a toolchain |
| script to perform post processing to generate an orderfile that can |
| be used for linking Chrome and creates tarball. The output of this |
| script is a tarball of the orderfile and a tarball of the NM output |
| of the built Chrome binary. |
| """ |
| |
| PROCESS_SCRIPT = ('/mnt/host/source/src/third_party/toolchain-utils/' |
| 'orderfile/post_process_orderfile.py') |
| CHROME_BINARY_PATH = ('/var/cache/chromeos-chrome/chrome-src-internal/' |
| 'src/out_${BOARD}/Release/chrome') |
| INPUT_ORDERFILE_PATH = ('/build/${BOARD}/opt/google/chrome/' |
| 'chrome.orderfile.txt') |
| |
| def __init__(self, board, output_dir, chrome_version, chroot_path, |
| chroot_args): |
| self.output_dir = output_dir |
| self.chrome_version = chrome_version |
| self.chrome_binary = self.CHROME_BINARY_PATH.replace('${BOARD}', board) |
| self.input_orderfile = self.INPUT_ORDERFILE_PATH.replace('${BOARD}', board) |
| self.working_dir = os.path.join(chroot_path, 'tmp') |
| self.working_dir_inchroot = '/tmp' |
| self.chroot_path = chroot_path |
| self.chroot_args = chroot_args |
| |
| def _CheckArguments(self): |
| if not os.path.isdir(self.output_dir): |
| raise GenerateChromeOrderfileError( |
| 'Non-existent directory %s specified for --out-dir', self.output_dir) |
| |
| chrome_binary_path_outside = os.path.join(self.chroot_path, |
| self.chrome_binary[1:]) |
| if not os.path.exists(chrome_binary_path_outside): |
| raise GenerateChromeOrderfileError( |
| 'Chrome binary does not exist at %s in chroot', |
| chrome_binary_path_outside) |
| |
| chrome_orderfile_path_outside = os.path.join(self.chroot_path, |
| self.input_orderfile[1:]) |
| if not os.path.exists(chrome_orderfile_path_outside): |
| raise GenerateChromeOrderfileError( |
| 'No orderfile generated in the builder.') |
| |
| def _GenerateChromeNM(self): |
| """This command runs inside chroot.""" |
| cmd = ['llvm-nm', '-n', self.chrome_binary] |
| result_inchroot = os.path.join(self.working_dir_inchroot, |
| self.chrome_version + '.nm') |
| result_out_chroot = os.path.join(self.working_dir, |
| self.chrome_version + '.nm') |
| |
| try: |
| cros_build_lib.RunCommand( |
| cmd, |
| log_stdout_to_file=result_out_chroot, |
| enter_chroot=True, |
| chroot_args=self.chroot_args) |
| except cros_build_lib.RunCommandError: |
| raise GenerateChromeOrderfileError( |
| 'Unable to run %s to get nm on Chrome binary' % (cmd)) |
| |
| # Return path inside chroot |
| return result_inchroot |
| |
| def _PostProcessOrderfile(self, chrome_nm): |
| """This command runs inside chroot.""" |
| result = os.path.join(self.working_dir_inchroot, |
| self.chrome_version + '.orderfile') |
| cmd = [ |
| self.PROCESS_SCRIPT, '--chrome', chrome_nm, '--input', |
| self.input_orderfile, '--output', result |
| ] |
| |
| try: |
| cros_build_lib.RunCommand( |
| cmd, enter_chroot=True, chroot_args=self.chroot_args) |
| except cros_build_lib.RunCommandError: |
| raise GenerateChromeOrderfileError( |
| 'Unable to run %s to process orderfile.' % (cmd)) |
| |
| # Return path inside chroot |
| return result |
| |
| def _CreateTarball(self, targets): |
| """This command runs outside of chroot.""" |
| ret = [] |
| for t in targets: |
| # The input t is the path inside chroot |
| input_path = os.path.join(self.working_dir, os.path.basename(t)) |
| # Put output tarball in the out_dir |
| compressed = os.path.basename(t) + ORDERFILE_COMPRESSION_SUFFIX |
| output_path = os.path.join(self.output_dir, compressed) |
| cros_build_lib.CompressFile(input_path, output_path) |
| # Only return the basename |
| ret.append(compressed) |
| return ret |
| |
| def Perform(self): |
| """Generate post-processed Chrome orderfile and create tarball.""" |
| self._CheckArguments() |
| chrome_nm = self._GenerateChromeNM() |
| orderfile = self._PostProcessOrderfile(chrome_nm) |
| self._CreateTarball([chrome_nm, orderfile]) |
| |
| |
| class UpdateChromeEbuildWithOrderfileError(Error): |
| """Error for UpdateChromeEbuildWithOrderfile class.""" |
| |
| class UpdateChromeEbuildWithOrderfile(object): |
| """Class to update Chrome ebuild with unvetted orderfile.""" |
| |
| # regex to find orderfile declaration within the ebuild file. |
| CHROME_EBUILD_ORDERFILE_REGEX = ( |
| r'^(?P<bef>UNVETTED_ORDERFILE=")(?P<name>.*)(?P<aft>")') |
| # and corresponding replacement string |
| CHROME_EBUILD_ORDERFILE_REPL = r'\g<bef>%s\g<aft>' |
| |
| def __init__(self, board, orderfile): |
| self.board = board |
| if ORDERFILE_COMPRESSION_SUFFIX in orderfile: |
| self.orderfile = orderfile.replace(ORDERFILE_COMPRESSION_SUFFIX, '') |
| else: |
| self.orderfile = orderfile |
| |
| def _FindChromeEbuild(self): |
| """Find the Chrome ebuild in the build root.""" |
| equery_prog = 'equery-%s' % self.board |
| equery_cmd = [equery_prog, 'w', 'chromeos-chrome'] |
| ebuild_file = cros_build_lib.RunCommand( |
| equery_cmd, enter_chroot=True, redirect_stdout=True).output.rstrip() |
| return ebuild_file |
| |
| def _PatchChromeEbuild(self, ebuild_file): |
| """Patch the Chrome ebuild to use the orderfile. |
| |
| Args: |
| ebuild_file: path to the ebuild file. |
| """ |
| original_ebuild = path_util.FromChrootPath(ebuild_file) |
| modified_ebuild = '%s.new' % original_ebuild |
| |
| pattern = re.compile(self.CHROME_EBUILD_ORDERFILE_REGEX) |
| orderfile = self.CHROME_EBUILD_ORDERFILE_REPL % self.orderfile |
| found = False |
| with open(original_ebuild) as original, \ |
| open(modified_ebuild, 'w') as modified: |
| for line in original: |
| matched = pattern.match(line) |
| if matched: |
| found = True |
| modified.write(pattern.sub(orderfile, line)) |
| else: |
| modified.write(line) |
| if not found: |
| logging.info("Unable to find markers for setting orderfile.") |
| raise UpdateChromeEbuildWithOrderfileError( |
| 'Chrome ebuild file does not have appropriate orderfile marker') |
| os.rename(modified_ebuild, original_ebuild) |
| logging.info("Patched %s with %s", original_ebuild, self.orderfile) |
| |
| def _UpdateManifest(self, ebuild_file): |
| """Regenerate the Manifest file. (To update orderfile) |
| |
| Args: |
| ebuild_file: path to the ebuild file |
| """ |
| ebuild_prog = 'ebuild-%s' % self.board |
| cmd = [ebuild_prog, ebuild_file, 'manifest', '--force'] |
| cros_build_lib.RunCommand(cmd, enter_chroot=True) |
| |
| def Perform(self): |
| """Main function to update Chrome ebuild with orderfile""" |
| ebuild = self._FindChromeEbuild() |
| self._PatchChromeEbuild(ebuild) |
| # Patch the chrome 9999 ebuild too, as the manifest will use |
| # 9999 ebuild. |
| ebuild_9999 = os.path.join( |
| os.path.dirname(ebuild), 'chromeos-chrome-9999.ebuild') |
| self._PatchChromeEbuild(ebuild_9999) |
| # Use 9999 ebuild to update manifest. |
| self._UpdateManifest(ebuild_9999) |
| |
| |
| def FindLatestChromeOrderfile(gs_url): |
| """Find the latest unvetted Chrome orderfile. |
| |
| Args: |
| gs_url: The full path to GS bucket URL. |
| |
| Returns: |
| The orderfile name. |
| """ |
| gs_context = gs.GSContext() |
| orderfiles = sorted( |
| gs_context.List(gs_url, details=True), key=lambda x: x.creation_time) |
| orderfile_url = orderfiles[-1].url |
| orderfile_name = os.path.basename(orderfile_url) |
| logging.info("Latest orderfile in %s is %s", gs_url, orderfile_name) |
| return orderfile_name |
| |
| |
| def CheckOrderfileExists(buildroot, orderfile_verify): |
| """Checks if the orderfile to be generated/verified already exists. |
| |
| Args: |
| buildroot: The path of build root |
| orderfile_verify: Whether it's for orderfile_verify or orderfile_generate. |
| For orderfile verify builder, it verifies the most recent orderfile from |
| the unvetted bucket. So we checks if it's in the vetted bucket or not. |
| For orderfile generate builder, it generates an orderfile based on the |
| Chrome version number. So we need to find out the Chrome is upreved than |
| the most recent unvetted orderfile or not. |
| |
| Returns: |
| True if the orderfile is in the GS bucket already. |
| """ |
| gs_context = gs.GSContext() |
| if orderfile_verify: |
| orderfile_name = FindLatestChromeOrderfile(ORDERFILE_GS_URL_UNVETTED) |
| # Check if the latest unvetted orderfile is already verified |
| return gs_context.Exists(ORDERFILE_GS_URL_VETTED + '/' + orderfile_name) |
| |
| # For orderfile_generate builder, get the chrome version |
| cpv = portage_util.PortageqBestVisible(constants.CHROME_CP, cwd=buildroot) |
| orderfile_name = '{0}-orderfile-{1}'.format(cpv.package, cpv.version) |
| return gs_context.Exists(ORDERFILE_GS_URL_UNVETTED + '/' + orderfile_name + |
| ORDERFILE_COMPRESSION_SUFFIX) |
| |
| |
| def OrderfileUpdateChromeEbuild(board): |
| """Update Chrome ebuild with latest unvetted orderfile. |
| |
| Args: |
| board: Board type that was built on this machine. |
| |
| Returns: |
| True, if the Chrome ebuild is successfully updated. |
| False, if the latest unvetted orderfile is already verified. |
| """ |
| |
| orderfile_name = FindLatestChromeOrderfile(ORDERFILE_GS_URL_UNVETTED) |
| updater = UpdateChromeEbuildWithOrderfile(board, orderfile_name) |
| updater.Perform() |
| return True |