| # Copyright 2014 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Module with utilities for archiving functionality.""" |
| |
| import logging |
| import os |
| |
| from chromite.cbuildbot import commands |
| from chromite.lib import config_lib |
| from chromite.lib import osutils |
| from chromite.utils import gs_urls_util |
| |
| |
| def GetBaseUploadURI(config, archive_base=None, bot_id=None): |
| """Get the base URL where artifacts from this builder are uploaded. |
| |
| Each build run stores its artifacts in a subdirectory of the base URI. |
| We also have LATEST files under the base URI which help point to the |
| latest build available for a given builder. |
| |
| Args: |
| config: The build config to examine. |
| archive_base: Optional. The root URL under which objects from all |
| builders are uploaded. If not specified, we use the default archive |
| bucket. |
| bot_id: The bot ID to archive files under. |
| |
| Returns: |
| Google Storage URI (i.e. 'gs://...') under which all archived files |
| should be uploaded. In other words, a path like a directory, even |
| through GS has no real directories. |
| """ |
| if not bot_id: |
| bot_id = config.name |
| |
| if archive_base: |
| gs_base = archive_base |
| elif config.gs_path == config_lib.GS_PATH_DEFAULT: |
| gs_base = config_lib.GetSiteParams().ARCHIVE_URL |
| else: |
| gs_base = config.gs_path |
| |
| return os.path.join(gs_base, bot_id) |
| |
| |
| def GetUploadACL(config): |
| """Get the ACL we should use to upload artifacts for a given config.""" |
| if config.internal: |
| # Use the bucket default ACL. |
| return None |
| |
| return "public-read" |
| |
| |
| class Archive: |
| """Class to represent the archive for one builder run. |
| |
| An Archive object is a read-only object with attributes and methods useful |
| for archive purposes. Most of the attributes are supported as properties |
| because they depend on the ChromeOS version and if they are calculated too |
| soon (i.e. before the sync stage) they will raise an exception. |
| |
| Attributes: |
| archive_path: The full local path where output from this builder is |
| stored. |
| download_url: The URL where we can download directory artifacts. |
| download_url_file: The URL where we can download file artifacts. |
| upload_url: The Google Storage location where we should upload |
| artifacts. |
| version: The ChromeOS version for this archive. |
| """ |
| |
| # TODO(davidriley): The use of a special download url for directories and |
| # files is a workaround for b/27653354. If that is ultimately fixed, revisit |
| # this workaround. |
| |
| _BUILDBOT_ARCHIVE = "buildbot_archive" |
| _TRYBOT_ARCHIVE = "trybot_archive" |
| |
| def __init__(self, bot_id, version_getter, options, config) -> None: |
| """Initialize. |
| |
| Args: |
| bot_id: The bot id associated with this archive. |
| version_getter: Functor that should return the ChromeOS version for |
| this run when called, if the version is known. Typically, this |
| is BuilderRun.GetVersion. |
| options: The command options object for this run. |
| config: The build config for this run. |
| """ |
| self._options = options |
| self._config = config |
| self._version_getter = version_getter |
| self._version = None |
| |
| self.bot_id = bot_id |
| |
| @property |
| def version(self): |
| if self._version is None: |
| self._version = self._version_getter() |
| |
| return self._version |
| |
| @property |
| def archive_path(self): |
| return os.path.join( |
| self.GetLocalArchiveRoot(), self.bot_id, self.version |
| ) |
| |
| @property |
| def upload_url(self): |
| base_upload_url = GetBaseUploadURI( |
| self._config, |
| archive_base=self._options.archive_base, |
| bot_id=self.bot_id, |
| ) |
| return "%s/%s" % (base_upload_url, self.version) |
| |
| @property |
| def upload_acl(self): |
| """Get the ACL we should use to upload artifacts for a given config.""" |
| return GetUploadACL(self._config) |
| |
| @property |
| def download_url(self): |
| if self._options.buildbot or self._options.remote_trybot: |
| # Translate the gs:// URI to the URL for downloading the same files. |
| # TODO(akeshet): The use of a special download url is a workaround |
| # for b/27653354. If that is ultimately fixed, revisit this |
| # workaround. This download link works for directories. |
| return self.upload_url.replace( |
| "gs://", gs_urls_util.PRIVATE_BASE_HTTPS_DOWNLOAD_URL |
| ) |
| else: |
| return self.archive_path |
| |
| @property |
| def download_url_file(self): |
| if self._options.buildbot or self._options.remote_trybot: |
| # Translate the gs:// URI to the URL for downloading the same files. |
| # TODO(akeshet): The use of a special download url is a workaround |
| # for b/27653354. If that is ultimately fixed, revisit this |
| # workaround. This download link works for files. |
| return self.upload_url.replace( |
| "gs://", gs_urls_util.PRIVATE_BASE_HTTPS_URL |
| ) |
| else: |
| return self.archive_path |
| |
| def GetLocalArchiveRoot(self, trybot=None): |
| """Return the location on disk where archive images are kept.""" |
| buildroot = os.path.abspath(self._options.buildroot) |
| |
| if trybot is None: |
| trybot = not self._options.buildbot or self._options.debug |
| |
| archive_base = ( |
| self._TRYBOT_ARCHIVE if trybot else self._BUILDBOT_ARCHIVE |
| ) |
| return os.path.join(buildroot, archive_base) |
| |
| def SetupArchivePath(self) -> None: |
| """Create a fresh directory for archiving a build.""" |
| logging.info( |
| 'Preparing local archive directory at "%s".', self.archive_path |
| ) |
| if self._options.buildbot: |
| # Buildbot: Clear out any leftover build artifacts, if present, for |
| # this particular run. The Clean stage is responsible for trimming |
| # back the number of archive paths to the last X runs. |
| osutils.RmDir(self.archive_path, ignore_missing=True) |
| else: |
| # Clear the list of uploaded file if it exists. In practice, the |
| # Clean stage deletes everything in the archive root, so this may |
| # not be doing anything at all. |
| osutils.SafeUnlink( |
| os.path.join(self.archive_path, commands.UPLOADED_LIST_FILENAME) |
| ) |
| |
| osutils.SafeMakedirs(self.archive_path) |
| |
| def UpdateLatestMarkers( |
| self, manifest_branch, debug, upload_urls=None |
| ) -> None: |
| """Update the LATEST markers in GS archive area. |
| |
| Args: |
| manifest_branch: The name of the branch in the manifest for this |
| run. |
| debug: Boolean debug value for this run. |
| upload_urls: Google storage urls to upload the Latest Markers to. |
| """ |
| if not upload_urls: |
| upload_urls = [self.upload_url] |
| # self.version will be one of these forms, shown through examples: |
| # R35-1234.5.6 or R35-1234.5.6-b123. In either case, we want |
| # "1234.5.6". |
| version_marker = self.version.split("-")[1] |
| |
| filenames = ( |
| "LATEST-%s" % manifest_branch, |
| "LATEST-%s" % version_marker, |
| ) |
| base_archive_path = os.path.dirname(self.archive_path) |
| base_upload_urls = [os.path.dirname(url) for url in upload_urls] |
| for base_upload_url in base_upload_urls: |
| for filename in filenames: |
| latest_path = os.path.join(base_archive_path, filename) |
| osutils.WriteFile(latest_path, self.version, mode="w") |
| commands.UploadArchivedFile( |
| base_archive_path, |
| [base_upload_url], |
| filename, |
| debug, |
| acl=self.upload_acl, |
| ) |