| # -*- coding: utf-8 -*- |
| # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Main module for parsing and interpreting XBuddy paths for the devserver.""" |
| |
| from __future__ import print_function |
| |
| import cherrypy |
| import ConfigParser |
| import datetime |
| import distutils.version |
| import operator |
| import os |
| import re |
| import shutil |
| import time |
| import threading |
| |
| import artifact_info |
| import build_artifact |
| import build_util |
| import common_util |
| import devserver_constants |
| import downloader |
| import log_util |
| |
| # Make sure that chromite is available to import. |
| import setup_chromite # pylint: disable=unused-import |
| |
| try: |
| from chromite.lib import gs |
| except ImportError: |
| gs = None |
| |
| # Module-local log function. |
| def _Log(message, *args): |
| return log_util.LogWithTag('XBUDDY', message, *args) |
| |
| # xBuddy config constants |
| CONFIG_FILE = 'xbuddy_config.ini' |
| SHADOW_CONFIG_FILE = 'shadow_xbuddy_config.ini' |
| PATH_REWRITES = 'PATH_REWRITES' |
| GENERAL = 'GENERAL' |
| LOCATION_SUFFIXES = 'LOCATION_SUFFIXES' |
| |
| # Path for shadow config in chroot. |
| CHROOT_SHADOW_DIR = '/mnt/host/source/src/platform/dev' |
| |
| # XBuddy aliases |
| TEST = 'test' |
| BASE = 'base' |
| DEV = 'dev' |
| FULL = 'full_payload' |
| SIGNED = 'signed' |
| RECOVERY = 'recovery' |
| STATEFUL = 'stateful' |
| AUTOTEST = 'autotest' |
| FACTORY_SHIM = 'factory_shim' |
| |
| # Local build constants |
| ANY = "ANY" |
| LATEST = "latest" |
| LOCAL = "local" |
| REMOTE = "remote" |
| |
| # TODO(sosa): Fix a lot of assumptions about these aliases. There is too much |
| # implicit logic here that's unnecessary. What should be done: |
| # 1) Collapse Alias logic to one set of aliases for xbuddy (not local/remote). |
| # 2) Do not use zip when creating these dicts. Better to not rely on ordering. |
| # 3) Move alias/artifact mapping to a central module rather than having it here. |
| # 4) Be explicit when things are missing i.e. no dev images in image.zip. |
| |
| LOCAL_ALIASES = [ |
| TEST, |
| DEV, |
| BASE, |
| RECOVERY, |
| FACTORY_SHIM, |
| FULL, |
| STATEFUL, |
| ANY, |
| ] |
| |
| LOCAL_FILE_NAMES = [ |
| devserver_constants.TEST_IMAGE_FILE, |
| devserver_constants.IMAGE_FILE, |
| devserver_constants.BASE_IMAGE_FILE, |
| devserver_constants.RECOVERY_IMAGE_FILE, |
| devserver_constants.FACTORY_SHIM_IMAGE_FILE, |
| devserver_constants.UPDATE_FILE, |
| devserver_constants.STATEFUL_FILE, |
| None, # For ANY. |
| ] |
| |
| LOCAL_ALIAS_TO_FILENAME = dict(zip(LOCAL_ALIASES, LOCAL_FILE_NAMES)) |
| |
| # Google Storage constants |
| GS_ALIASES = [ |
| TEST, |
| BASE, |
| RECOVERY, |
| SIGNED, |
| FACTORY_SHIM, |
| FULL, |
| STATEFUL, |
| AUTOTEST, |
| ] |
| |
| GS_FILE_NAMES = [ |
| devserver_constants.TEST_IMAGE_FILE, |
| devserver_constants.BASE_IMAGE_FILE, |
| devserver_constants.RECOVERY_IMAGE_FILE, |
| devserver_constants.SIGNED_IMAGE_FILE, |
| devserver_constants.FACTORY_SHIM_IMAGE_FILE, |
| devserver_constants.UPDATE_FILE, |
| devserver_constants.STATEFUL_FILE, |
| devserver_constants.AUTOTEST_DIR, |
| ] |
| |
| ARTIFACTS = [ |
| artifact_info.TEST_IMAGE, |
| artifact_info.BASE_IMAGE, |
| artifact_info.RECOVERY_IMAGE, |
| artifact_info.SIGNED_IMAGE, |
| artifact_info.FACTORY_SHIM_IMAGE, |
| artifact_info.FULL_PAYLOAD, |
| artifact_info.STATEFUL_PAYLOAD, |
| artifact_info.AUTOTEST, |
| ] |
| |
| GS_ALIAS_TO_FILENAME = dict(zip(GS_ALIASES, GS_FILE_NAMES)) |
| GS_ALIAS_TO_ARTIFACT = dict(zip(GS_ALIASES, ARTIFACTS)) |
| |
| LATEST_OFFICIAL = "latest-official" |
| |
| RELEASE = "-release" |
| |
| |
| class XBuddyException(Exception): |
| """Exception classes used by this module.""" |
| pass |
| |
| |
| # no __init__ method |
| #pylint: disable=W0232 |
| class Timestamp(object): |
| """Class to translate build path strings and timestamp filenames.""" |
| |
| _TIMESTAMP_DELIMITER = 'SLASH' |
| XBUDDY_TIMESTAMP_DIR = 'xbuddy_UpdateTimestamps' |
| |
| @staticmethod |
| def TimestampToBuild(timestamp_filename): |
| return timestamp_filename.replace(Timestamp._TIMESTAMP_DELIMITER, '/') |
| |
| @staticmethod |
| def BuildToTimestamp(build_path): |
| return build_path.replace('/', Timestamp._TIMESTAMP_DELIMITER) |
| |
| @staticmethod |
| def UpdateTimestamp(timestamp_dir, build_id): |
| """Update timestamp file of build with build_id.""" |
| common_util.MkDirP(timestamp_dir) |
| _Log("Updating timestamp for %s", build_id) |
| time_file = os.path.join(timestamp_dir, |
| Timestamp.BuildToTimestamp(build_id)) |
| with file(time_file, 'a'): |
| os.utime(time_file, None) |
| #pylint: enable=W0232 |
| |
| |
| class XBuddy(build_util.BuildObject): |
| """Class that manages image retrieval and caching by the devserver. |
| |
| Image retrieval by xBuddy path: |
| XBuddy accesses images and artifacts that it stores using an xBuddy |
| path of the form: board/version/alias |
| The primary xbuddy.Get call retrieves the correct artifact or url to where |
| the artifacts can be found. |
| |
| Image caching: |
| Images and other artifacts are stored identically to how they would have |
| been if devserver's stage rpc was called and the xBuddy cache replaces |
| build versions on a LRU basis. Timestamps are maintained by last accessed |
| times of representative files in the a directory in the static serve |
| directory (XBUDDY_TIMESTAMP_DIR). |
| |
| Private class members: |
| _true_values: used for interpreting boolean values |
| _staging_thread_count: track download requests |
| _timestamp_folder: directory with empty files standing in as timestamps |
| for each image currently cached by xBuddy |
| """ |
| _true_values = ['true', 't', 'yes', 'y'] |
| |
| # Number of threads that are staging images. |
| _staging_thread_count = 0 |
| # Lock used to lock increasing/decreasing count. |
| _staging_thread_count_lock = threading.Lock() |
| |
| def __init__(self, manage_builds=False, board=None, version=None, |
| images_dir=None, log_screen=True, **kwargs): |
| super(XBuddy, self).__init__(**kwargs) |
| |
| if not log_screen: |
| cherrypy.config.update({'log.screen': False}) |
| |
| self.config = self._ReadConfig() |
| self._manage_builds = manage_builds or self._ManageBuilds() |
| self._board = board |
| self._version = version |
| self._timestamp_folder = os.path.join(self.static_dir, |
| Timestamp.XBUDDY_TIMESTAMP_DIR) |
| if images_dir: |
| self.images_dir = images_dir |
| else: |
| self.images_dir = os.path.join(self.GetSourceRoot(), 'src/build/images') |
| |
| if common_util.IsRunningOnMoblab(): |
| self._ctx = gs.GSContext(cache_user='chronos') if gs else None |
| else: |
| self._ctx = gs.GSContext() if gs else None |
| |
| common_util.MkDirP(self._timestamp_folder) |
| |
| @classmethod |
| def ParseBoolean(cls, boolean_string): |
| """Evaluate a string to a boolean value""" |
| if boolean_string: |
| return boolean_string.lower() in cls._true_values |
| else: |
| return False |
| |
| def _ReadConfig(self): |
| """Read xbuddy config from ini files. |
| |
| Reads the base config from xbuddy_config.ini, and then merges in the |
| shadow config from shadow_xbuddy_config.ini |
| |
| Returns: |
| The merged configuration. |
| |
| Raises: |
| XBuddyException if the config file is missing. |
| """ |
| xbuddy_config = ConfigParser.ConfigParser() |
| config_file = os.path.join(self.devserver_dir, CONFIG_FILE) |
| if os.path.exists(config_file): |
| xbuddy_config.read(config_file) |
| else: |
| # Get the directory of xbuddy.py file. |
| file_dir = os.path.dirname(os.path.realpath(__file__)) |
| # Read the default xbuddy_config.ini from the directory. |
| xbuddy_config.read(os.path.join(file_dir, CONFIG_FILE)) |
| |
| # Read the shadow file if there is one. |
| if os.path.isdir(CHROOT_SHADOW_DIR): |
| shadow_config_file = os.path.join(CHROOT_SHADOW_DIR, SHADOW_CONFIG_FILE) |
| else: |
| shadow_config_file = os.path.join(self.devserver_dir, SHADOW_CONFIG_FILE) |
| |
| _Log('Using shadow config file stored at %s', shadow_config_file) |
| if os.path.exists(shadow_config_file): |
| shadow_xbuddy_config = ConfigParser.ConfigParser() |
| shadow_xbuddy_config.read(shadow_config_file) |
| |
| # Merge shadow config in. |
| sections = shadow_xbuddy_config.sections() |
| for s in sections: |
| if not xbuddy_config.has_section(s): |
| xbuddy_config.add_section(s) |
| options = shadow_xbuddy_config.options(s) |
| for o in options: |
| val = shadow_xbuddy_config.get(s, o) |
| xbuddy_config.set(s, o, val) |
| |
| return xbuddy_config |
| |
| def _ManageBuilds(self): |
| """Checks if xBuddy is managing local builds using the current config.""" |
| try: |
| return self.ParseBoolean(self.config.get(GENERAL, 'manage_builds')) |
| except ConfigParser.Error: |
| return False |
| |
| def _Capacity(self): |
| """Gets the xbuddy capacity from the current config.""" |
| try: |
| return int(self.config.get(GENERAL, 'capacity')) |
| except ConfigParser.Error: |
| return 5 |
| |
| def LookupAlias(self, alias, board=None, version=None): |
| """Given the full xbuddy config, look up an alias for path rewrite. |
| |
| Args: |
| alias: The xbuddy path that could be one of the aliases in the |
| rewrite table. |
| board: The board to fill in with when paths are rewritten. Can be from |
| the update request xml or the default board from devserver. If None, |
| defers to the value given during XBuddy initialization. |
| version: The version to fill in when rewriting paths. Could be a specific |
| version number or a version alias like LATEST. If None, defers to the |
| value given during XBuddy initialization, or LATEST. |
| |
| Returns: |
| A pair (val, suffix) where val is the rewritten path, or the original |
| string if no rewrite was found; and suffix is the assigned location |
| suffix, or the default suffix if none was found. |
| """ |
| try: |
| suffix = self.config.get(LOCATION_SUFFIXES, alias) |
| except ConfigParser.Error: |
| suffix = RELEASE |
| |
| try: |
| val = self.config.get(PATH_REWRITES, alias) |
| except ConfigParser.Error: |
| # No alias lookup found. Return original path. |
| val = None |
| |
| if not (val and val.strip()): |
| val = alias |
| else: |
| # The found value is not an empty string. |
| # Fill in the board and version. |
| val = val.replace("BOARD", "%(board)s") |
| val = val.replace("VERSION", "%(version)s") |
| val = val % {'board': board or self._board, |
| 'version': version or self._version or LATEST} |
| |
| _Log("Path is %s, location suffix is %s", val, suffix) |
| return val, suffix |
| |
| @staticmethod |
| def _ResolveImageDir(image_dir): |
| """Clean up and return the image dir to use. |
| |
| Args: |
| image_dir: directory in Google Storage to use. |
| |
| Returns: |
| |image_dir| if |image_dir| is not None. Otherwise, returns |
| devserver_constants.GS_IMAGE_DIR |
| """ |
| image_dir = image_dir or devserver_constants.GS_IMAGE_DIR |
| # Remove trailing slashes. |
| return image_dir.rstrip('/') |
| |
| def _LookupOfficial(self, board, suffix, image_dir=None): |
| """Check LATEST-master for the version number of interest.""" |
| _Log("Checking gs for latest %s-%s image", board, suffix) |
| image_dir = XBuddy._ResolveImageDir(image_dir) |
| latest_addr = (devserver_constants.GS_LATEST_MASTER % |
| {'image_dir': image_dir, |
| 'board': board, |
| 'suffix': suffix}) |
| # Full release + version is in the LATEST file. |
| version = self._ctx.Cat(latest_addr) |
| |
| return devserver_constants.IMAGE_DIR % {'board':board, |
| 'suffix':suffix, |
| 'version':version} |
| |
| def _LS(self, path, list_subdirectory=False): |
| """Does a directory listing of the given gs path. |
| |
| Args: |
| path: directory location on google storage to check. |
| list_subdirectory: whether to only list subdirectory for |path|. |
| |
| Returns: |
| A list of paths that matched |path|. |
| """ |
| if list_subdirectory: |
| return self._ctx.DoCommand( |
| ['ls', '-d', '--', path], redirect_stdout=True).output.splitlines() |
| else: |
| return self._ctx.LS(path) |
| |
| def _GetLatestVersionFromGsDir(self, path, list_subdirectory=False, |
| with_release=True): |
| """Returns most recent version number found in a google storage directory. |
| |
| This lists out the contents of the given GS bucket or regex to GS buckets, |
| and tries to grab the newest version found in the directory names. |
| |
| Args: |
| path: directory location on google storage to check. |
| list_subdirectory: whether to only list subdirectory for |path|. |
| with_release: whether versions include a release milestone (e.g. R12). |
| |
| Returns: |
| The most recent version number found. |
| """ |
| list_result = self._LS(path, list_subdirectory=list_subdirectory) |
| dir_names = [os.path.basename(p.rstrip('/')) for p in list_result] |
| try: |
| filter_re = re.compile(devserver_constants.VERSION_RE if with_release |
| else devserver_constants.VERSION) |
| versions = filter(filter_re.match, dir_names) |
| latest_version = max(versions, key=distutils.version.LooseVersion) |
| except ValueError: |
| raise gs.GSContextException( |
| 'Failed to find most recent builds at %s' % path) |
| |
| return latest_version |
| |
| def _LookupChannel(self, board, suffix, channel='stable', |
| image_dir=None): |
| """Check the channel folder for the version number of interest.""" |
| # Get all names in channel dir. Get 10 highest directories by version. |
| _Log("Checking channel '%s' for latest '%s' image", channel, board) |
| # Due to historical reasons, gs://chromeos-releases uses |
| # daisy-spring as opposed to the board name daisy_spring. Convert |
| # he board name for the lookup. |
| channel_dir = devserver_constants.GS_CHANNEL_DIR % { |
| 'channel':channel, |
| 'board':re.sub('_', '-', board)} |
| latest_version = self._GetLatestVersionFromGsDir(channel_dir, |
| with_release=False) |
| |
| # Figure out release number from the version number. |
| image_url = devserver_constants.IMAGE_DIR % { |
| 'board': board, |
| 'suffix': suffix, |
| 'version': 'R*' + latest_version} |
| image_dir = XBuddy._ResolveImageDir(image_dir) |
| gs_url = os.path.join(image_dir, image_url) |
| |
| # There should only be one match on cros-image-archive. |
| full_version = self._GetLatestVersionFromGsDir(gs_url, |
| list_subdirectory=True) |
| |
| return devserver_constants.IMAGE_DIR % {'board': board, |
| 'suffix': suffix, |
| 'version': full_version} |
| |
| def _LookupVersion(self, board, suffix, version): |
| """Search GS image releases for the highest match to a version prefix.""" |
| # Build the pattern for GS to match. |
| _Log("Checking gs for latest '%s' image with prefix '%s'", board, version) |
| image_url = devserver_constants.IMAGE_DIR % {'board': board, |
| 'suffix': suffix, |
| 'version': version + '*'} |
| image_dir = os.path.join(devserver_constants.GS_IMAGE_DIR, image_url) |
| |
| # Grab the newest version of the ones matched. |
| full_version = self._GetLatestVersionFromGsDir(image_dir, |
| list_subdirectory=True) |
| return devserver_constants.IMAGE_DIR % {'board': board, |
| 'suffix': suffix, |
| 'version': full_version} |
| |
| def _RemoteBuildId(self, board, suffix, version): |
| """Returns the remote build_id for the given board and version. |
| |
| Raises: |
| XBuddyException: If we failed to resolve the version to a valid build_id. |
| """ |
| build_id_as_is = devserver_constants.IMAGE_DIR % {'board': board, |
| 'suffix': '', |
| 'version': version} |
| build_id_suffix = devserver_constants.IMAGE_DIR % {'board': board, |
| 'suffix': suffix, |
| 'version': version} |
| # Return the first path that exists. We assume that what the user typed |
| # is better than with a default suffix added i.e. x86-generic/blah is |
| # more valuable than x86-generic-release/blah. |
| for build_id in build_id_as_is, build_id_suffix: |
| try: |
| version = self._ctx.LS( |
| '%s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id)) |
| return build_id |
| except (gs.GSCommandError, gs.GSContextException, gs.GSNoSuchKey): |
| continue |
| |
| raise XBuddyException('Could not find remote build_id for %s %s' % ( |
| board, version)) |
| |
| def _ResolveBuildVersion(self, board, suffix, base_version): |
| """Check LATEST-<base_version> and returns a full build version.""" |
| _Log('Checking gs for full version for %s of %s', base_version, board) |
| # TODO(garnold) We might want to accommodate version prefixes and pick the |
| # most recent found, as done in _LookupVersion(). |
| latest_addr = (devserver_constants.GS_LATEST_BASE_VERSION % |
| {'image_dir': devserver_constants.GS_IMAGE_DIR, |
| 'board': board, |
| 'suffix': suffix, |
| 'base_version': base_version}) |
| # Full release + version is in the LATEST file. |
| return self._ctx.Cat(latest_addr) |
| |
| def _ResolveVersionToBuildIdAndChannel(self, board, suffix, version, |
| image_dir=None): |
| """Handle version aliases for remote payloads in GS. |
| |
| Args: |
| board: as specified in the original call. (i.e. x86-generic, parrot) |
| suffix: The location suffix, to be added to board name. |
| version: as entered in the original call. can be |
| {TBD, 0. some custom alias as defined in a config file} |
| 1. fully qualified build version. |
| 2. latest |
| 3. latest-{channel} |
| 4. latest-official-{board suffix} |
| 5. version prefix (i.e. RX-Y.X, RX-Y, RX) |
| image_dir: image directory to check in Google Storage. If none, |
| the default bucket is used. |
| |
| Returns: |
| Tuple of (Location where the image dir is actually found on GS (build_id), |
| best guess for the channel). |
| |
| Raises: |
| XBuddyException: If we failed to resolve the version to a valid url. |
| """ |
| # Only the last segment of the alias is variable relative to the rest. |
| version_tuple = version.rsplit('-', 1) |
| |
| if re.match(devserver_constants.VERSION_RE, version): |
| return self._RemoteBuildId(board, suffix, version), None |
| elif re.match(devserver_constants.VERSION, version): |
| raise XBuddyException('\'%s\' is not valid. Should provide the fully ' |
| 'qualified version with a version prefix \'RX-\' ' |
| 'due to crbug.com/585914' % version) |
| elif version == LATEST_OFFICIAL: |
| # latest-official --> LATEST build in board-release |
| return self._LookupOfficial(board, suffix, image_dir=image_dir), None |
| elif version_tuple[0] == LATEST_OFFICIAL: |
| # latest-official-{suffix} --> LATEST build in board-{suffix} |
| return self._LookupOfficial(board, version_tuple[1], |
| image_dir=image_dir), None |
| elif version == LATEST: |
| # latest --> latest build on stable channel |
| return self._LookupChannel(board, suffix, image_dir=image_dir), 'stable' |
| elif version_tuple[0] == LATEST: |
| if re.match(devserver_constants.VERSION_RE, version_tuple[1]): |
| # latest-R* --> most recent qualifying build |
| return self._LookupVersion(board, suffix, version_tuple[1]), None |
| else: |
| # latest-{channel} --> latest build within that channel |
| return self._LookupChannel(board, suffix, channel=version_tuple[1], |
| image_dir=image_dir), version_tuple[1] |
| else: |
| # The given version doesn't match any known patterns. |
| raise XBuddyException("Version %s unknown. Can't find on GS." % version) |
| |
| @staticmethod |
| def _Symlink(link, target): |
| """Symlinks link to target, and removes whatever link was there before.""" |
| _Log("Linking to %s from %s", link, target) |
| if os.path.lexists(link): |
| os.unlink(link) |
| os.symlink(target, link) |
| |
| def _GetLatestLocalVersion(self, board): |
| """Get the version of the latest image built for board by build_image |
| |
| Updates the symlink reference within the xBuddy static dir to point to |
| the real image dir in the local /build/images directory. |
| |
| Args: |
| board: board that image was built for. |
| |
| Returns: |
| The discovered version of the image. |
| |
| Raises: |
| XBuddyException if neither test nor dev image was found in latest built |
| directory. |
| """ |
| latest_local_dir = self.GetLatestImageLink(board) |
| if not latest_local_dir or not os.path.exists(latest_local_dir): |
| raise XBuddyException('No builds found for %s. Did you run build_image?' % |
| board) |
| |
| # Assume that the version number is the name of the directory. |
| return os.path.basename(os.path.realpath(latest_local_dir)) |
| |
| @staticmethod |
| def _FindAny(local_dir): |
| """Returns the image_type for ANY given the local_dir.""" |
| test_image = os.path.join(local_dir, devserver_constants.TEST_IMAGE_FILE) |
| dev_image = os.path.join(local_dir, devserver_constants.IMAGE_FILE) |
| # Prioritize test images over dev images. |
| if os.path.exists(test_image): |
| return 'test' |
| |
| if os.path.exists(dev_image): |
| return 'dev' |
| |
| raise XBuddyException('No images found in %s' % local_dir) |
| |
| @staticmethod |
| def _InterpretPath(path, default_board=None, default_version=None): |
| """Split and return the pieces of an xBuddy path name |
| |
| Args: |
| path: the path xBuddy Get was called with. |
| default_board: board to use in case board isn't in path. |
| default_version: Version to use in case version isn't in path. |
| |
| Returns: |
| tuple of (image_type, board, version, whether the path is local) |
| |
| Raises: |
| XBuddyException: if the path can't be resolved into valid components |
| """ |
| path_list = filter(None, path.split('/')) |
| |
| # Do the stuff that is well known first. We know that if paths have a |
| # image_type, it must be one of the GS/LOCAL aliases and it must be at the |
| # end. Similarly, local/remote are well-known and must start the path list. |
| is_local = True |
| if path_list and path_list[0] in (REMOTE, LOCAL): |
| is_local = (path_list.pop(0) == LOCAL) |
| |
| # Default image type is determined by remote vs. local. |
| if is_local: |
| image_type = ANY |
| else: |
| image_type = TEST |
| |
| if path_list and path_list[-1] in GS_ALIASES + LOCAL_ALIASES: |
| image_type = path_list.pop(-1) |
| |
| # Now for the tricky part. We don't actually know at this point if the rest |
| # of the path is just a board | version (like R33-2341.0.0) or just a board |
| # or just a version. So we do our best to do the right thing. |
| board = default_board |
| version = default_version or LATEST |
| if len(path_list) == 1: |
| path = path_list.pop(0) |
| # Treat this as a version if it's one we know (contains default or |
| # latest), or we were given an actual default board. |
| if default_version in path or LATEST in path or default_board is not None: |
| version = path |
| else: |
| board = path |
| |
| elif len(path_list) == 2: |
| # Assumes board/version. |
| board = path_list.pop(0) |
| version = path_list.pop(0) |
| |
| if path_list: |
| raise XBuddyException("Path isn't valid. Could not figure out how to " |
| "parse remaining components: %s." % path_list) |
| |
| _Log("Get artifact '%s' with board %s and version %s'. Locally? %s", |
| image_type, board, version, is_local) |
| |
| return image_type, board, version, is_local |
| |
| def _SyncRegistryWithBuildImages(self): |
| """Crawl images_dir for build_ids of images generated from build_image. |
| |
| This will find images and symlink them in xBuddy's static dir so that |
| xBuddy's cache can serve them. |
| If xBuddy's _manage_builds option is on, then a timestamp will also be |
| generated, and xBuddy will clear them from the directory they are in, as |
| necessary. |
| """ |
| if not os.path.isdir(self.images_dir): |
| # Skip syncing if images_dir does not exist. |
| _Log('Cannot find %s; skip syncing image registry.', self.images_dir) |
| return |
| |
| build_ids = [] |
| for b in os.listdir(self.images_dir): |
| # Ignore random files in the build dir. |
| board_dir = os.path.join(self.images_dir, b) |
| if not os.path.isdir(board_dir): |
| continue |
| |
| # Ensure we have directories to track all boards in build/images |
| common_util.MkDirP(os.path.join(self.static_dir, b)) |
| build_ids.extend(['/'.join([b, v]) for v |
| in os.listdir(board_dir) if not v == LATEST]) |
| |
| # Symlink undiscovered images, and update timestamps if manage_builds is on. |
| for build_id in build_ids: |
| link = os.path.join(self.static_dir, build_id) |
| target = os.path.join(self.images_dir, build_id) |
| XBuddy._Symlink(link, target) |
| if self._manage_builds: |
| Timestamp.UpdateTimestamp(self._timestamp_folder, build_id) |
| |
| def _ListBuildTimes(self): |
| """Returns the currently cached builds and their last access timestamp. |
| |
| Returns: |
| list of tuples that matches xBuddy build/version to timestamps in long |
| """ |
| # Update currently cached builds. |
| build_dict = {} |
| |
| for f in os.listdir(self._timestamp_folder): |
| last_accessed = os.path.getmtime(os.path.join(self._timestamp_folder, f)) |
| build_id = Timestamp.TimestampToBuild(f) |
| stale_time = datetime.timedelta(seconds=(time.time() - last_accessed)) |
| build_dict[build_id] = stale_time |
| return_tup = sorted(build_dict.iteritems(), key=operator.itemgetter(1)) |
| return return_tup |
| |
| def _Download(self, gs_url, artifacts, build_id): |
| """Download the artifacts from the given gs_url. |
| |
| Raises: |
| build_artifact.ArtifactDownloadError: If we failed to download the |
| artifact. |
| """ |
| with XBuddy._staging_thread_count_lock: |
| XBuddy._staging_thread_count += 1 |
| try: |
| _Log("Downloading %s from %s", artifacts, gs_url) |
| dl = downloader.GoogleStorageDownloader(self.static_dir, gs_url, build_id) |
| factory = build_artifact.ChromeOSArtifactFactory( |
| dl.GetBuildDir(), artifacts, [], dl.GetBuild()) |
| dl.Download(factory) |
| finally: |
| with XBuddy._staging_thread_count_lock: |
| XBuddy._staging_thread_count -= 1 |
| |
| def CleanCache(self): |
| """Delete all builds besides the newest N builds""" |
| if not self._manage_builds: |
| return |
| cached_builds = [e[0] for e in self._ListBuildTimes()] |
| _Log('In cache now: %s', cached_builds) |
| |
| for b in range(self._Capacity(), len(cached_builds)): |
| b_path = cached_builds[b] |
| _Log("Clearing '%s' from cache", b_path) |
| |
| time_file = os.path.join(self._timestamp_folder, |
| Timestamp.BuildToTimestamp(b_path)) |
| os.unlink(time_file) |
| clear_dir = os.path.join(self.static_dir, b_path) |
| try: |
| # Handle symlinks, in the case of links to local builds if enabled. |
| if os.path.islink(clear_dir): |
| target = os.readlink(clear_dir) |
| _Log('Deleting locally built image at %s', target) |
| |
| os.unlink(clear_dir) |
| if os.path.exists(target): |
| shutil.rmtree(target) |
| elif os.path.exists(clear_dir): |
| _Log('Deleting downloaded image at %s', clear_dir) |
| shutil.rmtree(clear_dir) |
| |
| except Exception as err: |
| raise XBuddyException('Failed to clear %s: %s' % (clear_dir, err)) |
| |
| def _TranslateSignedGSUrl(self, build_id, channel=None): |
| """Translate the GS URL to be able to find signed images. |
| |
| Args: |
| build_id: Path to the image or update directory on the devserver or |
| in Google Storage. e.g. 'x86-generic/R26-4000.0.0' |
| channel: The channel for the image. If none, it tries to guess it in |
| order of stability. |
| |
| Returns: |
| The GS URL for the directory where the signed image can be found. |
| |
| Raises: |
| build_artifact.ArtifactDownloadError: If we failed to download the |
| artifact. |
| """ |
| match = re.match(r'^([^/]+?)(?:-release)?/R\d+-(.*)$', build_id) |
| |
| channels = [] |
| if channel: |
| channels.append(channel) |
| else: |
| # Attempt to enumerate all channels, in order of stability. |
| channels.extend(devserver_constants.CHANNELS[::-1]) |
| |
| for channel in channels: |
| image_dir = devserver_constants.GS_CHANNEL_DIR % { |
| 'channel': channel, |
| 'board': match.group(1), |
| } |
| gs_url = os.path.join(image_dir, match.group(2)) |
| try: |
| self._LS(gs_url) |
| return gs_url |
| except gs.GSNoSuchKey: |
| continue |
| raise build_artifact.ArtifactDownloadError( |
| 'Could not find signed image URL for %s in Google Storage' % |
| build_id) |
| |
| def _GetFromGS(self, build_id, image_type, image_dir=None, channel=None): |
| """Check if the artifact is available locally. Download from GS if not. |
| |
| Args: |
| build_id: Path to the image or update directory on the devserver or |
| in Google Storage. e.g. 'x86-generic/R26-4000.0.0' |
| image_type: Image type to download. Look at aliases at top of file for |
| options. |
| image_dir: Google Storage image archive to search in if requesting a |
| remote artifact. If none uses the default bucket. |
| channel: The channel for the image. If none, it tries to guess it in |
| order of stability. |
| |
| Raises: |
| build_artifact.ArtifactDownloadError: If we failed to download the |
| artifact. |
| """ |
| # Stage image if not found in cache. |
| file_name = GS_ALIAS_TO_FILENAME[image_type] |
| file_loc = os.path.join(self.static_dir, build_id, file_name) |
| cached = os.path.exists(file_loc) |
| |
| if not cached: |
| artifact = GS_ALIAS_TO_ARTIFACT[image_type] |
| if image_type == SIGNED: |
| gs_url = self._TranslateSignedGSUrl(build_id, channel=channel) |
| else: |
| image_dir = XBuddy._ResolveImageDir(image_dir) |
| gs_url = os.path.join(image_dir, build_id) |
| self._Download(gs_url, [artifact], build_id) |
| else: |
| _Log('Image already cached.') |
| |
| def _GetArtifact(self, path_list, board=None, version=None, |
| lookup_only=False, image_dir=None): |
| """Interpret an xBuddy path and return directory/file_name to resource. |
| |
| Note board can be passed that in but by default if self._board is set, |
| that is used rather than board. |
| |
| Args: |
| path_list: [board, version, alias] as split from the xbuddy call url. |
| board: Board whos artifacts we are looking for. Only used if no board was |
| given during XBuddy initialization. |
| version: Version whose artifacts we are looking for. Used if no version |
| was given during XBuddy initialization. If None, defers to LATEST. |
| lookup_only: If true just look up the artifact, if False stage it on |
| the devserver as well. |
| image_dir: Google Storage image archive to search in if requesting a |
| remote artifact. If none uses the default bucket. |
| |
| Returns: |
| build_id: Path to the image or update directory on the devserver or |
| in Google Storage. e.g. 'x86-generic/R26-4000.0.0' |
| file_name: of the artifact in the build_id directory. |
| |
| Raises: |
| XBuddyException: if the path could not be translated |
| build_artifact.ArtifactDownloadError: if we failed to download the |
| artifact. |
| """ |
| path = '/'.join(path_list) |
| default_board = self._board if self._board else board |
| default_version = self._version or version or LATEST |
| # Rewrite the path if there is an appropriate default. |
| path, suffix = self.LookupAlias(path, board=default_board, |
| version=default_version) |
| # Parse the path. |
| image_type, board, version, is_local = self._InterpretPath( |
| path, default_board, default_version) |
| if is_local: |
| # Get a local image. |
| if version == LATEST: |
| # Get the latest local image for the given board. |
| version = self._GetLatestLocalVersion(board) |
| |
| build_id = os.path.join(board, version) |
| artifact_dir = os.path.join(self.static_dir, build_id) |
| if image_type == ANY: |
| image_type = self._FindAny(artifact_dir) |
| |
| file_name = LOCAL_ALIAS_TO_FILENAME[image_type] |
| artifact_path = os.path.join(artifact_dir, file_name) |
| if not os.path.exists(artifact_path): |
| raise XBuddyException('Local %s artifact not in static_dir at %s' % |
| (image_type, artifact_path)) |
| |
| else: |
| # Get a remote image. |
| if image_type not in GS_ALIASES: |
| raise XBuddyException('Bad remote image type: %s. Use one of: %s' % |
| (image_type, GS_ALIASES)) |
| build_id, channel = self._ResolveVersionToBuildIdAndChannel( |
| board, suffix, version, image_dir=image_dir) |
| _Log('Resolved version %s to %s.', version, build_id) |
| file_name = GS_ALIAS_TO_FILENAME[image_type] |
| if not lookup_only: |
| self._GetFromGS(build_id, image_type, image_dir=image_dir, |
| channel=channel) |
| |
| return build_id, file_name |
| |
| ############################ BEGIN PUBLIC METHODS |
| |
| def List(self): |
| """Lists the currently available images & time since last access.""" |
| self._SyncRegistryWithBuildImages() |
| builds = self._ListBuildTimes() |
| return_string = '' |
| for build, timestamp in builds: |
| return_string += '<b>' + build + '</b> ' |
| return_string += '(time since last access: ' + str(timestamp) + ')<br>' |
| return return_string |
| |
| def Capacity(self): |
| """Returns the number of images cached by xBuddy.""" |
| return str(self._Capacity()) |
| |
| def Translate(self, path_list, board=None, version=None, image_dir=None): |
| """Translates an xBuddy path to a real path to artifact if it exists. |
| |
| Equivalent to the Get call, minus downloading and updating timestamps, |
| |
| Args: |
| path_list: [board, version, alias] as split from the xbuddy call url. |
| board: Board whos artifacts we are looking for. If None, use the board |
| XBuddy was initialized to use. |
| version: Version whose artifacts we are looking for. If None, use the |
| version XBuddy was initialized with, or LATEST. |
| image_dir: image directory to check in Google Storage. If none, |
| the default bucket is used. |
| |
| Returns: |
| build_id: Path to the image or update directory on the devserver. |
| e.g. 'x86-generic/R26-4000.0.0' |
| The returned path is always the path to the directory within |
| static_dir, so it is always the build_id of the image. |
| file_name: The file name of the artifact. Can take any of the file |
| values in devserver_constants. |
| e.g. 'chromiumos_test_image.bin' or 'update.gz' if the path list |
| specified 'test' or 'full_payload' artifacts, respectively. |
| |
| Raises: |
| XBuddyException: if the path couldn't be translated |
| """ |
| self._SyncRegistryWithBuildImages() |
| build_id, file_name = self._GetArtifact(path_list, board=board, |
| version=version, |
| lookup_only=True, |
| image_dir=image_dir) |
| |
| _Log('Returning path to payload: %s/%s', build_id, file_name) |
| return build_id, file_name |
| |
| def StageTestArtifactsForUpdate(self, path_list): |
| """Stages test artifacts for update and returns build_id. |
| |
| Raises: |
| XBuddyException: if the path could not be translated |
| build_artifact.ArtifactDownloadError: if we failed to download the test |
| artifacts. |
| """ |
| build_id, file_name = self.Translate(path_list) |
| if file_name == devserver_constants.TEST_IMAGE_FILE: |
| gs_url = os.path.join(devserver_constants.GS_IMAGE_DIR, |
| build_id) |
| artifacts = [FULL, STATEFUL] |
| self._Download(gs_url, artifacts, build_id) |
| return build_id |
| |
| def Get(self, path_list, image_dir=None): |
| """The full xBuddy call, returns resource specified by path_list. |
| |
| Please see devserver.py:xbuddy for full documentation. |
| |
| Args: |
| path_list: [board, version, alias] as split from the xbuddy call url. |
| image_dir: image directory to check in Google Storage. If none, |
| the default bucket is used. |
| |
| Returns: |
| build_id: Path to the image or update directory on the devserver. |
| e.g. 'x86-generic/R26-4000.0.0' |
| The returned path is always the path to the directory within |
| static_dir, so it is always the build_id of the image. |
| file_name: The file name of the artifact. Can take any of the file |
| values in devserver_constants. |
| e.g. 'chromiumos_test_image.bin' or 'update.gz' if the path list |
| specified 'test' or 'full_payload' artifacts, respectively. |
| |
| Raises: |
| XBuddyException: if the path could not be translated |
| build_artifact.ArtifactDownloadError: if we failed to download the |
| artifact. |
| """ |
| self._SyncRegistryWithBuildImages() |
| build_id, file_name = self._GetArtifact(path_list, image_dir=image_dir) |
| Timestamp.UpdateTimestamp(self._timestamp_folder, build_id) |
| #TODO (joyc): run in sep thread |
| self.CleanCache() |
| |
| _Log('Returning path to payload: %s/%s', build_id, file_name) |
| return build_id, file_name |
