blob: c02ebe2901d2d06f6ac2db0aad56babd583fd7a6 [file] [log] [blame]
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import collections
import os
import threading
from datetime import datetime
import build_artifact
import common_util
import log_util
class DownloaderException(Exception):
"""Exception that aggregates all exceptions raised during async download.
Exceptions could be raised in artifact.Process method, and saved to files.
When caller calls IsStaged to check the downloading progress, devserver can
retrieve the persisted exceptions from the files, wrap them into a
DownloaderException, and raise it.
def __init__(self, exceptions):
"""Initialize a DownloaderException instance with a list of exceptions.
@param exceptions: Exceptions raised when downloading artifacts.
message = 'Exceptions were raised when downloading artifacts.'
Exception.__init__(self, message)
self.exceptions = exceptions
def __repr__(self):
return self.__str__()
def __str__(self):
"""Return a custom exception message with all exceptions merged."""
return '--------\n'.join([str(exception) for exception in self.exceptions])
class Downloader(log_util.Loggable):
"""Downloader of images to the devsever.
Given a URL to a build on the archive server:
- Caches that build and the given artifacts onto the devserver.
- May also initiate caching of related artifacts in the background.
Private class members:
archive_url: a URL where to download build artifacts from.
static_dir: local filesystem directory to store all artifacts.
build_dir: the local filesystem directory to store artifacts for the given
build defined by the archive_url.
# This filename must be kept in sync with
_TIMESTAMP_FILENAME = 'staged.timestamp'
def __init__(self, static_dir, archive_url):
super(Downloader, self).__init__()
self._archive_url = archive_url
self._static_dir = static_dir
self._build_dir = Downloader.GetBuildDir(static_dir, archive_url)
def ParseUrl(archive_url):
"""Parses archive_url into rel_path and build.
Parses archive_url into rel_path and build e.g.
archive_url: a URL at which build artifacts are archived.
A tuple of (build relative path, short build name)
# The archive_url is of the form gs://server/[some_path/target]/...]/build
# This function discards 'gs://server/' and extracts the [some_path/target]
# as rel_path and the build as build.
sub_url = archive_url.partition('://')[2]
split_sub_url = sub_url.split('/')
rel_path = '/'.join(split_sub_url[1:-1])
build = split_sub_url[-1]
return rel_path, build
def GetBuildDir(static_dir, archive_url):
"""Returns the path to where the artifacts will be staged.
static_dir: The base static dir that will be used.
archive_url: The gs path to the archive url.
# Parse archive_url into rel_path (contains the build target) and
# build e.g. gs://chromeos-image-archive/{rel_path}/{build}.
rel_path, build = Downloader.ParseUrl(archive_url)
return os.path.join(static_dir, rel_path, build)
def _TouchTimestampForStaged(directory_path):
file_name = os.path.join(directory_path, Downloader._TIMESTAMP_FILENAME)
# Easiest python version of |touch file_name|
with file(file_name, 'a'):
os.utime(file_name, None)
def _TryRemoveStageDir(directory_path):
"""If download failed, try to remove the stage dir.
If the download attempt failed (ArtifactDownloadError) and staged.timestamp
is the only file in that directory. The build could be non-existing, and
the directory should be removed.
@param directory_path: directory used to stage the image.
file_name = os.path.join(directory_path, Downloader._TIMESTAMP_FILENAME)
if os.path.exists(file_name) and len(os.listdir(directory_path)) == 1:
def ListBuildDir(self):
"""List the files in the build directory.
Only lists files a single level into the build directory. Includes
timestamp information in the listing.
A string with information about the files in the build directory.
None if the build directory doesn't exist.
build_artifact.ArtifactDownloadError: If the build_dir path exists
but is not a directory.
if not os.path.exists(self._build_dir):
return None
if not os.path.isdir(self._build_dir):
raise build_artifact.ArtifactDownloadError(
'Artifacts %s improperly staged to build_dir path %s. The path is '
'not a directory.' % (self._archive_url, self._build_dir))
ls_format = collections.namedtuple(
'ls', ['name', 'accessed', 'modified', 'size'])
output_format = ('Name: %(name)s Accessed: %(accessed)s '
'Modified: %(modified)s Size: %(size)s bytes.\n')
build_dir_info = 'Listing contents of :%s \n' % self._build_dir
for file_name in os.listdir(self._build_dir):
file_path = os.path.join(self._build_dir, file_name)
file_info = os.stat(file_path)
ls_info = ls_format(file_path,
build_dir_info += output_format % ls_info._asdict()
return build_dir_info
def Download(self, artifacts, files, async=False):
"""Downloads and caches the |artifacts|.
Downloads and caches the |artifacts|. Returns once these
are present on the devserver. A call to this will attempt to cache
non-specified artifacts in the background following the principle of
spatial locality.
artifacts: A list of artifact names that correspond to
artifacts defined in to stage.
files: A list of filenames to stage from an archive_url.
async: If True, return without waiting for download to complete.
build_artifact.ArtifactDownloadError: If failed to download the artifact.
# We are doing some work on this build -- let's touch it to indicate that
# we shouldn't be cleaning it up anytime soon.
# Create factory to create build_artifacts from artifact names.
build = self.ParseUrl(self._archive_url)[1]
factory = build_artifact.ArtifactFactory(
self._build_dir, self._archive_url, artifacts, files,
background_artifacts = factory.OptionalArtifacts()
if background_artifacts:
required_artifacts = factory.RequiredArtifacts()
str_repr = [str(a) for a in required_artifacts]
self._Log('Downloading artifacts %s.', ' '.join(str_repr))
if async:
self._DownloadArtifactsSerially(required_artifacts, no_wait=True)
def IsStaged(self, artifacts, files):
"""Check if all artifacts have been downloaded.
artifacts: A list of artifact names that correspond to
artifacts defined in to stage.
files: A list of filenames to stage from an archive_url.
@returns: True if all artifacts are staged.
@raise exception: that was raised by any artifact when calling Process.
# Create factory to create build_artifacts from artifact names.
build = self.ParseUrl(self._archive_url)[1]
factory = build_artifact.ArtifactFactory(
self._build_dir, self._archive_url, artifacts, files, build)
required_artifacts = factory.RequiredArtifacts()
exceptions = [artifact.GetException() for artifact in required_artifacts if
if exceptions:
raise DownloaderException(exceptions)
return all([artifact.ArtifactStaged() for artifact in required_artifacts])
def _DownloadArtifactsSerially(self, artifacts, no_wait):
"""Simple function to download all the given artifacts serially.
artifacts: A list of build_artifact.BuildArtifact instances to
no_wait: If True, don't block waiting for artifact to exist if we
fail to immediately find it.
build_artifact.ArtifactDownloadError: If we failed to download the
for artifact in artifacts:
except build_artifact.ArtifactDownloadError:
def _DownloadArtifactsInBackground(self, artifacts):
"""Downloads |artifacts| in the background.
Downloads |artifacts| in the background. As these are backgrounded
artifacts, they are done best effort and may not exist.
artifacts: List of build_artifact.BuildArtifact instances to download.
self._Log('Invoking background download of artifacts for %r', artifacts)
thread = threading.Thread(target=self._DownloadArtifactsSerially,
args=(artifacts, False))