blob: 22d3fe59c4f7e9f93cd8c190e0b1a0a8024f8510 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module containing gsutil helper methods."""
import distutils.version
import logging
import random
import re
import subprocess
import time
import devserver_constants
import log_util
# Module-local log function.
def _Log(message, *args):
return log_util.LogWithTag('GSUTIL_UTIL', message, *args)
class GSUtilError(Exception):
"""Exception raised when we run into an error running gsutil."""
class PatternNotSpecific(Exception):
"""Raised when unexpectedly more than one item is returned for a pattern."""
def GSUtilRun(cmd, err_msg):
"""Runs a GSUTIL command up to GSUTIL_ATTEMPTS number of times.
Attempts are tried with exponential backoff.
stdout of the called gsutil command.
subprocess.CalledProcessError if all attempt to run gsutil cmd fails.
proc = None
sleep_timeout = 1
stderr = None
for _attempt in range(GSUTIL_ATTEMPTS):
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stdout, stderr = proc.communicate()
if proc.returncode == 0:
return stdout
elif stderr and ('matched no objects' in stderr or
'non-existent object' in stderr):
# TODO(sosa): Note this is a heuristic that makes us not re-attempt
# unnecessarily. However, if it fails, the worst that can happen is just
# waiting longer than necessary.
elif proc.returncode == 127:
raise GSUtilError('gsutil tool not found in your path.')
sleep_timeout *= 2
raise GSUtilError('%s GSUTIL cmd %s failed with return code %d:\n\n%s' % (
err_msg, cmd, proc.returncode, stderr))
def DownloadFromGS(src, dst):
"""Downloads object from gs_url |src| to |dst|.
GSUtilError: if an error occurs during the download.
cmd = 'gsutil cp %s %s' % (src, dst)
msg = 'Failed to download "%s".' % src
GSUtilRun(cmd, msg)
def _GetGSNamesFromList(filename_list, pattern):
"""Given a list of filenames, returns the filenames that match pattern."""
matches = []
re_pattern = re.compile(pattern)
for filename in filename_list:
if re_pattern.match(filename):
return matches
def GetGSNamesWithWait(pattern, archive_url, err_str, single_item=True,
timeout=600, delay=10):
"""Returns the google storage names specified by the given pattern.
This method polls Google Storage until the target artifacts specified by the
pattern is available or until the timeout occurs. Because we may not know the
exact name of the target artifacts, the method accepts a filename pattern,
to identify whether an artifact whose name matches the pattern exists (e.g.
use pattern '_full_' to search for the full payload
'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'). Returns the name only if
found before the timeout.
pattern: Regular expression pattern to identify the target artifact.
archive_url: URL of the Google Storage bucket.
err_str: String to display in the error message on error.
single_item: Only a single item should be returned. If more than one item
matches the pattern errors out unless pattern matches one
timeout/delay: optional and self-explanatory.
The list of artifacts matching the pattern in Google Storage bucket or None
if not found.
PatternNotSpecific: If caller sets single_item but multiple items match.
deadline = time.time() + timeout
while True:
uploaded_list = []
cmd = 'gsutil cat %s/%s' % (archive_url, UPLOADED_LIST)
msg = 'Failed to get a list of uploaded files.'
uploaded_list = GSUtilRun(cmd, msg).splitlines()
except GSUtilError:
# For backward compatibility, falling back to use "gsutil ls"
# when the manifest file is not present.
cmd = 'gsutil ls %s/*' % archive_url
msg = 'Failed to list payloads.'
returned_list = GSUtilRun(cmd, msg).splitlines()
for item in returned_list:
uploaded_list.append(item.rsplit('/', 1)[1])
except IndexError:
# Check if all target artifacts are available.
found_names = _GetGSNamesFromList(uploaded_list, pattern)
if found_names:
if single_item and len(found_names) > 1:
found_names_exact = _GetGSNamesFromList(uploaded_list, '^%s$' % pattern)
if not found_names_exact:
raise PatternNotSpecific(
'Too many items %s returned by pattern %s in %s' % (
str(found_names), pattern, archive_url))
else:'More than one item returned but one file matched'
' exactly so returning that: %s.', found_names_exact)
found_names = found_names_exact
return found_names
# Don't delay past deadline.
to_delay = random.uniform(1.5 * delay, 2.5 * delay)
if to_delay < (deadline - time.time()):
_Log('Retrying in %f seconds...%s', to_delay, err_str)
return None
def GetLatestVersionFromGSDir(gsutil_dir, with_release=True):
"""Returns most recent version number found in a GS directory.
This lists out the contents of the given GS bucket or regex to GS buckets,
and tries to grab the newest version found in the directory names.
cmd = 'gsutil ls %s' % gsutil_dir
msg = 'Failed to find most recent builds at %s' % gsutil_dir
dir_names = [p.split('/')[-2] for p in GSUtilRun(cmd, msg).splitlines()]
if with_release:
versions = filter(lambda x: re.match(devserver_constants.VERSION_RE, x),
versions = filter(lambda x: re.match(devserver_constants.VERSION, x),
latest_version = max(versions, key=distutils.version.LooseVersion)
except ValueError:
raise GSUtilError(msg)
return latest_version