blob: a264cf7103d2f4926578fff007b96d8e64c8b987 [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module containing gsutil helper methods."""
import random
import re
import subprocess
import time
import log_util
# Module-local log function.
def _Log(message, *args):
return log_util.LogWithTag('GSUTIL_UTIL', message, *args)
class GSUtilError(Exception):
"""Exception raised when we run into an error running gsutil."""
class PatternNotSpecific(Exception):
"""Raised when unexpectedly more than one item is returned for a pattern."""
def GSUtilRun(cmd, err_msg):
"""Runs a GSUTIL command up to GSUTIL_ATTEMPTS number of times.
Attempts are tried with exponential backoff.
stdout of the called gsutil command.
subprocess.CalledProcessError if all attempt to run gsutil cmd fails.
proc = None
sleep_timeout = 1
stderr = None
for _attempt in range(GSUTIL_ATTEMPTS):
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stdout, stderr = proc.communicate()
if proc.returncode == 0:
return stdout
elif 'matched no objects' in stderr or 'non-existent object' in stderr:
# TODO(sosa): Note this is a heuristic that makes us not re-attempt
# unnecessarily. However, if it fails, the worst that can happen is just
# waiting longer than necessary.
sleep_timeout *= 2
raise GSUtilError('%s GSUTIL cmd %s failed with return code %d:\n\n%s' % (
err_msg, cmd, proc.returncode, stderr))
def DownloadFromGS(src, dst):
"""Downloads object from gs_url |src| to |dst|.
GSUtilError: if an error occurs during the download.
cmd = 'gsutil cp %s %s' % (src, dst)
msg = 'Failed to download "%s".' % src
GSUtilRun(cmd, msg)
def _GetGSNamesFromList(filename_list, pattern):
"""Given a list of filenames, returns the filenames that match pattern."""
matches = []
re_pattern = re.compile(pattern)
for filename in filename_list:
if re_pattern.match(filename):
return matches
def GetGSNamesWithWait(pattern, archive_url, err_str, single_item=True,
timeout=600, delay=10):
"""Returns the google storage names specified by the given pattern.
This method polls Google Storage until the target artifacts specified by the
pattern is available or until the timeout occurs. Because we may not know the
exact name of the target artifacts, the method accepts a filename pattern,
to identify whether an artifact whose name matches the pattern exists (e.g.
use pattern '_full_' to search for the full payload
'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'). Returns the name only if
found before the timeout.
pattern: Regular expression pattern to identify the target artifact.
archive_url: URL of the Google Storage bucket.
err_str: String to display in the error message on error.
single_item: Only a single item should be returned.
timeout/delay: optional and self-explanatory.
The list of artifacts matching the pattern in Google Storage bucket or None
if not found.
PatternNotSpecific: If caller sets single_item but multiple items match.
deadline = time.time() + timeout
while time.time() <= deadline:
uploaded_list = []
to_delay = delay + random.uniform(.5 * delay, 1.5 * delay)
cmd = 'gsutil cat %s/%s' % (archive_url, UPLOADED_LIST)
msg = 'Failed to get a list of uploaded files.'
uploaded_list = GSUtilRun(cmd, msg).splitlines()
except GSUtilError:
# For backward compatibility, falling back to use "gsutil ls"
# when the manifest file is not present.
cmd = 'gsutil ls %s/*' % archive_url
msg = 'Failed to list payloads.'
returned_list = GSUtilRun(cmd, msg).splitlines()
for item in returned_list:
uploaded_list.append(item.rsplit('/', 1)[1])
# Check if all target artifacts are available.
found_names = _GetGSNamesFromList(uploaded_list, pattern)
if found_names:
if single_item and len(found_names) > 1:
raise PatternNotSpecific(
'Too many items %s returned by pattern %s in %s' % (
str(found_names), pattern, archive_url))
return found_names
_Log('Retrying in %f seconds...%s', to_delay, err_str)
return None