dev: Let devserver use gsutil functions in chromite.
Together with CL:378576, this CL is to make platform/dev to use GS-related
functions in chromite. This could help reduce the maintain cost and
inconsistency errors caused by different versions of gsutil.
BUG=chromium:640609
TEST=Ran unittest.
Test 2 functions with local devserver: trigger_download &
get_latest_build_in_gs (including xbuddy/latest, xbuddy/latest-official,
xbuddy/latest-{dev|beta|stable}, xbuddy/latest-R53).
Change-Id: I27f97ebebb76ea6b6b5ef9cc4d5b81c524f6968c
Reviewed-on: https://chromium-review.googlesource.com/381693
Commit-Ready: Xixuan Wu <xixuan@chromium.org>
Tested-by: Xixuan Wu <xixuan@chromium.org>
Reviewed-by: Don Garrett <dgarrett@chromium.org>
diff --git a/downloader.py b/downloader.py
index 3bd0d03..37dd54b 100755
--- a/downloader.py
+++ b/downloader.py
@@ -1,5 +1,5 @@
-#!/usr/bin/python2
-#
+#!/usr/bin/env python2
+
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -18,9 +18,16 @@
import build_artifact
import common_util
-import gsutil_util
import log_util
+# Make sure that chromite is available to import.
+import setup_chromite # pylint: disable=unused-import
+
+try:
+ from chromite.lib import gs
+except ImportError as e:
+ gs = None
+
try:
import android_build
except ImportError as e:
@@ -167,7 +174,6 @@
Raises:
build_artifact.ArtifactDownloadError: If failed to download the artifact.
-
"""
common_util.MkDirP(self._build_dir)
@@ -202,7 +208,6 @@
Raises:
DownloaderException: A wrapper for exceptions raised by any artifact when
calling Process.
-
"""
required_artifacts = factory.RequiredArtifacts()
exceptions = [artifact.GetException() for artifact in required_artifacts if
@@ -224,7 +229,6 @@
Raises:
build_artifact.ArtifactDownloadError: If we failed to download the
artifact.
-
"""
try:
for artifact in artifacts:
@@ -304,6 +308,8 @@
self._archive_url = archive_url
+ self._ctx = gs.GSContext() if gs else None
+
def Wait(self, name, is_regex_name, timeout):
"""Waits for artifact to exist and returns the appropriate names.
@@ -318,8 +324,8 @@
Raises:
ArtifactDownloadError: An error occurred when obtaining artifact.
"""
- names = gsutil_util.GetGSNamesWithWait(
- name, self._archive_url, str(self), timeout=timeout,
+ names = self._ctx.GetGsNamesWithWait(
+ name, self._archive_url, timeout=timeout,
is_regex_pattern=is_regex_name)
if not names:
raise build_artifact.ArtifactDownloadError(
@@ -331,7 +337,7 @@
"""Downloads artifact from Google Storage to a local directory."""
install_path = os.path.join(local_path, remote_name)
gs_path = '/'.join([self._archive_url, remote_name])
- gsutil_util.DownloadFromGS(gs_path, local_path)
+ self._ctx.Copy(gs_path, local_path)
return install_path
def DescribeSource(self):
diff --git a/xbuddy.py b/xbuddy.py
index d0bd8e3..b0fa5f5 100644
--- a/xbuddy.py
+++ b/xbuddy.py
@@ -9,6 +9,7 @@
import cherrypy
import ConfigParser
import datetime
+import distutils.version
import operator
import os
import re
@@ -22,9 +23,16 @@
import common_util
import devserver_constants
import downloader
-import gsutil_util
import log_util
+# Make sure that chromite is available to import.
+import setup_chromite # pylint: disable=unused-import
+
+try:
+ from chromite.lib import gs
+except ImportError as e:
+ gs = None
+
# Module-local log function.
def _Log(message, *args):
return log_util.LogWithTag('XBUDDY', message, *args)
@@ -205,6 +213,8 @@
else:
self.images_dir = os.path.join(self.GetSourceRoot(), 'src/build/images')
+ self._ctx = gs.GSContext() if gs else None
+
common_util.MkDirP(self._timestamp_folder)
@classmethod
@@ -339,15 +349,57 @@
{'image_dir': image_dir,
'board': board,
'suffix': suffix})
- cmd = 'gsutil cat %s' % latest_addr
- msg = 'Failed to find build at %s' % latest_addr
# Full release + version is in the LATEST file.
- version = gsutil_util.GSUtilRun(cmd, msg)
+ version = self._ctx.Cat(latest_addr)
return devserver_constants.IMAGE_DIR % {'board':board,
'suffix':suffix,
'version':version}
+ def _LS(self, path, list_subdirectory=False):
+ """Does a directory listing of the given gs path.
+
+ Args:
+ path: directory location on google storage to check.
+ list_subdirectory: whether to only list subdirectory for |path|.
+
+ Returns:
+ A list of paths that matched |path|.
+ """
+ if list_subdirectory:
+ return self._ctx.DoCommand(
+ ['ls', '-d', '--', path]).output.splitlines()
+ else:
+ return self._ctx.LS(path)
+
+ def _GetLatestVersionFromGsDir(self, path, list_subdirectory=False,
+ with_release=True):
+ """Returns most recent version number found in a google storage directory.
+
+ This lists out the contents of the given GS bucket or regex to GS buckets,
+ and tries to grab the newest version found in the directory names.
+
+ Args:
+ path: directory location on google storage to check.
+ list_subdirectory: whether to only list subdirectory for |path|.
+ with_release: whether versions include a release milestone (e.g. R12).
+
+ Returns:
+ The most recent version number found.
+ """
+ list_result = self._LS(path, list_subdirectory=list_subdirectory)
+ dir_names = [os.path.basename(p.rstrip('/')) for p in list_result]
+ try:
+ filter_re = re.compile(devserver_constants.VERSION_RE if with_release
+ else devserver_constants.VERSION)
+ versions = filter(filter_re.match, dir_names)
+ latest_version = max(versions, key=distutils.version.LooseVersion)
+ except ValueError:
+ raise gs.GSContextException(
+ 'Failed to find most recent builds at %s' % path)
+
+ return latest_version
+
def _LookupChannel(self, board, suffix, channel='stable',
image_dir=None):
"""Check the channel folder for the version number of interest."""
@@ -355,12 +407,12 @@
_Log("Checking channel '%s' for latest '%s' image", channel, board)
# Due to historical reasons, gs://chromeos-releases uses
# daisy-spring as opposed to the board name daisy_spring. Convert
- # the board name for the lookup.
+ # he board name for the lookup.
channel_dir = devserver_constants.GS_CHANNEL_DIR % {
'channel':channel,
'board':re.sub('_', '-', board)}
- latest_version = gsutil_util.GetLatestVersionFromGSDir(
- channel_dir, with_release=False)
+ latest_version = self._GetLatestVersionFromGsDir(channel_dir,
+ with_release=False)
# Figure out release number from the version number.
image_url = devserver_constants.IMAGE_DIR % {
@@ -371,7 +423,8 @@
gs_url = os.path.join(image_dir, image_url)
# There should only be one match on cros-image-archive.
- full_version = gsutil_util.GetLatestVersionFromGSDir(gs_url)
+ full_version = self._GetLatestVersionFromGsDir(gs_url,
+ list_subdirectory=True)
return devserver_constants.IMAGE_DIR % {'board': board,
'suffix': suffix,
@@ -387,7 +440,8 @@
image_dir = os.path.join(devserver_constants.GS_IMAGE_DIR, image_url)
# Grab the newest version of the ones matched.
- full_version = gsutil_util.GetLatestVersionFromGSDir(image_dir)
+ full_version = self._GetLatestVersionFromGsDir(image_dir,
+ list_subdirectory=True)
return devserver_constants.IMAGE_DIR % {'board': board,
'suffix': suffix,
'version': full_version}
@@ -408,11 +462,11 @@
# is better than with a default suffix added i.e. x86-generic/blah is
# more valuable than x86-generic-release/blah.
for build_id in build_id_as_is, build_id_suffix:
- cmd = 'gsutil ls %s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id)
try:
- version = gsutil_util.GSUtilRun(cmd, None)
+ version = self._ctx.LS(
+ '%s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id))
return build_id
- except gsutil_util.GSUtilError:
+ except (gs.GSCommandError, gs.GSContextException, gs.GSNoSuchKey):
continue
raise XBuddyException('Could not find remote build_id for %s %s' % (
@@ -428,10 +482,8 @@
'board': board,
'suffix': suffix,
'base_version': base_version})
- cmd = 'gsutil cat %s' % latest_addr
- msg = 'Failed to find build at %s' % latest_addr
# Full release + version is in the LATEST file.
- return gsutil_util.GSUtilRun(cmd, msg)
+ return self._ctx.Cat(latest_addr)
def _ResolveVersionToBuildId(self, board, suffix, version, image_dir=None):
"""Handle version aliases for remote payloads in GS.
diff --git a/xbuddy_unittest.py b/xbuddy_unittest.py
index fb4f3d1..5fe7e15 100755
--- a/xbuddy_unittest.py
+++ b/xbuddy_unittest.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python2
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -17,14 +17,21 @@
import mox
-import gsutil_util
import xbuddy
+# Make sure that chromite is available to import.
+import setup_chromite # pylint: disable=unused-import
+
+try:
+ from chromite.lib import gs
+except ImportError as e:
+ gs = None
+
#pylint: disable=W0212
+#pylint: disable=no-value-for-parameter
GS_ALTERNATE_DIR = 'gs://chromeos-alternate-archive/'
-
class xBuddyTest(mox.MoxTestBase):
"""Regression tests for xbuddy."""
def setUp(self):
@@ -52,11 +59,40 @@
self.assertEqual(xbuddy.XBuddy.ParseBoolean('true'), True)
self.assertEqual(xbuddy.XBuddy.ParseBoolean('y'), True)
+ def testGetLatestVersionFromGsDir(self):
+ """Test that we can get the most recent version from gsutil calls."""
+ self.mox.StubOutWithMock(self.mock_xb, '_LS')
+ mock_data1 = """gs://chromeos-releases/stable-channel/parrot/3701.96.0/
+ gs://chromeos-releases/stable-channel/parrot/3701.98.0/
+ gs://chromeos-releases/stable-channel/parrot/3912.100.0/
+ gs://chromeos-releases/stable-channel/parrot/3912.101.0/
+ gs://chromeos-releases/stable-channel/parrot/3912.79.0/
+ gs://chromeos-releases/stable-channel/parrot/3912.79.1/"""
+
+ mock_data2 = """gs://chromeos-image-archive/parrot-release/R26-3912.101.0
+ gs://chromeos-image-archive/parrot-release/R27-3912.101.0
+ gs://chromeos-image-archive/parrot-release/R28-3912.101.0"""
+
+ self.mock_xb._LS(mox.IgnoreArg(), list_subdirectory=False).AndReturn(
+ mock_data1.splitlines())
+ self.mock_xb._LS(mox.IgnoreArg(), list_subdirectory=True).AndReturn(
+ mock_data2.splitlines())
+
+ self.mox.ReplayAll()
+ url = ''
+ self.assertEqual(
+ self.mock_xb._GetLatestVersionFromGsDir(url, with_release=False),
+ '3912.101.0')
+ self.assertEqual(
+ self.mock_xb._GetLatestVersionFromGsDir(url, list_subdirectory=True,
+ with_release=True),
+ 'R28-3912.101.0')
+ self.mox.VerifyAll()
+
def testLookupOfficial(self):
"""Basic test of _LookupOfficial. Checks that a given suffix is handled."""
- self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
- gsutil_util.GSUtilRun(mox.IgnoreArg(),
- mox.IgnoreArg()).AndReturn('v')
+ self.mox.StubOutWithMock(gs.GSContext, 'Cat')
+ gs.GSContext.Cat(mox.IgnoreArg()).AndReturn('v')
expected = 'b-s/v'
self.mox.ReplayAll()
self.assertEqual(self.mock_xb._LookupOfficial('b', suffix='-s'), expected)
@@ -64,12 +100,13 @@
def testLookupChannel(self):
"""Basic test of _LookupChannel. Checks that a given suffix is handled."""
- self.mox.StubOutWithMock(gsutil_util, 'GetLatestVersionFromGSDir')
+ self.mox.StubOutWithMock(self.mock_xb, '_GetLatestVersionFromGsDir')
mock_data1 = '4100.68.0'
- gsutil_util.GetLatestVersionFromGSDir(
+ self.mock_xb._GetLatestVersionFromGsDir(
mox.IgnoreArg(), with_release=False).AndReturn(mock_data1)
mock_data2 = 'R28-4100.68.0'
- gsutil_util.GetLatestVersionFromGSDir(mox.IgnoreArg()).AndReturn(mock_data2)
+ self.mock_xb._GetLatestVersionFromGsDir(
+ mox.IgnoreArg(), list_subdirectory=True).AndReturn(mock_data2)
self.mox.ReplayAll()
expected = 'b-release/R28-4100.68.0'
self.assertEqual(self.mock_xb._LookupChannel('b', '-release'),
@@ -297,16 +334,16 @@
"""Caching & replacement of timestamp files."""
path_a = ('remote', 'a', 'R0', 'test')
path_b = ('remote', 'b', 'R0', 'test')
- self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
+ self.mox.StubOutWithMock(gs.GSContext, 'LS')
self.mox.StubOutWithMock(self.mock_xb, '_Download')
for _ in range(8):
self.mock_xb._Download(mox.IsA(str), mox.In(mox.IsA(str)))
# All non-release urls are invalid so we can meet expectations.
- gsutil_util.GSUtilRun(mox.Not(mox.StrContains('-release')),
- None).MultipleTimes().AndRaise(
- gsutil_util.GSUtilError('bad url'))
- gsutil_util.GSUtilRun(mox.StrContains('-release'), None).MultipleTimes()
+ gs.GSContext.LS(
+ mox.Not(mox.StrContains('-release'))).MultipleTimes().AndRaise(
+ gs.GSContextException('bad url'))
+ gs.GSContext.LS(mox.StrContains('-release')).MultipleTimes()
self.mox.ReplayAll()
