Allow xbuddy to accept any remote urls (including trybot urls).
This changes xbuddy to attempt to lookup a remote board by using the remote
path as is before trying it with the commonly used + -release suffix.
This also makes xbuddy fail faster if someone tries to use a remote url that
doesn't exist since the exception is raised during url resolution rather than
when we are trying to use the artifact.
BUG=chromium:306609
TEST=image_to_live with --image=xbuddy:remote/trybot-test-ap/R32-4777.0.0-b0
+ unit tests.
Change-Id: I352cd93ca2af2b048353dc349b08f734cf8095e7
Reviewed-on: https://chromium-review.googlesource.com/172764
Reviewed-by: Chris Sosa <sosa@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
diff --git a/devserver_constants.py b/devserver_constants.py
index f8f18c6..c0ac645 100644
--- a/devserver_constants.py
+++ b/devserver_constants.py
@@ -8,8 +8,8 @@
# TODO (joyc) move the google storage filenames of artfacts here
CHANNELS = 'canary', 'dev', 'beta', 'stable'
GS_IMAGE_DIR = 'gs://chromeos-image-archive'
-GS_LATEST_MASTER = GS_IMAGE_DIR + '/%(board)s-%(suffix)s/LATEST-master'
-IMAGE_DIR = '%(board)s-%(suffix)s/%(version)s'
+GS_LATEST_MASTER = GS_IMAGE_DIR + '/%(board)s%(suffix)s/LATEST-master'
+IMAGE_DIR = '%(board)s%(suffix)s/%(version)s'
GS_RELEASES_DIR = 'gs://chromeos-releases'
GS_CHANNEL_DIR = GS_RELEASES_DIR + '/%(channel)s-channel/%(board)s/'
diff --git a/xbuddy.py b/xbuddy.py
index 5fff21a..7b5c857 100644
--- a/xbuddy.py
+++ b/xbuddy.py
@@ -106,7 +106,7 @@
LATEST_OFFICIAL = "latest-official"
-RELEASE = "release"
+RELEASE = "-release"
class XBuddyException(Exception):
@@ -288,7 +288,6 @@
return devserver_constants.IMAGE_DIR % {'board':board,
'suffix':suffix,
'version':version}
-
def _LookupChannel(self, board, channel='stable'):
"""Check the channel folder for the version number of interest."""
# Get all names in channel dir. Get 10 highest directories by version.
@@ -327,7 +326,33 @@
'suffix':RELEASE,
'version':full_version}
- def _ResolveVersionToUrl(self, board, version):
+ def _RemoteBuildId(self, board, version):
+ """Returns the remote build_id for the given board and version.
+
+ Raises:
+ XBuddyException: If we failed to resolve the version to a valid build_id.
+ """
+ build_id_as_is = devserver_constants.IMAGE_DIR % {'board':board,
+ 'suffix':'',
+ 'version':version}
+ build_id_release = devserver_constants.IMAGE_DIR % {'board':board,
+ 'suffix':RELEASE,
+ 'version':version}
+ # Return the first path that exists. We assume that what the user typed
+ # is better than with a default suffix added i.e. x86-generic/blah is
+ # more valuable than x86-generic-release/blah.
+ for build_id in build_id_as_is, build_id_release:
+ cmd = 'gsutil ls %s/%s' % (devserver_constants.GS_IMAGE_DIR, build_id)
+ try:
+ version = gsutil_util.GSUtilRun(cmd, None)
+ return build_id
+ except gsutil_util.GSUtilError:
+ continue
+ else:
+ raise XBuddyException('Could not find remote build_id for %s %s' % (
+ board, version))
+
+ def _ResolveVersionToBuildId(self, board, version):
"""Handle version aliases for remote payloads in GS.
Args:
@@ -340,19 +365,16 @@
4. version prefix (i.e. RX-Y.X, RX-Y, RX)
Returns:
- Location where the image dir is actually found on GS
+ Location where the image dir is actually found on GS (build_id)
+ Raises:
+ XBuddyException: If we failed to resolve the version to a valid url.
"""
- # TODO(joychen): Convert separate calls to a dict + error out bad paths.
-
# Only the last segment of the alias is variable relative to the rest.
version_tuple = version.rsplit('-', 1)
if re.match(devserver_constants.VERSION_RE, version):
- # This is supposed to be a complete version number on GS. Return it.
- return devserver_constants.IMAGE_DIR % {'board':board,
- 'suffix':RELEASE,
- 'version':version}
+ return self._RemoteBuildId(board, version)
elif version == LATEST_OFFICIAL:
# latest-official --> LATEST build in board-release
return self._LookupOfficial(board)
@@ -628,7 +650,7 @@
if image_type not in GS_ALIASES:
raise XBuddyException('Bad remote image type: %s. Use one of: %s' %
(image_type, GS_ALIASES))
- build_id = self._ResolveVersionToUrl(board, version)
+ build_id = self._ResolveVersionToBuildId(board, version)
_Log('Resolved version %s to %s.', version, build_id)
file_name = GS_ALIAS_TO_FILENAME[image_type]
if not lookup_only:
diff --git a/xbuddy_unittest.py b/xbuddy_unittest.py
index a509541..9bd2a31 100755
--- a/xbuddy_unittest.py
+++ b/xbuddy_unittest.py
@@ -54,7 +54,7 @@
mox.IgnoreArg()).AndReturn('v')
expected = 'b-s/v'
self.mox.ReplayAll()
- self.assertEqual(self.mock_xb._LookupOfficial('b', 's'), expected)
+ self.assertEqual(self.mock_xb._LookupOfficial('b', '-s'), expected)
self.mox.VerifyAll()
def testLookupChannel(self):
@@ -70,24 +70,25 @@
self.assertEqual(self.mock_xb._LookupChannel('b'), expected)
self.mox.VerifyAll()
- def testResolveVersionToUrl_Official(self):
- """Check _ResolveVersionToUrl recognizes aliases for official builds."""
+ def testResolveVersionToBuildId_Official(self):
+ """Check _ResolveVersionToBuildId recognizes aliases for official builds."""
board = 'b'
# aliases that should be redirected to LookupOfficial
+
self.mox.StubOutWithMock(self.mock_xb, '_LookupOfficial')
self.mock_xb._LookupOfficial(board)
self.mock_xb._LookupOfficial(board, 'paladin')
self.mox.ReplayAll()
version = 'latest-official'
- self.mock_xb._ResolveVersionToUrl(board, version)
+ self.mock_xb._ResolveVersionToBuildId(board, version)
version = 'latest-official-paladin'
- self.mock_xb._ResolveVersionToUrl(board, version)
+ self.mock_xb._ResolveVersionToBuildId(board, version)
self.mox.VerifyAll()
- def testResolveVersionToUrl_Channel(self):
- """Check _ResolveVersionToUrl recognizes aliases for channels."""
+ def testResolveVersionToBuildId_Channel(self):
+ """Check _ResolveVersionToBuildId recognizes aliases for channels."""
board = 'b'
# aliases that should be redirected to LookupChannel
@@ -97,9 +98,9 @@
self.mox.ReplayAll()
version = 'latest'
- self.mock_xb._ResolveVersionToUrl(board, version)
+ self.mock_xb._ResolveVersionToBuildId(board, version)
version = 'latest-dev'
- self.mock_xb._ResolveVersionToUrl(board, version)
+ self.mock_xb._ResolveVersionToBuildId(board, version)
self.mox.VerifyAll()
def testBasicInterpretPath(self):
@@ -184,11 +185,18 @@
"""Caching & replacement of timestamp files."""
path_a = ('remote', 'a', 'R0', 'test')
path_b = ('remote', 'b', 'R0', 'test')
-
+ self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
+ self.mox.StubOutWithMock(self.mock_xb, '_Download')
self.mox.StubOutWithMock(self.mock_xb, '_Download')
for _ in range(8):
self.mock_xb._Download(mox.IsA(str), mox.In(mox.IsA(str)))
+ # All non-release urls are invalid so we can meet expectations.
+ gsutil_util.GSUtilRun(mox.Not(mox.StrContains('-release')),
+ None).MultipleTimes().AndRaise(
+ gsutil_util.GSUtilError('bad url'))
+ gsutil_util.GSUtilRun(mox.StrContains('-release'), None).MultipleTimes()
+
self.mox.ReplayAll()
# requires default capacity
