autoupdate_EndToEndTest: Use quick-provision for installing the source image
Currently, autoupdate_EndToEndTests use AU to install very old images in
order to test AU success on old images. This means that newer
update_engines should be able to apply a very old update payload. This
has caused us issues in many places as it has hindered us to deprecated
old features and improve update_engine to a more efficient
manner. Now that we have quick provision payloads in release buckets, we
can use them to install the source image of end to end test. This CL
does that.
However, the only caveat here is that the fallback code to provision
with AU when the QP fails does not work because the release buckets do
not have the test AU payloads in them. So we added a flag (au_fallback)
and disabled that fallback. And we added a temporary fallback to the
original provision mechanism using the AU payloads. We need to remove
this once the EndToEnd tests are stable and showing no sign of failures
in quick-provision.
BUG=chromium:991421
TEST=test_that --autotest_dir ~/trunk/src/third_party/autotest/files/ --args="target_release=13280.0.0 target_payload_uri='gs://chromeos-releases/dev-channel/reef/13290.0.0/payloads/chromeos_13290.0.0_reef_dev-channel_full_test.bin-gvswkobwg43gf4byzyw75rj4io3sl5y5' source_release=13280.0.0 source_payload_uri='gs://chromeos-releases/dev-channel/reef/13290.0.0/payloads/chromeos_13290.0.0_reef_dev-channel_full_test.bin-gvswkobwg43gf4byzyw75rj4io3sl5y5' update_type=full" chromeos6-row4-rack10-host12.cros.corp.google.com autoupdate_EndToEndTest
TEST=above test after raising an exception in
_quick_provision_with_gs_cache() to make sure
_quick_provision_with_devserver() works too.
Change-Id: I1ac1828f58c9a63f4a7d8888e651248033d567ef
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2261239
Tested-by: Amin Hassani <ahassani@chromium.org>
Reviewed-by: Kyle Shimabukuro <kyleshima@chromium.org>
Reviewed-by: David Haddock <dhaddock@chromium.org>
Commit-Queue: Amin Hassani <ahassani@chromium.org>
diff --git a/server/afe_utils.py b/server/afe_utils.py
index 147055e..8003723 100644
--- a/server/afe_utils.py
+++ b/server/afe_utils.py
@@ -146,7 +146,9 @@
def machine_install_and_update_labels(host, update_url,
use_quick_provision=False,
- with_cheets=False, staging_server=None):
+ with_cheets=False, staging_server=None,
+ is_release_bucket=False,
+ au_fallback=True):
"""Install a build and update the version labels on a host.
@param host: Host object where the build is to be installed.
@@ -157,12 +159,16 @@
version of Android for a target running ARC.
@param staging_server: Server where images have been staged. Typically,
an instance of dev_server.ImageServer.
+ @param is_release_bucket: If True, use release bucket
+ gs://chromeos-releases.
+ @param au_fallback: If True, we fallback to AU provisioning if the
+ quick-provisioning fails.
"""
clean_provision_labels(host)
if use_quick_provision:
image_name, host_attributes = _provision_with_quick_provision(
- host, update_url)
+ host, update_url, is_release_bucket, au_fallback)
else:
image_name, host_attributes = _provision_with_au(host, update_url,
staging_server)
@@ -210,11 +216,16 @@
image_name, host_attributes = updater.run_update()
return image_name, host_attributes
-def _provision_with_quick_provision(host, update_url):
+def _provision_with_quick_provision(host, update_url, is_release_bucket,
+ au_fallback):
"""Installs a build on the host using autoupdater quick-provision.
@param host: Host object where the build is to be installed.
@param update_url: URL of the build to install.
+ @param is_release_bucket: If True, use release bucket
+ gs://chromeos-releases.
+ @param au_fallback: If True, we fallback to AU provisioning if the
+ quick-provisioning fails.
@returns A tuple of the form `(image_name, host_attributes)`, where
'image_name' is the name of the image installed, and 'host_attributes'
@@ -222,5 +233,7 @@
"""
logging.debug('Attempting to provision with autoupdater quick-provision.')
updater = autoupdater.ChromiumOSUpdater(update_url, host=host,
- use_quick_provision=True)
+ use_quick_provision=True,
+ is_release_bucket=is_release_bucket,
+ au_fallback=au_fallback)
return updater.run_update()
diff --git a/server/cros/autoupdater.py b/server/cros/autoupdater.py
index 5bcc88f..b2f71f6 100644
--- a/server/cros/autoupdater.py
+++ b/server/cros/autoupdater.py
@@ -226,7 +226,7 @@
@returns a string representing the image name in the update_url.
"""
- return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:])
+ return urlparse.urlparse(update_url).path[len('/update/'):]
def get_update_failure_reason(exception):
@@ -330,7 +330,8 @@
"""Chromium OS specific DUT update functionality."""
def __init__(self, update_url, host=None, interactive=True,
- use_quick_provision=False):
+ use_quick_provision=False, is_release_bucket=None,
+ au_fallback=True):
"""Initializes the object.
@param update_url: The URL we want the update to use.
@@ -338,13 +339,18 @@
@param interactive: Bool whether we are doing an interactive update.
@param use_quick_provision: Whether we should attempt to perform
the update using the quick-provision script.
+ @param is_release_bucket: If True, use release bucket
+ gs://chromeos-releases.
+ @param au_fallback: If True, we fallback to AU provisioning if the
+ quick-provisioning fails.
"""
self.update_url = update_url
self.host = host
self.interactive = interactive
self.update_version = _url_to_version(update_url)
self._use_quick_provision = use_quick_provision
-
+ self._is_release_bucket = is_release_bucket
+ self._au_fallback = au_fallback
def _run(self, cmd, *args, **kwargs):
"""Abbreviated form of self.host.run(...)"""
@@ -470,7 +476,11 @@
def _set_target_version(self):
"""Set the "target version" for the update."""
- version_number = self.update_version.split('-')[1]
+ # Version strings that come from release buckets do not have RXX- at the
+ # beginning. So remove this prefix only if the version has it.
+ version_number = (self.update_version.split('-')[1]
+ if '-' in self.update_version
+ else self.update_version)
self._run('echo %s > %s' % (version_number, _TARGET_VERSION))
@@ -678,8 +688,10 @@
# If enabled, GsCache server listion on different port on the
# devserver.
gs_cache_server = devserver_name.replace(DEVSERVER_PORT, GS_CACHE_PORT)
- gs_cache_url = ('http://%s/download/chromeos-image-archive'
- % gs_cache_server)
+ gs_cache_url = ('http://%s/download/%s'
+ % (gs_cache_server,
+ 'chromeos-releases' if self._is_release_bucket
+ else 'chromeos-image-archive'))
# Check if GS_Cache server is enabled on the server.
self._run('curl -s -o /dev/null %s' % gs_cache_url)
@@ -701,8 +713,11 @@
"""
logging.info('Try quick provision with devserver.')
ds = dev_server.ImageServer('http://%s' % devserver_name)
+ archive_url = ('gs://chromeos-releases/%s' % image_name
+ if self._is_release_bucket else None)
try:
- ds.stage_artifacts(image_name, ['quick_provision', 'stateful'])
+ ds.stage_artifacts(image_name, ['quick_provision', 'stateful'],
+ archive_url=archive_url)
except dev_server.DevServerException as e:
raise error.TestFail, str(e), sys.exc_info()[2]
@@ -725,7 +740,8 @@
if not self._use_quick_provision:
return None
image_name = url_to_image_name(self.update_url)
- logging.info('Installing image using quick-provision.')
+ logging.info('Installing image using quick-provision with url: %s '
+ 'and image_name %s', self.update_url, image_name)
provision_command = self._get_remote_script(_QUICK_PROVISION_SCRIPT)
server_name = urlparse.urlparse(self.update_url)[1]
try:
@@ -761,9 +777,15 @@
"""
logging.info('Installing image at %s onto %s',
self.update_url, self.host.hostname)
+ result = self._install_via_quick_provision()
+ if result:
+ return result
+
+ if not self._au_fallback:
+ raise Exception('quick-provision failed and AU fallback was not '
+ 'requested. So assuming provision failure.')
try:
- return (self._install_via_quick_provision()
- or self._install_via_update_engine())
+ return self._install_via_update_engine()
except:
# N.B. This handling code includes non-Exception classes such
# as KeyboardInterrupt. We need to clean up, but we also must
diff --git a/server/site_tests/autoupdate_EndToEndTest/autoupdate_EndToEndTest.py b/server/site_tests/autoupdate_EndToEndTest/autoupdate_EndToEndTest.py
index 59768dc..8ec7fa0 100755
--- a/server/site_tests/autoupdate_EndToEndTest/autoupdate_EndToEndTest.py
+++ b/server/site_tests/autoupdate_EndToEndTest/autoupdate_EndToEndTest.py
@@ -8,6 +8,7 @@
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib.cros import kernel_utils
from autotest_lib.client.cros import constants
+from autotest_lib.server import afe_utils
from autotest_lib.server.cros.update_engine import update_engine_test
@@ -114,10 +115,23 @@
self._stage_payloads(test_conf['target_payload_uri'],
test_conf['target_archive_uri'])
- # Install source image
+ # Install source image with quick-provision.
source_payload_uri = test_conf['source_payload_uri']
if source_payload_uri is not None:
- self.update_device(source_payload_uri, clobber_stateful=True)
+ try:
+ build_name, _ = self._get_update_parameters_from_uri(
+ source_payload_uri)
+ url = self._autotest_devserver.get_update_url(build_name)
+ logging.info('Installing source image with update url: %s', url)
+ afe_utils.machine_install_and_update_labels(
+ self._host, url, use_quick_provision=True,
+ is_release_bucket=True, au_fallback=False)
+ except Exception:
+ logging.warning('quick-provision failed, trying with AU.')
+ # TODO(crbug.com/991421): Remove this fallback once the quick
+ # provision use case is stabilized.
+ self.update_device(source_payload_uri, clobber_stateful=True)
+
self._run_client_test_and_check_result(self._LOGIN_TEST,
tag='source')
# Start the update to the target image.
