Add a bunch of integration tests for xbuddy and autotest workflows.
This CL starts the process of adding integration testing to both the
xbuddy and lab workflows. Anyone making substantial changes to the
devserver will be required to run all of these tests (including adding
more if adding new functionality).
This CL adds testing of the following RPCs:
stage
is_staged
update
xbuddy
This CL also removes the hardcoding of 8080 and tries all ports from
8080-8090.
BUG=chromium:216959
TEST=Ran them.
Change-Id: I7a547698963e2c114c480977058059ca5d9de3ba
Reviewed-on: https://chromium-review.googlesource.com/173162
Reviewed-by: Chris Sosa <sosa@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
diff --git a/autoupdate_unittest.py b/autoupdate_unittest.py
index 9a45b69..988b038 100755
--- a/autoupdate_unittest.py
+++ b/autoupdate_unittest.py
@@ -64,7 +64,6 @@
cherrypy.request.base = 'http://%s' % self.hostname
common_util.MkDirP(self.static_image_dir)
self._xbuddy = xbuddy.XBuddy(False,
- root_dir=None,
static_dir=self.static_image_dir)
self.mox.StubOutWithMock(xbuddy.XBuddy, '_GetArtifact')
@@ -75,7 +74,6 @@
def _DummyAutoupdateConstructor(self, **kwargs):
"""Creates a dummy autoupdater. Used to avoid using constructor."""
dummy = autoupdate.Autoupdate(self._xbuddy,
- root_dir=None,
static_dir=self.static_image_dir,
**kwargs)
return dummy
diff --git a/build_util.py b/build_util.py
index 82dc2df..29c408b 100644
--- a/build_util.py
+++ b/build_util.py
@@ -13,20 +13,19 @@
Classes that inherit from BuildObject can access scripts in the src/scripts
directory, and have a handle to the static directory of the devserver.
"""
- def __init__(self, root_dir, static_dir):
+ def __init__(self, static_dir):
self.devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
self.static_dir = static_dir
- try:
- chroot_dir = os.environ['CROS_WORKON_SRCROOT']
- self.scripts_dir = os.path.join(chroot_dir, 'src/scripts')
- self.images_dir = os.path.join(chroot_dir, 'src/build/images')
- except KeyError:
- # Outside of chroot: This is a corner case. Since we live either in
- # platform/dev or /usr/bin/, scripts have to live in ../../../src/scripts
- self.scripts_dir = os.path.abspath(os.path.join(
- self.devserver_dir, '../../../src/scripts'))
- self.images_dir = os.path.abspath(os.path.join(
- self.devserver_dir, '../../../src/build/images'))
+ self.scripts_dir = os.path.join(self.GetSourceRoot(), 'src/scripts')
+
+ @staticmethod
+ def GetSourceRoot():
+ """Returns the path to the source root."""
+ src_root = os.environ.get('CROS_WORKON_SRCROOT')
+ if not src_root:
+ src_root = os.path.join(os.path.dirname(__file__), '..', '..', '..')
+
+ return os.path.realpath(src_root)
def GetLatestImageDir(self, board):
"""Returns the latest image dir based on shell script."""
diff --git a/devserver.py b/devserver.py
index be8352d..1971ee6 100755
--- a/devserver.py
+++ b/devserver.py
@@ -61,6 +61,7 @@
import build_artifact
import common_util
import downloader
+import gsutil_util
import log_util
import xbuddy
@@ -187,6 +188,8 @@
}
if options.production:
base_config['global'].update({'server.thread_pool': 150})
+ # TODO(sosa): Do this more cleanly.
+ gsutil_util.GSUTIL_ATTEMPTS = 5
return base_config
@@ -1050,8 +1053,6 @@
cherrypy.log.error_log.addHandler(hdlr)
# pylint: enable=E1101
- root_dir = os.path.realpath('%s/../..' % devserver_dir)
-
# set static_dir, from which everything will be served
options.static_dir = os.path.realpath(options.static_dir)
@@ -1065,12 +1066,10 @@
os.makedirs(cache_dir)
_Log('Using cache directory %s' % cache_dir)
- _Log('Source root is %s' % root_dir)
_Log('Serving from %s' % options.static_dir)
_xbuddy = xbuddy.XBuddy(options.xbuddy_manage_builds,
options.board,
- root_dir=root_dir,
static_dir=options.static_dir)
if options.clear_cache and options.xbuddy_manage_builds:
_xbuddy.CleanCache()
@@ -1080,7 +1079,6 @@
global updater
updater = autoupdate.Autoupdate(
_xbuddy,
- root_dir=root_dir,
static_dir=options.static_dir,
urlbase=options.urlbase,
forced_image=options.image,
diff --git a/devserver_integration_test.py b/devserver_integration_test.py
new file mode 100755
index 0000000..d78e3b8
--- /dev/null
+++ b/devserver_integration_test.py
@@ -0,0 +1,465 @@
+#!/usr/bin/python
+
+# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Integration tests for the devserver.
+
+This module is responsible for testing the actual devserver APIs and should be
+run whenever changes are made to the devserver.
+
+Note there are two classes of tests here and they can be run separately.
+
+To just run the short-running "unittests" run:
+ ./devserver_integration_tests.py DevserverUnittests
+
+To just run the longer-running tests, run:
+ ./devserver_integration_tests.py DevserverIntegrationTests
+"""
+
+import devserver_constants
+import json
+import logging
+from xml.dom import minidom
+import os
+import psutil
+import shutil
+import signal
+import subprocess
+import tempfile
+import time
+import unittest
+import urllib2
+
+
+# Paths are relative to this script's base directory.
+LABEL = 'devserver'
+TEST_IMAGE_PATH = 'testdata/devserver'
+TEST_IMAGE_NAME = 'update.gz'
+EXPECTED_HASH = 'kGcOinJ0vA8vdYX53FN0F5BdwfY='
+
+# Update request based on Omaha v3 protocol format.
+UPDATE_REQUEST = """<?xml version="1.0" encoding="UTF-8"?>
+<request version="ChromeOSUpdateEngine-0.1.0.0" updaterversion="ChromeOSUpdateEngine-0.1.0.0" protocol="3.0" ismachine="1">
+ <os version="Indy" platform="Chrome OS" sp="0.11.254.2011_03_09_1814_i686"></os>
+ <app appid="{DEV-BUILD}" version="11.254.2011_03_09_1814" lang="en-US" track="developer-build" board="x86-generic" hardware_class="BETA DVT" delta_okay="true">
+ <updatecheck></updatecheck>
+ </app>
+</request>
+"""
+
+# RPC constants.
+STAGE = 'stage'
+IS_STAGED = 'is_staged'
+STATIC = 'static'
+UPDATE = 'update'
+CHECK_HEALTH = 'check_health'
+CONTROL_FILES = 'controlfiles'
+XBUDDY = 'xbuddy'
+
+# API rpcs and constants.
+API_HOST_INFO = 'api/hostinfo'
+API_SET_NEXT_UPDATE = 'api/setnextupdate'
+API_SET_UPDATE_REQUEST = 'new_update-test/the-new-update'
+API_TEST_IP_ADDR = '127.0.0.1'
+
+DEVSERVER_START_TIMEOUT = 15
+
+
+class DevserverFailedToStart(Exception):
+ """Raised if we could not start the devserver."""
+
+
+class DevserverTestCommon(unittest.TestCase):
+ """Class containing common logic between devserver test classes."""
+
+ def setUp(self):
+ """Copies in testing files."""
+ self.test_data_path = tempfile.mkdtemp()
+ self.src_dir = os.path.dirname(__file__)
+
+ # Current location of testdata payload.
+ image_src = os.path.join(self.src_dir, TEST_IMAGE_PATH, TEST_IMAGE_NAME)
+
+ # Copy the payload to the location of the update label "devserver."
+ os.makedirs(os.path.join(self.test_data_path, LABEL))
+ shutil.copy(image_src, os.path.join(self.test_data_path, LABEL,
+ TEST_IMAGE_NAME))
+
+ # Copy the payload to the location of forced label.
+ os.makedirs(os.path.join(self.test_data_path, API_SET_UPDATE_REQUEST))
+ shutil.copy(image_src, os.path.join(self.test_data_path,
+ API_SET_UPDATE_REQUEST,
+ TEST_IMAGE_NAME))
+
+ self.pidfile = tempfile.mktemp('devserver_unittest')
+ self.logfile = tempfile.mktemp('devserver_unittest')
+
+ # Devserver url set in start server.
+ self.devserver_url = None
+ self.pid = self._StartServer()
+
+ def tearDown(self):
+ """Removes testing files."""
+ shutil.rmtree(self.test_data_path)
+ if self.pid:
+ os.kill(self.pid, signal.SIGKILL)
+
+ if os.path.exists(self.pidfile):
+ os.remove(self.pidfile)
+
+ # Helper methods begin here.
+
+ def _StartServerOnPort(self, port):
+ """Attempts to start devserver on |port|.
+
+ Returns:
+ The pid of the devserver.
+
+ Raises:
+ DevserverFailedToStart: If the devserver could not be started.
+ """
+ cmd = [
+ 'python',
+ os.path.join(self.src_dir, 'devserver.py'),
+ 'devserver.py',
+ '--static_dir', self.test_data_path,
+ '--pidfile', self.pidfile,
+ '--port', str(port),
+ '--logfile', self.logfile]
+
+ # Pipe all output. Use logfile to get devserver log.
+ subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+
+ # wait for devserver to start
+ current_time = time.time()
+ deadline = current_time + DEVSERVER_START_TIMEOUT
+ while current_time < deadline:
+ current_time = time.time()
+ try:
+ self.devserver_url = 'http://127.0.0.1:%d' % port
+ self._MakeRPC(CHECK_HEALTH, timeout=0.1)
+ break
+ except Exception:
+ continue
+ else:
+ raise DevserverFailedToStart('Devserver failed to start within timeout.')
+
+ if not os.path.exists(self.pidfile):
+ raise DevserverFailedToStart('Devserver did not drop a pidfile.')
+ else:
+ pid_value = open(self.pidfile).read()
+ try:
+ return int(pid_value)
+ except ValueError:
+ raise DevserverFailedToStart('Devserver did not drop a pid in the '
+ 'pidfile.')
+
+ def _StartServer(self):
+ """Starts devserver on a port, returns pid.
+
+ Raises:
+ DevserverFailedToStart: If all attempts to start the devserver fail.
+ """
+ # TODO(sosa): Fixing crbug.com/308686 will allow the devserver to do this
+ # for us and be cleaner.
+ for port in range(8080, 8090):
+ try:
+ return self._StartServerOnPort(port)
+ except DevserverFailedToStart:
+ logging.error('Devserver could not start on port %s', port)
+ continue
+ else:
+ logging.error(open(self.logfile).read())
+ raise DevserverFailedToStart('Devserver could not be started on all '
+ 'ports.')
+
+ def VerifyHandleUpdate(self, label, use_test_payload=True):
+ """Verifies that we can send an update request to the devserver.
+
+ This method verifies (using a fake update_request blob) that the devserver
+ can interpret the payload and give us back the right payload.
+
+ Args:
+ label: Label that update is served from e.g. <board>-release/<version>
+ use_test_payload: If set to true, expects to serve payload under
+ testdata/ and does extra checks i.e. compares hash and content of
+ payload.
+ Returns:
+ url of the update payload if we verified the update.
+ """
+ update_label = '/'.join([UPDATE, label])
+ response = self._MakeRPC(update_label, data=UPDATE_REQUEST)
+ self.assertNotEqual('', response)
+
+ # Parse the response and check if it contains the right result.
+ dom = minidom.parseString(response)
+ update = dom.getElementsByTagName('updatecheck')[0]
+ expected_static_url = '/'.join([self.devserver_url, STATIC, label])
+ expected_hash = EXPECTED_HASH if use_test_payload else None
+ url = self.VerifyV3Response(update, expected_static_url,
+ expected_hash=expected_hash)
+
+ # Verify the image we download is correct since we already know what it is.
+ if use_test_payload:
+ connection = urllib2.urlopen(url)
+ contents = connection.read()
+ connection.close()
+ self.assertEqual('Developers, developers, developers!\n', contents)
+
+ return url
+
+ def VerifyV3Response(self, update, expected_static_url, expected_hash):
+ """Verifies the update DOM from a v3 response and returns the url."""
+ # Parse the response and check if it contains the right result.
+ urls = update.getElementsByTagName('urls')[0]
+ url = urls.getElementsByTagName('url')[0]
+
+ static_url = url.getAttribute('codebase')
+ # Static url's end in /.
+ self.assertEqual(expected_static_url + '/', static_url)
+
+ manifest = update.getElementsByTagName('manifest')[0]
+ packages = manifest.getElementsByTagName('packages')[0]
+ package = packages.getElementsByTagName('package')[0]
+ filename = package.getAttribute('name')
+ self.assertEqual(TEST_IMAGE_NAME, filename)
+
+ if expected_hash:
+ hash_value = package.getAttribute('hash')
+ self.assertEqual(EXPECTED_HASH, hash_value)
+
+ url = os.path.join(static_url, filename)
+ return url
+
+ def _MakeRPC(self, rpc, data=None, timeout=None, **kwargs):
+ """Makes a RPC to the devserver using the kwargs and returns output.
+
+ Args:
+ data: Optional post data to send.
+ timeout: Optional timeout to pass to urlopen.
+
+ For example: localhost:8080/stage with artifact_url=blah/blah.
+ """
+ request = '/'.join([self.devserver_url, rpc])
+ if kwargs:
+ # Join the kwargs to the URL.
+ request += '?' + '&'.join('%s=%s' % item for item in kwargs.iteritems())
+
+ output = None
+ try:
+ # Let's log output for all rpc's without timeouts because we only
+ # use timeouts to check to see if something is up and these checks tend
+ # to be small and so logging it will be extremely repetitive.
+ if not timeout:
+ logging.info('Making request using %s', request)
+
+ connection = urllib2.urlopen(request, data=data, timeout=timeout)
+ output = connection.read()
+ connection.close()
+ except urllib2.HTTPError:
+ raise
+
+ return output
+
+
+class DevserverUnittests(DevserverTestCommon):
+ """Short running integration/unittests for the devserver (no remote deps).
+
+ These are technically not unittests because they depend on being able to
+ start a devserver locally which technically requires external resources so
+ they are lumped with the remote tests here.
+ """
+
+ def testHandleUpdateV3(self):
+ self.VerifyHandleUpdate(label=LABEL)
+
+ def testApiBadSetNextUpdateRequest(self):
+ """Tests sending a bad setnextupdate request."""
+ # Send bad request and ensure it fails...
+ self.assertRaises(urllib2.URLError,
+ self._MakeRPC,
+ '/'.join([API_SET_NEXT_UPDATE, API_TEST_IP_ADDR]))
+
+ def testApiBadSetNextUpdateURL(self):
+ """Tests contacting a bad setnextupdate url."""
+ # Send bad request and ensure it fails...
+ self.assertRaises(urllib2.URLError,
+ self._MakeRPC, API_SET_NEXT_UPDATE)
+
+ def testApiBadHostInfoURL(self):
+ """Tests contacting a bad hostinfo url."""
+ # Host info should be invalid without a specified address.
+ self.assertRaises(urllib2.URLError,
+ self._MakeRPC, API_HOST_INFO)
+
+ def testApiHostInfoAndSetNextUpdate(self):
+ """Tests using the setnextupdate and hostinfo api commands."""
+ # Send setnextupdate command.
+ self._MakeRPC('/'.join([API_SET_NEXT_UPDATE, API_TEST_IP_ADDR]),
+ data=API_SET_UPDATE_REQUEST)
+
+ # Send hostinfo command and verify the setnextupdate worked.
+ response = self._MakeRPC('/'.join([API_HOST_INFO, API_TEST_IP_ADDR]))
+
+ self.assertEqual(
+ json.loads(response)['forced_update_label'], API_SET_UPDATE_REQUEST)
+
+ def testXBuddyLocalAlias(self):
+ """Extensive local image xbuddy unittest.
+
+ This test verifies all the local xbuddy logic by creating a new local folder
+ with the necessary update items and verifies we can use all of them.
+ """
+ update_data = 'TEST UPDATE'
+ image_data = 'TEST IMAGE'
+ stateful_data = 'STATEFUL STUFFS'
+ build_id = 'x86-generic/R32-9999.0.0-a1'
+ xbuddy_path = 'x86-generic/R32-9999.0.0-a1/test'
+ build_dir = os.path.join(self.test_data_path, build_id)
+ os.makedirs(build_dir)
+ test_image_file = os.path.join(build_dir,
+ devserver_constants.TEST_IMAGE_FILE)
+ update_file = os.path.join(build_dir, devserver_constants.UPDATE_FILE)
+ stateful_file = os.path.join(build_dir, devserver_constants.STATEFUL_FILE)
+
+ logging.info('Creating dummy files')
+
+ for item, filename, data in zip(
+ ['full_payload', 'test', 'stateful'],
+ [update_file, test_image_file, stateful_file],
+ [update_data, image_data, stateful_data]):
+ logging.info('Creating file %s', filename)
+ with open(filename, 'w') as fh:
+ fh.write(data)
+
+ xbuddy_path = '/'.join([build_id, item])
+ logging.info('Testing xbuddy path %s', xbuddy_path)
+ response = self._MakeRPC('/'.join([XBUDDY, xbuddy_path]))
+ self.assertEqual(response, data)
+
+ expected_dir = '/'.join([self.devserver_url, STATIC, build_id])
+ response = self._MakeRPC('/'.join([XBUDDY, xbuddy_path]), return_dir=True)
+ self.assertEqual(response, expected_dir)
+
+ xbuddy_path = '/'.join([build_id, 'test'])
+ logging.info('Testing for_update for %s', xbuddy_path)
+ response = self._MakeRPC('/'.join([XBUDDY, xbuddy_path]), for_update=True)
+ expected_path = '/'.join([self.devserver_url, UPDATE, build_id])
+ self.assertTrue(response, expected_path)
+
+ logging.info('Verifying the actual payload data')
+ url = self.VerifyHandleUpdate(build_id, use_test_payload=False)
+ logging.info('Verify the actual content of the update payload')
+ connection = urllib2.urlopen(url)
+ contents = connection.read()
+ connection.close()
+ self.assertEqual(update_data, contents)
+
+ def testPidFile(self):
+ """Test that using a pidfile works correctly."""
+ with open(self.pidfile, 'r') as f:
+ pid = f.read()
+
+ # Let's assert some process information about the devserver.
+ self.assertTrue(pid.isdigit())
+ process = psutil.Process(int(pid))
+ self.assertTrue(process.is_running())
+ self.assertTrue('devserver.py' in process.cmdline)
+
+
+class DevserverIntegrationTests(DevserverTestCommon):
+ """Longer running integration tests that test interaction with Google Storage.
+
+ Note: due to the interaction with Google Storage, these tests both require
+ 1) runner has access to the Google Storage bucket where builders store builds.
+ 2) time. These tests actually download the artifacts needed.
+ """
+
+ def testStageAndUpdate(self):
+ """Tests core autotest workflow where we stage/update with a test payload.
+ """
+ build_id = 'x86-mario-release/R32-4810.0.0'
+ archive_url = 'gs://chromeos-image-archive/%s' % build_id
+
+ response = self._MakeRPC(IS_STAGED, archive_url=archive_url,
+ artifacts='full_payload,stateful')
+ self.assertEqual(response, 'False')
+
+ logging.info('Staging update artifacts')
+ self._MakeRPC(STAGE, archive_url=archive_url,
+ artifacts='full_payload,stateful')
+ logging.info('Staging complete. '
+ 'Verifying files exist and are staged in the staging '
+ 'directory.')
+ response = self._MakeRPC(IS_STAGED, archive_url=archive_url,
+ artifacts='full_payload,stateful')
+ self.assertEqual(response, 'True')
+ staged_dir = os.path.join(self.test_data_path, build_id)
+ self.assertTrue(os.path.isdir(staged_dir))
+ self.assertTrue(os.path.exists(
+ os.path.join(staged_dir, devserver_constants.UPDATE_FILE)))
+ self.assertTrue(os.path.exists(
+ os.path.join(staged_dir, devserver_constants.STATEFUL_FILE)))
+
+ logging.info('Verifying we can update using the stage update artifacts.')
+ self.VerifyHandleUpdate(build_id, use_test_payload=False)
+
+ def testStageAutotestAndGetPackages(self):
+ """Another autotest workflow test where we stage/update with a test payload.
+ """
+ build_id = 'x86-mario-release/R32-4810.0.0'
+ archive_url = 'gs://chromeos-image-archive/%s' % build_id
+ autotest_artifacts = 'autotest,test_suites,au_suite'
+ logging.info('Staging autotest artifacts (may take a while).')
+ self._MakeRPC(STAGE, archive_url=archive_url, artifacts=autotest_artifacts)
+
+ response = self._MakeRPC(IS_STAGED, archive_url=archive_url,
+ artifacts=autotest_artifacts)
+ self.assertEqual(response, 'True')
+
+ # Verify the files exist and are staged in the staging directory.
+ logging.info('Checking directories exist after we staged the files.')
+ staged_dir = os.path.join(self.test_data_path, build_id)
+ autotest_dir = os.path.join(staged_dir, 'autotest')
+ package_dir = os.path.join(autotest_dir, 'packages')
+ self.assertTrue(os.path.isdir(staged_dir))
+ self.assertTrue(os.path.isdir(autotest_dir))
+ self.assertTrue(os.path.isdir(package_dir))
+
+ control_files = self._MakeRPC(CONTROL_FILES, build=build_id,
+ suite_name='bvt')
+ logging.info('Checking for known control file in bvt suite.')
+ self.assertTrue('client/site_tests/platform_FilePerms/'
+ 'control' in control_files)
+
+ def testRemoteXBuddyAlias(self):
+ """Another autotest workflow test where we stage/update with a test payload.
+ """
+ build_id = 'x86-mario-release/R32-4810.0.0'
+ xbuddy_path = 'remote/x86-mario/R32-4810.0.0/full_payload'
+ xbuddy_bad_path = 'remote/x86-mario/R32-9999.9999.9999'
+ logging.info('Staging artifacts using xbuddy.')
+ response = self._MakeRPC('/'.join([XBUDDY, xbuddy_path]), return_dir=True)
+
+ logging.info('Verifying static url returned is valid.')
+ expected_static_url = '/'.join([self.devserver_url, STATIC, build_id])
+ self.assertEqual(response, expected_static_url)
+
+ logging.info('Checking for_update returns an update_url for what we just '
+ 'staged.')
+ expected_update_url = '/'.join([self.devserver_url, UPDATE, build_id])
+ response = self._MakeRPC('/'.join([XBUDDY, xbuddy_path]), for_update=True)
+ self.assertEqual(response, expected_update_url)
+
+ logging.info('Now give xbuddy a bad path.')
+ self.assertRaises(urllib2.HTTPError,
+ self._MakeRPC,
+ '/'.join([XBUDDY, xbuddy_bad_path]))
+
+
+if __name__ == '__main__':
+ logging_format = '%(levelname)-8s: %(message)s'
+ logging.basicConfig(level=logging.DEBUG, format=logging_format)
+ unittest.main()
diff --git a/devserver_unittest.py b/devserver_unittest.py
deleted file mode 100755
index 44db7ed..0000000
--- a/devserver_unittest.py
+++ /dev/null
@@ -1,231 +0,0 @@
-#!/usr/bin/python
-
-# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Regression tests for devserver."""
-
-import json
-from xml.dom import minidom
-import os
-import psutil
-import shutil
-import signal
-import subprocess
-import tempfile
-import time
-import unittest
-import urllib2
-
-
-# Paths are relative to this script's base directory.
-LABEL = 'devserver'
-TEST_IMAGE_PATH = 'testdata/devserver'
-TEST_IMAGE_NAME = 'update.gz'
-TEST_IMAGE = TEST_IMAGE_PATH + '/' + TEST_IMAGE_NAME
-EXPECTED_HASH = 'kGcOinJ0vA8vdYX53FN0F5BdwfY='
-
-# Update request based on Omaha v2 protocol format.
-UPDATE_REQUEST = """<?xml version="1.0" encoding="UTF-8"?>
-<request version="ChromeOSUpdateEngine-0.1.0.0" updaterversion="ChromeOSUpdateEngine-0.1.0.0" protocol="3.0" ismachine="1">
- <os version="Indy" platform="Chrome OS" sp="0.11.254.2011_03_09_1814_i686"></os>
- <app appid="{DEV-BUILD}" version="11.254.2011_03_09_1814" lang="en-US" track="developer-build" board="x86-generic" hardware_class="BETA DVT" delta_okay="true">
- <updatecheck></updatecheck>
- <event eventtype="3" eventresult="2" previousversion="0.11.216.2011_03_02_1358"></event>
- </app>
-</request>
-"""
-
-# TODO(girts): use a random available port.
-UPDATE_URL = 'http://127.0.0.1:8080/update/devserver'
-CHECK_HEALTH_URL = 'http://127.0.0.1:8080/check_health'
-STATIC_URL = 'http://127.0.0.1:8080/static/'
-SERVE_URL = STATIC_URL + LABEL + '/'
-
-API_HOST_INFO_BAD_URL = 'http://127.0.0.1:8080/api/hostinfo/'
-API_HOST_INFO_URL = API_HOST_INFO_BAD_URL + '127.0.0.1'
-
-API_SET_UPDATE_BAD_URL = 'http://127.0.0.1:8080/api/setnextupdate/'
-API_SET_UPDATE_URL = API_SET_UPDATE_BAD_URL + '127.0.0.1'
-
-API_SET_UPDATE_REQUEST = 'new_update-test/the-new-update'
-
-DEVSERVER_START_TIMEOUT = 15
-
-class DevserverTest(unittest.TestCase):
- """Regressions tests for devserver."""
-
- def setUp(self):
- """Copies in testing files."""
-
- # Copy in developer-test.gz, as "static/" directory is hardcoded, and it
- # would be very hard to change it (static file serving is handled deep
- # inside webpy).
- self.test_data_path = tempfile.mkdtemp()
- self.src_dir = os.path.dirname(__file__)
-
- # Copy the payload to the root of static_dir.
- self.image_src = os.path.join(self.src_dir, TEST_IMAGE)
- self.image = os.path.join(self.test_data_path, TEST_IMAGE_NAME)
- shutil.copy(self.image_src, self.image)
-
- # Copy the payload to the location of the update label "devserver."
- os.makedirs(os.path.join(self.test_data_path, LABEL))
- shutil.copy(self.image_src, os.path.join(self.test_data_path, LABEL,
- TEST_IMAGE_NAME))
-
- # Copy the payload to the location of forced label.
- os.makedirs(os.path.join(self.test_data_path, API_SET_UPDATE_REQUEST))
- shutil.copy(self.image_src, os.path.join(self.test_data_path,
- API_SET_UPDATE_REQUEST,
- TEST_IMAGE_NAME))
-
- self.pidfile = tempfile.mktemp('devserver_unittest')
- self.devserver_process = self._StartServer(self.pidfile)
-
- def tearDown(self):
- """Removes testing files."""
- shutil.rmtree(self.test_data_path)
- os.kill(self.devserver_process.pid, signal.SIGKILL)
-
- # Helper methods begin here.
-
- def _StartServer(self, pidfile):
- """Starts devserver, returns process."""
- cmd = [
- 'python',
- os.path.join(self.src_dir, 'devserver.py'),
- 'devserver.py',
- '--static_dir', self.test_data_path,
- '--pidfile', pidfile
- ]
- if pidfile:
- cmd.extend(['--pidfile', pidfile])
-
- process = subprocess.Popen(cmd,
- stderr=subprocess.PIPE)
-
- # wait for devserver to start
- current_time = time.time()
- deadline = current_time + DEVSERVER_START_TIMEOUT
- while current_time < deadline:
- current_time = time.time()
- try:
- urllib2.urlopen(CHECK_HEALTH_URL, timeout=0.05)
- break
- except Exception:
- continue
- else:
- self.fail('Devserver failed to start within timeout.')
-
- return process
-
- def VerifyHandleUpdate(self):
- """Tests running the server and getting an update for the given protocol."""
- request = urllib2.Request(UPDATE_URL, UPDATE_REQUEST)
- connection = urllib2.urlopen(request)
- response = connection.read()
- connection.close()
- self.assertNotEqual('', response)
-
- # Parse the response and check if it contains the right result.
- dom = minidom.parseString(response)
- update = dom.getElementsByTagName('updatecheck')[0]
- url = self.VerifyV3Response(update)
-
- # Try to fetch the image.
- connection = urllib2.urlopen(url)
- contents = connection.read()
- connection.close()
- self.assertEqual('Developers, developers, developers!\n', contents)
-
- def VerifyV3Response(self, update):
- """Verifies the update DOM from a v3 response and returns the url."""
- # Parse the response and check if it contains the right result.
- urls = update.getElementsByTagName('urls')[0]
- url = urls.getElementsByTagName('url')[0]
-
- codebase = url.getAttribute('codebase')
- self.assertEqual(SERVE_URL, codebase)
-
- manifest = update.getElementsByTagName('manifest')[0]
- packages = manifest.getElementsByTagName('packages')[0]
- package = packages.getElementsByTagName('package')[0]
- filename = package.getAttribute('name')
- self.assertEqual(TEST_IMAGE_NAME, filename)
-
- hash_value = package.getAttribute('hash')
- self.assertEqual(EXPECTED_HASH, hash_value)
-
- url = os.path.join(codebase, filename)
- return url
-
- # Tests begin here.
- def testHandleUpdateV3(self):
- self.VerifyHandleUpdate()
-
- def testApiBadSetNextUpdateRequest(self):
- """Tests sending a bad setnextupdate request."""
- # Send bad request and ensure it fails...
- try:
- request = urllib2.Request(API_SET_UPDATE_URL, '')
- connection = urllib2.urlopen(request)
- connection.read()
- connection.close()
- self.fail('Invalid setnextupdate request did not fail!')
- except urllib2.URLError:
- pass
-
- def testApiBadSetNextUpdateURL(self):
- """Tests contacting a bad setnextupdate url."""
- # Send bad request and ensure it fails...
- try:
- connection = urllib2.urlopen(API_SET_UPDATE_BAD_URL)
- connection.read()
- connection.close()
- self.fail('Invalid setnextupdate url did not fail!')
- except urllib2.URLError:
- pass
-
- def testApiBadHostInfoURL(self):
- """Tests contacting a bad hostinfo url."""
- # Send bad request and ensure it fails...
- try:
- connection = urllib2.urlopen(API_HOST_INFO_BAD_URL)
- connection.read()
- connection.close()
- self.fail('Invalid hostinfo url did not fail!')
- except urllib2.URLError:
- pass
-
- def testApiHostInfoAndSetNextUpdate(self):
- """Tests using the setnextupdate and hostinfo api commands."""
- # Send setnextupdate command.
- request = urllib2.Request(API_SET_UPDATE_URL, API_SET_UPDATE_REQUEST)
- connection = urllib2.urlopen(request)
- response = connection.read()
- connection.close()
-
- # Send hostinfo command and verify the setnextupdate worked.
- connection = urllib2.urlopen(API_HOST_INFO_URL)
- response = connection.read()
- connection.close()
-
- self.assertEqual(
- json.loads(response)['forced_update_label'], API_SET_UPDATE_REQUEST)
-
- def testPidFile(self):
- """Test that using a pidfile works correctly."""
- with open(self.pidfile, 'r') as f:
- pid = f.read()
-
- # Let's assert some process information about the devserver.
- self.assertTrue(pid.isdigit())
- process = psutil.Process(int(pid))
- self.assertTrue(process.is_running())
- self.assertTrue('devserver.py' in process.cmdline)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/gsutil_util.py b/gsutil_util.py
index fe66dee..709bc64 100644
--- a/gsutil_util.py
+++ b/gsutil_util.py
@@ -16,7 +16,7 @@
import log_util
-GSUTIL_ATTEMPTS = 5
+GSUTIL_ATTEMPTS = 1
UPLOADED_LIST = 'UPLOADED'
diff --git a/xbuddy.py b/xbuddy.py
index 7b5c857..e723fef 100644
--- a/xbuddy.py
+++ b/xbuddy.py
@@ -59,6 +59,7 @@
DEV,
BASE,
FULL,
+ STATEFUL,
ANY,
]
@@ -67,6 +68,7 @@
devserver_constants.IMAGE_FILE,
devserver_constants.BASE_IMAGE_FILE,
devserver_constants.UPDATE_FILE,
+ devserver_constants.STATEFUL_FILE,
None, # For ANY.
]
@@ -171,7 +173,8 @@
# Lock used to lock increasing/decreasing count.
_staging_thread_count_lock = threading.Lock()
- def __init__(self, manage_builds=False, board=None, **kwargs):
+ def __init__(self, manage_builds=False, board=None, images_dir=None,
+ **kwargs):
super(XBuddy, self).__init__(**kwargs)
self.config = self._ReadConfig()
@@ -179,6 +182,11 @@
self._board = board
self._timestamp_folder = os.path.join(self.static_dir,
Timestamp.XBUDDY_TIMESTAMP_DIR)
+ if images_dir:
+ self.images_dir = images_dir
+ else:
+ self.images_dir = os.path.join(self.GetSourceRoot(), 'src/build/images')
+
common_util.MkDirP(self._timestamp_folder)
@classmethod
diff --git a/xbuddy_unittest.py b/xbuddy_unittest.py
index 9bd2a31..ea86072 100755
--- a/xbuddy_unittest.py
+++ b/xbuddy_unittest.py
@@ -24,11 +24,9 @@
mox.MoxTestBase.setUp(self)
self.static_image_dir = tempfile.mkdtemp('xbuddy_unittest_static')
- self.root_dir = tempfile.mkdtemp('xbuddy_unittest_ds_root')
self.mock_xb = xbuddy.XBuddy(
True,
- root_dir=self.root_dir,
static_dir=self.static_image_dir
)
self.images_dir = tempfile.mkdtemp('xbuddy_unittest_images')
