blob: fcd73fb137267421b40a748911e5e179714967a9 [file] [log] [blame]
# Copyright (c) 2009-2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A CherryPy-based webserver to host images and build packages."""
import cherrypy
import logging
import multiprocessing
import optparse
import os
import re
import sys
import subprocess
import tempfile
import autoupdate
import devserver_util
import downloader
# Sets up global to share between classes.
global updater
updater = None
class DevServerError(Exception):
"""Exception class used by this module."""
def _LeadingWhiteSpaceCount(string):
"""Count the amount of leading whitespace in a string.
string: The string to count leading whitespace in.
number of white space chars before characters start.
matched = re.match('^\s+', string)
if matched:
return len(
return 0
def _PrintDocStringAsHTML(func):
"""Make a functions docstring somewhat HTML style.
func: The function to return the docstring from.
A string that is somewhat formated for a web browser.
# TODO(scottz): Make this parse Args/Returns in a prettier way.
# Arguments could be bolded and indented etc.
html_doc = []
for line in func.__doc__.splitlines():
leading_space = _LeadingWhiteSpaceCount(line)
if leading_space > 0:
line = ' ' * leading_space + line
html_doc.append('<BR>%s' % line)
return '\n'.join(html_doc)
def _GetConfig(options):
"""Returns the configuration for the devserver."""
base_config = { 'global':
{ 'server.log_request_headers': True,
'server.protocol_version': 'HTTP/1.1',
'server.socket_host': '::',
'server.socket_port': int(options.port),
'response.timeout': 6000,
'server.socket_timeout': 60,
# Gets rid of cherrypy parsing post file for args.
'request.process_request_body': False,
'response.timeout': 100000,
# Gets rid of cherrypy parsing post file for args.
'request.process_request_body': False,
'response.timeout': 10000,
# Sets up the static dir for file hosting.
{ 'tools.staticdir.dir': 'static',
'tools.staticdir.on': True,
'response.timeout': 10000,
if options.log_dir:
base_config['global']['log.access_file'] = os.path.join(
options.log_dir, 'devserver_access.log')
base_config['global']['log.error_file'] = os.path.join(
options.log_dir, 'devserver_error.log')
if options.production:
base_config['global']['server.environment'] = 'production'
return base_config
def _PrepareToServeUpdatesOnly(image_dir, static_dir):
"""Sets up symlink to image_dir for serving purposes."""
assert os.path.exists(image_dir), '%s must exist.' % image_dir
# If we're serving out of an archived build dir (e.g. a
# buildbot), prepare this webserver's magic 'static/' dir with a
# link to the build archive.
cherrypy.log('Preparing autoupdate for "serve updates only" mode.',
if os.path.lexists('%s/archive' % static_dir):
if image_dir != os.readlink('%s/archive' % static_dir):
cherrypy.log('removing stale symlink to %s' % image_dir, 'DEVSERVER')
os.unlink('%s/archive' % static_dir)
os.symlink(image_dir, '%s/archive' % static_dir)
os.symlink(image_dir, '%s/archive' % static_dir)
cherrypy.log('archive dir: %s ready to be used to serve images.' % image_dir,
class ApiRoot(object):
"""RESTful API for Dev Server information."""
exposed = True
def hostinfo(self, ip):
"""Returns a JSON dictionary containing information about the given ip.
Not all information may be known at the time the request is made. The
possible keys are:
last_event_type: int
Last update event type received.
last_event_status: int
Last update event status received.
last_known_version: string
Last known version recieved for update ping.
forced_update_label: string
Update label to force next update ping to use. Set by setnextupdate.
See the OmahaEvent class in update_engine/omaha_request_action.h for status
code definitions. If the ip does not exist an empty string is returned."""
return updater.HandleHostInfoPing(ip)
def hostlog(self, ip):
"""Returns a JSON object containing a log of events pertaining to a
particular host, or all hosts. Log events contain a timestamp and any
subset of the attributes listed for the hostinfo method."""
return updater.HandleHostLogPing(ip)
def setnextupdate(self, ip):
"""Allows the response to the next update ping from a host to be set.
Takes the IP of the host and an update label as normally provided to the
/update command."""
body_length = int(cherrypy.request.headers['Content-Length'])
label =
if label:
label = label.strip()
if label:
return updater.HandleSetUpdatePing(ip, label)
raise cherrypy.HTTPError(400, 'No label provided.')
class DevServerRoot(object):
"""The Root Class for the Dev Server.
CherryPy works as follows:
For each method in this class, cherrpy interprets root/path
as a call to an instance of DevServerRoot->method_name. For example,
a call to http://myhost/build will call build. CherryPy automatically
parses http args and places them as keyword arguments in each method.
For paths http://myhost/update/dir1/dir2, you can use *args so that
cherrypy uses the update method and puts the extra paths in args.
api = ApiRoot()
def __init__(self):
self._builder = None
self._lock_dict_lock = multiprocessing.Lock()
self._lock_dict = {}
self._downloader_dict = {}
def build(self, board, pkg, **kwargs):
"""Builds the package specified."""
import builder
if self._builder is None:
self._builder = builder.Builder()
return self._builder.Build(board, pkg, kwargs)
def _get_lock_for_archive_url(self, archive_url):
"""Return a multiprocessing lock to use per archive_url.
Use this lock to protect critical zones per archive_url.
with DevserverInstance._get_lock_for_archive_url(archive_url):
A multiprocessing lock that is archive_url specific.
with self._lock_dict_lock:
lock = self._lock_dict.get(archive_url)
if lock:
return lock
lock = multiprocessing.Lock()
self._lock_dict[archive_url] = lock
return lock
def _canonicalize_archive_url(archive_url):
"""Canonicalizes archive_url strings.
DevserverError: if archive_url is not set.
if archive_url:
return archive_url.rstrip('/')
raise DevServerError("Must specify an archive_url in the request")
def download(self, **kwargs):
"""Downloads and archives full/delta payloads from Google Storage.
This methods downloads artifacts. It may download artifacts in the
background in which case a caller should call wait_for_status to get
the status of the background artifact downloads. They should use the same
args passed to download.
archive_url: Google Storage URL for the build.
Example URL:
archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
# Guarantees that no two downloads for the same url can run this code
# at the same time.
with self._get_lock_for_archive_url(archive_url):
# If we are currently downloading, return. Note, due to the above lock
# we know that the foreground artifacts must have finished downloading
# and returned Success if this downloader instance exists.
if (self._downloader_dict.get(archive_url) or
downloader.Downloader.BuildStaged(archive_url, updater.static_dir)):
cherrypy.log('Build %s has already been processed.' % archive_url,
return 'Success'
downloader_instance = downloader.Downloader(updater.static_dir)
self._downloader_dict[archive_url] = downloader_instance
return downloader_instance.Download(archive_url, background=True)
# On any exception, reset the state of the downloader_dict.
self._downloader_dict[archive_url] = None
def wait_for_status(self, **kwargs):
"""Waits for background artifacts to be downloaded from Google Storage.
archive_url: Google Storage URL for the build.
Example URL:
archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
downloader_instance = self._downloader_dict.get(archive_url)
if downloader_instance:
status = downloader_instance.GetStatusOfBackgroundDownloads()
self._downloader_dict[archive_url] = None
return status
# We may have previously downloaded but removed the downloader instance
# from the cache.
if downloader.Downloader.BuildStaged(archive_url, updater.static_dir):'%s not found in downloader cache but previously staged.',
return 'Success'
raise DevServerError('No download for the given archive_url found.')
def stage_debug(self, **kwargs):
"""Downloads and stages debug symbol payloads from Google Storage.
This methods downloads the debug symbol build artifact synchronously,
and then stages it for use by symbolicate_dump/.
archive_url: Google Storage URL for the build.
Example URL:
archive_url = self._canonicalize_archive_url(kwargs.get('archive_url'))
return downloader.SymbolDownloader(updater.static_dir).Download(archive_url)
def symbolicate_dump(self, minidump):
"""Symbolicates a minidump using pre-downloaded symbols, returns it.
Callers will need to POST to this URL with a body of MIME-type
The body should include a single argument, 'minidump', containing the
binary-formatted minidump to symbolicate.
It is up to the caller to ensure that the symbols they want are currently
minidump: The binary minidump file to symbolicate.
to_return = ''
with tempfile.NamedTemporaryFile() as local:
while True:
data =
if not data:
stackwalk = subprocess.Popen(['minidump_stackwalk',,
updater.static_dir + '/debug/breakpad'],
to_return, error_text = stackwalk.communicate()
if stackwalk.returncode != 0:
raise DevServerError("Can't generate stack trace: %s (rc=%d)" % (
error_text, stackwalk.returncode))
return to_return
def latestbuild(self, **params):
"""Return a string representing the latest build for a given target.
target: The build target, typically a combination of the board and the
type of build e.g. x86-mario-release.
milestone: The milestone to filter builds on. E.g. R16. Optional, if not
provided the latest RXX build will be returned.
A string representation of the latest build if one exists, i.e.
An empty string if no latest could be found.
if not params:
return _PrintDocStringAsHTML(self.latestbuild)
if 'target' not in params:
raise cherrypy.HTTPError('500 Internal Server Error',
'Error: target= is required!')
return devserver_util.GetLatestBuildVersion(
updater.static_dir, params['target'],
except devserver_util.DevServerUtilError as errmsg:
raise cherrypy.HTTPError('500 Internal Server Error', str(errmsg))
def controlfiles(self, **params):
"""Return a control file or a list of all known control files.
Example URL:
To List all control files:
To return the contents of a path:
build: The build i.e. x86-alex-release/R18-1514.0.0-a1-b1450.
control_path: If you want the contents of a control file set this
to the path. E.g. client/site_tests/sleeptest/control
Optional, if not provided return a list of control files is returned.
Contents of a control file if control_path is provided.
A list of control files if no control_path is provided.
if not params:
return _PrintDocStringAsHTML(self.controlfiles)
if 'build' not in params:
raise cherrypy.HTTPError('500 Internal Server Error',
'Error: build= is required!')
if 'control_path' not in params:
return devserver_util.GetControlFileList(updater.static_dir,
return devserver_util.GetControlFile(updater.static_dir, params['build'],
def index(self):
return 'Welcome to the Dev Server!'
def update(self, *args):
label = '/'.join(args)
body_length = int(cherrypy.request.headers.get('Content-Length', 0))
data =
return updater.HandleUpdatePing(data, label)
def main():
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage=usage)
parser.add_option('--archive_dir', dest='archive_dir',
help='serve archived builds only.')
parser.add_option('--board', dest='board',
help='When pre-generating update, board for latest image.')
parser.add_option('--clear_cache', action='store_true', default=False,
help='Clear out all cached udpates and exit')
parser.add_option('--client_prefix', dest='client_prefix_deprecated',
help='No longer used. It is still here so we don\'t break '
'scripts that used it.', default='')
parser.add_option('--critical_update', dest='critical_update',
action='store_true', default=False,
help='Present update payload as critical')
parser.add_option('--data_dir', dest='data_dir',
help='Writable directory where static lives',
parser.add_option('--exit', action='store_true', default=False,
help='Don\'t start the server (still pregenerate or clear'
parser.add_option('--factory_config', dest='factory_config',
help='Config file for serving images from factory floor.')
parser.add_option('--for_vm', dest='vm', default=False, action='store_true',
help='Update is for a vm image.')
parser.add_option('--image', dest='image',
help='Force update using this image.')
parser.add_option('-p', '--pregenerate_update', action='store_true',
default=False, help='Pre-generate update payload.')
parser.add_option('--payload', dest='payload',
help='Use update payload from specified directory.')
parser.add_option('--port', default=8080,
help='Port for the dev server to use (default: 8080).')
parser.add_option('--private_key', default=None,
help='Path to the private key in pem format.')
parser.add_option('--production', action='store_true', default=False,
help='Have the devserver use production values.')
parser.add_option('--proxy_port', default=None,
help='Port to have the client connect to (testing support)')
parser.add_option('--src_image', default='',
help='Image on remote machine for generating delta update.')
parser.add_option('-t', action='store_true', dest='test_image')
parser.add_option('-u', '--urlbase', dest='urlbase',
help='base URL, other than devserver, for update images.')
parser.add_option('--validate_factory_config', action="store_true",
help='Validate factory config file, then exit.')
parser.add_option('-l', '--log-dir', default=None,
help=('Specify a directory for error and access logs. '
'Default None, i.e. no logging.'))
(options, _) = parser.parse_args()
devserver_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
root_dir = os.path.realpath('%s/../..' % devserver_dir)
serve_only = False
static_dir = os.path.realpath('%s/static' % options.data_dir)
os.system('mkdir -p %s' % static_dir)
if options.log_dir and not os.path.isdir(options.log_dir):
parser.error('%s is not a valid dir, provide a valid dir to --log-dir' %
if options.archive_dir:
# TODO(zbehan) Remove legacy support:
# archive_dir is the directory where static/archive will point.
# If this is an absolute path, all is fine. If someone calls this
# using a relative path, that is relative to src/platform/dev/.
# That use case is unmaintainable, but since applications use it
# with =./static, instead of a boolean flag, we'll make this relative
# to devserver_dir to keep these unbroken. For now.
archive_dir = options.archive_dir
if not os.path.isabs(archive_dir):
archive_dir = os.path.realpath(os.path.join(devserver_dir, archive_dir))
_PrepareToServeUpdatesOnly(archive_dir, static_dir)
static_dir = os.path.realpath(archive_dir)
serve_only = True
cache_dir = os.path.join(static_dir, 'cache')
cherrypy.log('Using cache directory %s' % cache_dir, 'DEVSERVER')
if os.path.exists(cache_dir):
if options.clear_cache:
# Clear the cache and exit on error.
cmd = 'rm -rf %s/*' % cache_dir
if os.system(cmd) != 0:
cherrypy.log('Failed to clear the cache with %s' % cmd,
# Clear all but the last N cached updates
cmd = ('cd %s; ls -tr | head --lines=-%d | xargs rm -rf' %
(cache_dir, CACHED_ENTRIES))
if os.system(cmd) != 0:
cherrypy.log('Failed to clean up old delta cache files with %s' % cmd,
if options.client_prefix_deprecated:
cherrypy.log('The --client_prefix argument is DEPRECATED, '
'and is no longer needed.', 'DEVSERVER')
cherrypy.log('Data dir is %s' % options.data_dir, 'DEVSERVER')
cherrypy.log('Source root is %s' % root_dir, 'DEVSERVER')
cherrypy.log('Serving from %s' % static_dir, 'DEVSERVER')
global updater
updater = autoupdate.Autoupdate(
copy_to_static_root=not options.exit,
# Sanity-check for use of validate_factory_config.
if not options.factory_config and options.validate_factory_config:
parser.error('You need a factory_config to validate.')
if options.factory_config:
# We don't run the dev server with this option.
if options.validate_factory_config:
elif options.pregenerate_update:
if not updater.PreGenerateUpdate():
# If the command line requested after setup, it's time to do it.
if not options.exit:
cherrypy.quickstart(DevServerRoot(), config=_GetConfig(options))
if __name__ == '__main__':