blob: 8eb5325b4eaf63965994925bfdccb43ffe714836 [file] [log] [blame]
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Library for uploading command stats to AppEngine."""
import contextlib
import logging
import os
import parallel
import urllib
import urllib2
from chromite.buildbot import constants
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import timeout_util
class Stats(object):
"""Entity object for a stats entry."""
# These attributes correspond to the fields of a stats record.
__slots__ = (
def __init__(self, **kwargs):
"""Initialize the record.
**kwargs keys need to correspond to elements in __slots__. These arguments
can be lists:
- cmd_args
- cmd_base
- cmd_line
If unset, the |username| and |host| attributes will be determined
if kwargs.get('username') is None:
kwargs['username'] = git.GetProjectUserEmail(os.path.dirname(__file__))
if kwargs.get('host') is None:
kwargs['host'] = cros_build_lib.GetHostName(fully_qualified=True)
for attr in ('cmd_args', 'cmd_base', 'cmd_line'):
val = kwargs.get(attr)
if isinstance(val, (list, tuple,)):
kwargs[attr] = ' '.join(map(repr, val))
for arg in self.__slots__:
setattr(self, arg, kwargs.pop(arg, None))
if kwargs:
raise TypeError('Unknown options specified %r:' % kwargs)
def data(self):
"""Retrieves a dictionary representing the fields that are set."""
data = {}
for arg in self.__slots__:
val = getattr(self, arg)
if val is not None:
data[arg] = val
return data
def SafeInit(cls, **kwargs):
"""Construct a Stats object, catching any exceptions.
See Stats.__init__() for argument list.
A Stats() instance if things went smoothly, and None if exceptions were
caught in the process.
inst = cls(**kwargs)
except Exception:
logging.error('Exception during stats upload.', exc_info=True)
return inst
class StatsUploader(object):
"""Functionality to upload the stats to the AppEngine server."""
# To test with an app engine instance on localhost, set envvar
# export CROS_BUILD_STATS_SITE="http://localhost:8080"
_PAGE = 'upload_command_stats'
URL = '%s/%s' % (_SITE, _PAGE)
_DISABLE_FILE = '~/.disable_build_stats_upload'
TIMEOUT_ERROR = 'Timed out during command stat upload - waited %s seconds'
ENVIRONMENT_ERROR = 'Exception during command stat upload.'
HTTPURL_ERROR = 'Exception during command stat upload to %s.'
def _UploadConditionsMet(cls, stats):
"""Return True if upload conditions are met."""
def CheckDomain(hostname):
return any(hostname.endswith(d) for d in cls._DOMAIN_WHITELIST)
def CheckEmail(email):
return any(email.endswith(e) for e in cls._EMAIL_WHITELIST)
upload = False
# Verify that host domain is in or
if not or not CheckDomain(
logging.debug('Host %s is not a Google machine.',
elif not stats.username:
logging.debug('Unable to determine current "git id".')
elif not CheckEmail(stats.username):
logging.debug('%s is not a Google or Chromium user.', stats.username)
elif os.path.exists(osutils.ExpandPath(cls._DISABLE_FILE)):
logging.debug('Found %s', cls._DISABLE_FILE)
upload = True
if not upload:
logging.debug('Skipping stats upload.')
return upload
def Upload(cls, stats, url=None, timeout=None):
"""Upload |stats| to |url|.
Does nothing if upload conditions aren't met.
stats: A Stats object to upload.
url: The url to send the request to.
timeout: A timeout value to set, in seconds.
if url is None:
url = cls.URL
if timeout is None:
timeout = cls.UPLOAD_TIMEOUT
if not cls._UploadConditionsMet(stats):
with timeout_util.Timeout(timeout):
cls._Upload(stats, url)
# Stats upload errors are silenced, for the sake of user experience.
except timeout_util.TimeoutError:
logging.debug(cls.TIMEOUT_ERROR, timeout)
except urllib2.HTTPError as e:
# HTTPError has a geturl() method, but it relies on self.url, which
# is not always set. In looking at source, self.filename equals url.
logging.debug(cls.HTTPURL_ERROR, e.filename, exc_info=True)
except EnvironmentError:
logging.debug(cls.ENVIRONMENT_ERROR, exc_info=True)
def _Upload(cls, stats, url):
logging.debug('Uploading command stats to %r', url)
data = urllib.urlencode(
request = urllib2.Request(url)
urllib2.urlopen(request, data)
UNCAUGHT_UPLOAD_ERROR = 'Uncaught command stats exception'
def UploadContext():
"""Provides a context where stats are uploaded in the background.
A queue that accepts an arg-list of the format [stats, url, timeout].
# We need to use parallel.BackgroundTaskRunner, and not
# parallel.RunParallelTasks, because with RunParallelTasks, both the
# uploader and the subcommand are treated as background tasks, and the
# subcommand will lose responsiveness, since its output will be buffered.
with parallel.BackgroundTaskRunner(
StatsUploader.Upload, processes=1) as queue:
yield queue
except parallel.BackgroundFailure:
# Display unexpected errors, but don't propagate the error.
logging.error('Uncaught command stats exception', exc_info=True)