cros_utils: clean up unused file/code
As an effort to fix up naming in crbug.com/1099035, we can just
remove the unused code and variable in these two files.
BUG=chromium:1099035
TEST=buildbot_utils_unittest.py pass after the removal.
Change-Id: Id4c12aeab9bf44b67e58b11c4297ceafe441421d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2270723
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: Caroline Tice <cmtice@chromium.org>
Commit-Queue: Tiancong Wang <tcwang@google.com>
Tested-by: Tiancong Wang <tcwang@google.com>
diff --git a/cros_utils/buildbot_json.py b/cros_utils/buildbot_json.py
deleted file mode 100755
index 08a8ae0..0000000
--- a/cros_utils/buildbot_json.py
+++ /dev/null
@@ -1,1534 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# NOTE: This file is NOT under GPL. See above.
-
-"""Queries buildbot through the json interface.
-"""
-
-from __future__ import print_function
-
-__author__ = 'maruel@chromium.org'
-__version__ = '1.2'
-
-import code
-import datetime
-import functools
-import json
-
-# Pylint recommends we use "from chromite.lib import cros_logging as logging".
-# Chromite specific policy message, we want to keep using the standard logging.
-# pylint: disable=cros-logging-import
-import logging
-
-# pylint: disable=deprecated-module
-import optparse
-
-import time
-import sys
-import urllib.error
-import urllib.parse
-import urllib.request
-
-try:
- from natsort import natsorted
-except ImportError:
- # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted
- # after "vm7". Defaults to normal sorting.
- natsorted = sorted
-
-# These values are buildbot constants used for Build and BuildStep.
-# This line was copied from master/buildbot/status/builder.py.
-SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = list(range(6))
-
-## Generic node caching code.
-
-
-class Node(object):
- """Root class for all nodes in the graph.
-
- Provides base functionality for any node in the graph, independent if it has
- children or not or if its content can be addressed through an url or needs to
- be fetched as part of another node.
-
- self.printable_attributes is only used for self documentation and for str()
- implementation.
- """
- printable_attributes = []
-
- def __init__(self, parent, url):
- self.printable_attributes = self.printable_attributes[:]
- if url:
- self.printable_attributes.append('url')
- url = url.rstrip('/')
- if parent is not None:
- self.printable_attributes.append('parent')
- self.url = url
- self.parent = parent
-
- def __str__(self):
- return self.to_string()
-
- def __repr__(self):
- """Embeds key if present."""
- key = getattr(self, 'key', None)
- if key is not None:
- return '<%s key=%s>' % (self.__class__.__name__, key)
- cached_keys = getattr(self, 'cached_keys', None)
- if cached_keys is not None:
- return '<%s keys=%s>' % (self.__class__.__name__, cached_keys)
- return super(Node, self).__repr__()
-
- def to_string(self, maximum=100):
- out = ['%s:' % self.__class__.__name__]
- assert not 'printable_attributes' in self.printable_attributes
-
- def limit(txt):
- txt = str(txt)
- if maximum > 0:
- if len(txt) > maximum + 2:
- txt = txt[:maximum] + '...'
- return txt
-
- for k in sorted(self.printable_attributes):
- if k == 'parent':
- # Avoid infinite recursion.
- continue
- out.append(limit(' %s: %r' % (k, getattr(self, k))))
- return '\n'.join(out)
-
- def refresh(self):
- """Refreshes the data."""
- self.discard()
- return self.cache()
-
- def cache(self): # pragma: no cover
- """Caches the data."""
- raise NotImplementedError()
-
- def discard(self): # pragma: no cover
- """Discards cached data.
-
- Pretty much everything is temporary except completed Build.
- """
- raise NotImplementedError()
-
-
-class AddressableBaseDataNode(Node): # pylint: disable=W0223
- """A node that contains a dictionary of data that can be fetched with an url.
-
- The node is directly addressable. It also often can be fetched by the parent.
- """
- printable_attributes = Node.printable_attributes + ['data']
-
- def __init__(self, parent, url, data):
- super(AddressableBaseDataNode, self).__init__(parent, url)
- self._data = data
-
- @property
- def cached_data(self):
- return self._data
-
- @property
- def data(self):
- self.cache()
- return self._data
-
- def cache(self):
- if self._data is None:
- self._data = self._readall()
- return True
- return False
-
- def discard(self):
- self._data = None
-
- def read(self, suburl):
- assert self.url, self.__class__.__name__
- url = self.url
- if suburl:
- url = '%s/%s' % (self.url, suburl)
- return self.parent.read(url)
-
- def _readall(self):
- return self.read('')
-
-
-class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223
- """Automatically encodes the url."""
-
- def __init__(self, parent, url, data):
- super(AddressableDataNode, self).__init__(parent, urllib.parse.quote(url),
- data)
-
-
-class NonAddressableDataNode(Node): # pylint: disable=W0223
- """A node that cannot be addressed by an unique url.
-
- The data comes directly from the parent.
- """
-
- def __init__(self, parent, subkey):
- super(NonAddressableDataNode, self).__init__(parent, None)
- self.subkey = subkey
-
- @property
- def cached_data(self):
- if self.parent.cached_data is None:
- return None
- return self.parent.cached_data[self.subkey]
-
- @property
- def data(self):
- return self.parent.data[self.subkey]
-
- def cache(self):
- self.parent.cache()
-
- def discard(self): # pragma: no cover
- """Avoid invalid state when parent recreate the object."""
- raise AttributeError('Call parent discard() instead')
-
-
-class VirtualNodeList(Node):
- """Base class for every node that has children.
-
- Adds partial supports for keys and iterator functionality. 'key' can be a
- string or a int. Not to be used directly.
- """
- printable_attributes = Node.printable_attributes + ['keys']
-
- def __init__(self, parent, url):
- super(VirtualNodeList, self).__init__(parent, url)
- # Keeps the keys independently when ordering is needed.
- self._is_cached = False
- self._has_keys_cached = False
-
- def __contains__(self, key):
- """Enables 'if i in obj:'."""
- return key in self.keys
-
- def __iter__(self):
- """Enables 'for i in obj:'. It returns children."""
- self.cache_keys()
- for key in self.keys:
- yield self[key]
-
- def __len__(self):
- """Enables 'len(obj)' to get the number of childs."""
- return len(self.keys)
-
- def discard(self):
- """Discards data.
-
- The default behavior is to not invalidate cached keys. The only place where
- keys need to be invalidated is with Builds.
- """
- self._is_cached = False
- self._has_keys_cached = False
-
- @property
- def cached_children(self): # pragma: no cover
- """Returns an iterator over the children that are cached."""
- raise NotImplementedError()
-
- @property
- def cached_keys(self): # pragma: no cover
- raise NotImplementedError()
-
- @property
- def keys(self): # pragma: no cover
- """Returns the keys for every children."""
- raise NotImplementedError()
-
- def __getitem__(self, key): # pragma: no cover
- """Returns a child, without fetching its data.
-
- The children could be invalid since no verification is done.
- """
- raise NotImplementedError()
-
- def cache(self): # pragma: no cover
- """Cache all the children."""
- raise NotImplementedError()
-
- def cache_keys(self): # pragma: no cover
- """Cache all children's keys."""
- raise NotImplementedError()
-
-
-class NodeList(VirtualNodeList): # pylint: disable=W0223
- """Adds a cache of the keys."""
-
- def __init__(self, parent, url):
- super(NodeList, self).__init__(parent, url)
- self._keys = []
-
- @property
- def cached_keys(self):
- return self._keys
-
- @property
- def keys(self):
- self.cache_keys()
- return self._keys
-
-
-class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223
- """A node that contains children but retrieves all its data from its parent.
-
- I.e. there's no url to get directly this data.
- """
- # Child class object for children of this instance. For example, BuildSteps
- # has BuildStep children.
- _child_cls = None
-
- def __init__(self, parent, subkey):
- super(NonAddressableNodeList, self).__init__(parent, None)
- self.subkey = subkey
- assert (not isinstance(self._child_cls, NonAddressableDataNode) and
- issubclass(self._child_cls, NonAddressableDataNode)), (
- self._child_cls.__name__)
-
- @property
- def cached_children(self):
- if self.parent.cached_data is not None:
- for i in range(len(self.parent.cached_data[self.subkey])):
- yield self[i]
-
- @property
- def cached_data(self):
- if self.parent.cached_data is None:
- return None
- return self.parent.data.get(self.subkey, None)
-
- @property
- def cached_keys(self):
- if self.parent.cached_data is None:
- return None
- return list(range(len(self.parent.data.get(self.subkey, []))))
-
- @property
- def data(self):
- return self.parent.data[self.subkey]
-
- def cache(self):
- self.parent.cache()
-
- def cache_keys(self):
- self.parent.cache()
-
- def discard(self): # pragma: no cover
- """Do not call.
-
- Avoid infinite recursion by having the caller calls the parent's
- discard() explicitely.
- """
- raise AttributeError('Call parent discard() instead')
-
- def __iter__(self):
- """Enables 'for i in obj:'. It returns children."""
- if self.data:
- for i in range(len(self.data)):
- yield self[i]
-
- def __getitem__(self, key):
- """Doesn't cache the value, it's not needed.
-
- TODO(maruel): Cache?
- """
- if isinstance(key, int) and key < 0:
- key = len(self.data) + key
- # pylint: disable=E1102
- return self._child_cls(self, key)
-
-
-class AddressableNodeList(NodeList):
- """A node that has children that can be addressed with an url."""
-
- # Child class object for children of this instance. For example, Builders has
- # Builder children and Builds has Build children.
- _child_cls = None
-
- def __init__(self, parent, url):
- super(AddressableNodeList, self).__init__(parent, url)
- self._cache = {}
- assert (not isinstance(self._child_cls, AddressableDataNode) and
- issubclass(self._child_cls, AddressableDataNode)), (
- self._child_cls.__name__)
-
- @property
- def cached_children(self):
- for item in self._cache.values():
- if item.cached_data is not None:
- yield item
-
- @property
- def cached_keys(self):
- return list(self._cache.keys())
-
- def __getitem__(self, key):
- """Enables 'obj[i]'."""
- if self._has_keys_cached and not key in self._keys:
- raise KeyError(key)
-
- if not key in self._cache:
- # Create an empty object.
- self._create_obj(key, None)
- return self._cache[key]
-
- def cache(self):
- if not self._is_cached:
- data = self._readall()
- for key in sorted(data):
- self._create_obj(key, data[key])
- self._is_cached = True
- self._has_keys_cached = True
-
- def cache_partial(self, children):
- """Caches a partial number of children.
-
- This method is more efficient since it does a single request for all the
- children instead of one request per children.
-
- It only grab objects not already cached.
- """
- # pylint: disable=W0212
- if not self._is_cached:
- to_fetch = [
- child for child in children
- if not (child in self._cache and self._cache[child].cached_data)
- ]
- if to_fetch:
- # Similar to cache(). The only reason to sort is to simplify testing.
- params = '&'.join(
- 'select=%s' % urllib.parse.quote(str(v)) for v in sorted(to_fetch))
- data = self.read('?' + params)
- for key in sorted(data):
- self._create_obj(key, data[key])
-
- def cache_keys(self):
- """Implement to speed up enumeration. Defaults to call cache()."""
- if not self._has_keys_cached:
- self.cache()
- assert self._has_keys_cached
-
- def discard(self):
- """Discards temporary children."""
- super(AddressableNodeList, self).discard()
- for v in self._cache.values():
- v.discard()
-
- def read(self, suburl):
- assert self.url, self.__class__.__name__
- url = self.url
- if suburl:
- url = '%s/%s' % (self.url, suburl)
- return self.parent.read(url)
-
- def _create_obj(self, key, data):
- """Creates an object of type self._child_cls."""
- # pylint: disable=E1102
- obj = self._child_cls(self, key, data)
- # obj.key and key may be different.
- # No need to overide cached data with None.
- if data is not None or obj.key not in self._cache:
- self._cache[obj.key] = obj
- if obj.key not in self._keys:
- self._keys.append(obj.key)
-
- def _readall(self):
- return self.read('')
-
-
-class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223
- """A node that shows a subset of children that comes from another structure.
-
- The node is not addressable.
-
- E.g. the keys are retrieved from parent but the actual data comes from
- virtual_parent.
- """
-
- def __init__(self, parent, virtual_parent, subkey):
- super(SubViewNodeList, self).__init__(parent, None)
- self.subkey = subkey
- self.virtual_parent = virtual_parent
- assert isinstance(self.parent, AddressableDataNode)
- assert isinstance(self.virtual_parent, NodeList)
-
- @property
- def cached_children(self):
- if self.parent.cached_data is not None:
- for item in self.keys:
- if item in self.virtual_parent.keys:
- child = self[item]
- if child.cached_data is not None:
- yield child
-
- @property
- def cached_keys(self):
- return (self.parent.cached_data or {}).get(self.subkey, [])
-
- @property
- def keys(self):
- self.cache_keys()
- return self.parent.data.get(self.subkey, [])
-
- def cache(self):
- """Batch request for each child in a single read request."""
- if not self._is_cached:
- self.virtual_parent.cache_partial(self.keys)
- self._is_cached = True
-
- def cache_keys(self):
- if not self._has_keys_cached:
- self.parent.cache()
- self._has_keys_cached = True
-
- def discard(self):
- if self.parent.cached_data is not None:
- for child in self.virtual_parent.cached_children:
- if child.key in self.keys:
- child.discard()
- self.parent.discard()
- super(SubViewNodeList, self).discard()
-
- def __getitem__(self, key):
- """Makes sure the key is in our key but grab it from the virtual parent."""
- return self.virtual_parent[key]
-
- def __iter__(self):
- self.cache()
- return super(SubViewNodeList, self).__iter__()
-
-
-# Buildbot-specific code
-
-
-class Slave(AddressableDataNode):
- """Buildbot slave class."""
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'name',
- 'key',
- 'connected',
- 'version',
- ]
-
- def __init__(self, parent, name, data):
- super(Slave, self).__init__(parent, name, data)
- self.name = name
- self.key = self.name
- # TODO(maruel): Add SlaveBuilders and a 'builders' property.
- # TODO(maruel): Add a 'running_builds' property.
-
- @property
- def connected(self):
- return self.data.get('connected', False)
-
- @property
- def version(self):
- return self.data.get('version')
-
-
-class Slaves(AddressableNodeList):
- """Buildbot slaves."""
- _child_cls = Slave
- printable_attributes = AddressableNodeList.printable_attributes + ['names']
-
- def __init__(self, parent):
- super(Slaves, self).__init__(parent, 'slaves')
-
- @property
- def names(self):
- return self.keys
-
-
-class BuilderSlaves(SubViewNodeList):
- """Similar to Slaves but only list slaves connected to a specific builder."""
- printable_attributes = SubViewNodeList.printable_attributes + ['names']
-
- def __init__(self, parent):
- super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves,
- 'slaves')
-
- @property
- def names(self):
- return self.keys
-
-
-class BuildStep(NonAddressableDataNode):
- """Class for a buildbot build step."""
- printable_attributes = NonAddressableDataNode.printable_attributes + [
- 'name',
- 'number',
- 'start_time',
- 'end_time',
- 'duration',
- 'is_started',
- 'is_finished',
- 'is_running',
- 'result',
- 'simplified_result',
- ]
-
- def __init__(self, parent, number):
- """Pre-loaded, since the data is retrieved via the Build object."""
- assert isinstance(number, int)
- super(BuildStep, self).__init__(parent, number)
- self.number = number
-
- @property
- def start_time(self):
- if self.data.get('times'):
- return int(round(self.data['times'][0]))
-
- @property
- def end_time(self):
- times = self.data.get('times')
- if times and len(times) == 2 and times[1]:
- return int(round(times[1]))
-
- @property
- def duration(self):
- if self.start_time:
- return (self.end_time or int(round(time.time()))) - self.start_time
-
- @property
- def name(self):
- return self.data['name']
-
- @property
- def is_started(self):
- return self.data.get('isStarted', False)
-
- @property
- def is_finished(self):
- return self.data.get('isFinished', False)
-
- @property
- def is_running(self):
- return self.is_started and not self.is_finished
-
- @property
- def result(self):
- result = self.data.get('results')
- if result is None:
- # results may be 0, in that case with filter=1, the value won't be
- # present.
- if self.data.get('isFinished'):
- result = self.data.get('results', 0)
- while isinstance(result, list):
- result = result[0]
- return result
-
- @property
- def simplified_result(self):
- """Returns a simplified 3 state value, True, False or None."""
- result = self.result
- if result in (SUCCESS, WARNINGS):
- return True
- elif result in (FAILURE, EXCEPTION, RETRY):
- return False
- assert result in (None, SKIPPED), (result, self.data)
- return None
-
-
-class BuildSteps(NonAddressableNodeList):
- """Duplicates keys to support lookup by both step number and step name."""
- printable_attributes = NonAddressableNodeList.printable_attributes + [
- 'failed',
- ]
- _child_cls = BuildStep
-
- def __init__(self, parent):
- """Pre-loaded, since the data is retrieved via the Build object."""
- super(BuildSteps, self).__init__(parent, 'steps')
-
- @property
- def keys(self):
- """Returns the steps name in order."""
- return [i['name'] for i in self.data or []]
-
- @property
- def failed(self):
- """Shortcuts that lists the step names of steps that failed."""
- return [step.name for step in self if step.simplified_result is False]
-
- def __getitem__(self, key):
- """Accept step name in addition to index number."""
- if isinstance(key, str):
- # It's a string, try to find the corresponding index.
- for i, step in enumerate(self.data):
- if step['name'] == key:
- key = i
- break
- else:
- raise KeyError(key)
- return super(BuildSteps, self).__getitem__(key)
-
-
-class Build(AddressableDataNode):
- """Buildbot build info."""
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'key',
- 'number',
- 'steps',
- 'blame',
- 'reason',
- 'revision',
- 'result',
- 'simplified_result',
- 'start_time',
- 'end_time',
- 'duration',
- 'slave',
- 'properties',
- 'completed',
- ]
-
- def __init__(self, parent, key, data):
- super(Build, self).__init__(parent, str(key), data)
- self.number = int(key)
- self.key = self.number
- self.steps = BuildSteps(self)
-
- @property
- def blame(self):
- return self.data.get('blame', [])
-
- @property
- def builder(self):
- """Returns the Builder object.
-
- Goes up the hierarchy to find the Buildbot.builders[builder] instance.
- """
- return self.parent.parent.parent.parent.builders[self.data['builderName']]
-
- @property
- def start_time(self):
- if self.data.get('times'):
- return int(round(self.data['times'][0]))
-
- @property
- def end_time(self):
- times = self.data.get('times')
- if times and len(times) == 2 and times[1]:
- return int(round(times[1]))
-
- @property
- def duration(self):
- if self.start_time:
- return (self.end_time or int(round(time.time()))) - self.start_time
-
- @property
- def eta(self):
- return self.data.get('eta', 0)
-
- @property
- def completed(self):
- return self.data.get('currentStep') is None
-
- @property
- def properties(self):
- return self.data.get('properties', [])
-
- @property
- def reason(self):
- return self.data.get('reason')
-
- @property
- def result(self):
- result = self.data.get('results')
- while isinstance(result, list):
- result = result[0]
- if result is None and self.steps:
- # results may be 0, in that case with filter=1, the value won't be
- # present.
- result = self.steps[-1].result
- return result
-
- @property
- def revision(self):
- return self.data.get('sourceStamp', {}).get('revision')
-
- @property
- def simplified_result(self):
- """Returns a simplified 3 state value, True, False or None."""
- result = self.result
- if result in (SUCCESS, WARNINGS, SKIPPED):
- return True
- elif result in (FAILURE, EXCEPTION, RETRY):
- return False
- assert result is None, (result, self.data)
- return None
-
- @property
- def slave(self):
- """Returns the Slave object.
-
- Goes up the hierarchy to find the Buildbot.slaves[slave] instance.
- """
- return self.parent.parent.parent.parent.slaves[self.data['slave']]
-
- def discard(self):
- """Completed Build isn't discarded."""
- if self._data and self.result is None:
- assert not self.steps or not self.steps[-1].data.get('isFinished')
- self._data = None
-
-
-class CurrentBuilds(SubViewNodeList):
- """Lists of the current builds."""
-
- def __init__(self, parent):
- super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds')
-
-
-class PendingBuilds(AddressableDataNode):
- """List of the pending builds."""
-
- def __init__(self, parent):
- super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None)
-
-
-class Builds(AddressableNodeList):
- """Supports iteration.
-
- Recommends using .cache() to speed up if a significant number of builds are
- iterated over.
- """
- _child_cls = Build
-
- def __init__(self, parent):
- super(Builds, self).__init__(parent, 'builds')
-
- def __getitem__(self, key):
- """Support for negative reference and enable retrieving non-cached builds.
-
- e.g. -1 is the last build, -2 is the previous build before the last one.
- """
- key = int(key)
- if key < 0:
- # Convert negative to positive build number.
- self.cache_keys()
- # Since the negative value can be outside of the cache keys range, use the
- # highest key value and calculate from it.
- key = max(self._keys) + key + 1
-
- if key not in self._cache:
- # Create an empty object.
- self._create_obj(key, None)
- return self._cache[key]
-
- def __iter__(self):
- """Returns cached Build objects in reversed order.
-
- The most recent build is returned first and then in reverse chronological
- order, up to the oldest cached build by the server. Older builds can be
- accessed but will trigger significantly more I/O so they are not included by
- default in the iteration.
-
- To access the older builds, use self.iterall() instead.
- """
- self.cache()
- return reversed(list(self._cache.values()))
-
- def iterall(self):
- """Returns Build objects in decreasing order unbounded up to build 0.
-
- The most recent build is returned first and then in reverse chronological
- order. Older builds can be accessed and will trigger significantly more I/O
- so use this carefully.
- """
- # Only cache keys here.
- self.cache_keys()
- if self._keys:
- for i in range(max(self._keys), -1, -1):
- yield self[i]
-
- def cache_keys(self):
- """Grabs the keys (build numbers) from the builder."""
- if not self._has_keys_cached:
- for i in self.parent.data.get('cachedBuilds', []):
- i = int(i)
- self._cache.setdefault(i, Build(self, i, None))
- if i not in self._keys:
- self._keys.append(i)
- self._has_keys_cached = True
-
- def discard(self):
- super(Builds, self).discard()
- # Can't keep keys.
- self._has_keys_cached = False
-
- def _readall(self):
- return self.read('_all')
-
-
-class Builder(AddressableDataNode):
- """Builder status."""
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'name',
- 'key',
- 'builds',
- 'slaves',
- 'pending_builds',
- 'current_builds',
- ]
-
- def __init__(self, parent, name, data):
- super(Builder, self).__init__(parent, name, data)
- self.name = name
- self.key = name
- self.builds = Builds(self)
- self.slaves = BuilderSlaves(self)
- self.current_builds = CurrentBuilds(self)
- self.pending_builds = PendingBuilds(self)
-
- def discard(self):
- super(Builder, self).discard()
- self.builds.discard()
- self.slaves.discard()
- self.current_builds.discard()
-
-
-class Builders(AddressableNodeList):
- """Root list of builders."""
- _child_cls = Builder
-
- def __init__(self, parent):
- super(Builders, self).__init__(parent, 'builders')
-
-
-class Buildbot(AddressableBaseDataNode):
- """This object should be recreated on a master restart as it caches data."""
- # Throttle fetches to not kill the server.
- auto_throttle = None
- printable_attributes = AddressableDataNode.printable_attributes + [
- 'slaves',
- 'builders',
- 'last_fetch',
- ]
-
- def __init__(self, url):
- super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None)
- self._builders = Builders(self)
- self._slaves = Slaves(self)
- self.last_fetch = None
-
- @property
- def builders(self):
- return self._builders
-
- @property
- def slaves(self):
- return self._slaves
-
- def discard(self):
- """Discards information about Builders and Slaves."""
- super(Buildbot, self).discard()
- self._builders.discard()
- self._slaves.discard()
-
- def read(self, suburl):
- if self.auto_throttle:
- if self.last_fetch:
- delta = datetime.datetime.utcnow() - self.last_fetch
- remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta)
- if remaining > datetime.timedelta(seconds=0):
- logging.debug('Sleeping for %ss', remaining)
- time.sleep(remaining.seconds)
- self.last_fetch = datetime.datetime.utcnow()
- url = '%s/%s' % (self.url, suburl)
- if '?' in url:
- url += '&filter=1'
- else:
- url += '?filter=1'
- logging.info('read(%s)', suburl)
- channel = urllib.request.urlopen(url)
- data = channel.read()
- try:
- return json.loads(data)
- except ValueError:
- if channel.getcode() >= 400:
- # Convert it into an HTTPError for easier processing.
- raise urllib.error.HTTPError(url, channel.getcode(),
- '%s:\n%s' % (url, data), channel.headers,
- None)
- raise
-
- def _readall(self):
- return self.read('project')
-
-
-# Controller code
-
-
-def usage(more):
-
- def hook(fn):
- fn.func_usage_more = more
- return fn
-
- return hook
-
-
-def need_buildbot(fn):
- """Post-parse args to create a buildbot object."""
-
- @functools.wraps(fn)
- def hook(parser, args, *extra_args, **kwargs):
- old_parse_args = parser.parse_args
-
- def new_parse_args(args):
- options, args = old_parse_args(args)
- if len(args) < 1:
- parser.error('Need to pass the root url of the buildbot')
- url = args.pop(0)
- if not url.startswith('http'):
- url = 'http://' + url
- buildbot = Buildbot(url)
- buildbot.auto_throttle = options.throttle
- return options, args, buildbot
-
- parser.parse_args = new_parse_args
- # Call the original function with the modified parser.
- return fn(parser, args, *extra_args, **kwargs)
-
- hook.func_usage_more = '[options] <url>'
- return hook
-
-
-@need_buildbot
-def CMDpending(parser, args):
- """Lists pending jobs."""
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.builders:
- options.builders = buildbot.builders.keys
- for builder in options.builders:
- builder = buildbot.builders[builder]
- pending_builds = builder.data.get('pendingBuilds', 0)
- if not pending_builds:
- continue
- print('Builder %s: %d' % (builder.name, pending_builds))
- if not options.quiet:
- for pending in builder.pending_builds.data:
- if 'revision' in pending['source']:
- print(' revision: %s' % pending['source']['revision'])
- for change in pending['source']['changes']:
- print(' change:')
- print(' comment: %r' % change['comments'][:50])
- print(' who: %s' % change['who'])
- return 0
-
-
-@usage('[options] <url> [commands] ...')
-@need_buildbot
-def CMDrun(parser, args):
- """Runs commands passed as parameters.
-
- When passing commands on the command line, each command will be run as if it
- was on its own line.
- """
- parser.add_option('-f', '--file', help='Read script from file')
- parser.add_option(
- '-i', dest='use_stdin', action='store_true', help='Read script on stdin')
- # Variable 'buildbot' is not used directly.
- # pylint: disable=W0612
- options, args, _ = parser.parse_args(args)
- if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1:
- parser.error('Need to pass only one of: <commands>, -f <file> or -i')
- if options.use_stdin:
- cmds = sys.stdin.read()
- elif options.file:
- cmds = open(options.file).read()
- else:
- cmds = '\n'.join(args)
- compiled = compile(cmds, '<cmd line>', 'exec')
- # pylint: disable=eval-used
- eval(compiled, globals(), locals())
- return 0
-
-
-@need_buildbot
-def CMDinteractive(parser, args):
- """Runs an interactive shell to run queries."""
- _, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- prompt = ('Buildbot interactive console for "%s".\n'
- "Hint: Start with typing: 'buildbot.printable_attributes' or "
- "'print str(buildbot)' to explore.") % buildbot.url[:-len('/json')]
- local_vars = {'buildbot': buildbot, 'b': buildbot}
- code.interact(prompt, None, local_vars)
-
-
-@need_buildbot
-def CMDidle(parser, args):
- """Lists idle slaves."""
- return find_idle_busy_slaves(parser, args, True)
-
-
-@need_buildbot
-def CMDbusy(parser, args):
- """Lists idle slaves."""
- return find_idle_busy_slaves(parser, args, False)
-
-
-@need_buildbot
-def CMDdisconnected(parser, args):
- """Lists disconnected slaves."""
- _, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- for slave in buildbot.slaves:
- if not slave.connected:
- print(slave.name)
- return 0
-
-
-def find_idle_busy_slaves(parser, args, show_idle):
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '-s',
- '--slave',
- dest='slaves',
- action='append',
- default=[],
- help='Slaves to filter on')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.builders:
- options.builders = buildbot.builders.keys
- for builder in options.builders:
- builder = buildbot.builders[builder]
- if options.slaves:
- # Only the subset of slaves connected to the builder.
- slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
- if not slaves:
- continue
- else:
- slaves = builder.slaves.names
- busy_slaves = [build.slave.name for build in builder.current_builds]
- if show_idle:
- slaves = natsorted(set(slaves) - set(busy_slaves))
- else:
- slaves = natsorted(set(slaves) & set(busy_slaves))
- if options.quiet:
- for slave in slaves:
- print(slave)
- else:
- if slaves:
- print('Builder %s: %s' % (builder.name, ', '.join(slaves)))
- return 0
-
-
-def last_failure(buildbot,
- builders=None,
- slaves=None,
- steps=None,
- no_cache=False):
- """Returns Build object with last failure with the specific filters."""
- builders = builders or buildbot.builders.keys
- for builder in builders:
- builder = buildbot.builders[builder]
- if slaves:
- # Only the subset of slaves connected to the builder.
- builder_slaves = list(set(slaves).intersection(set(builder.slaves.names)))
- if not builder_slaves:
- continue
- else:
- builder_slaves = builder.slaves.names
-
- if not no_cache and len(builder.slaves) > 2:
- # Unless you just want the last few builds, it's often faster to
- # fetch the whole thing at once, at the cost of a small hickup on
- # the buildbot.
- # TODO(maruel): Cache only N last builds or all builds since
- # datetime.
- builder.builds.cache()
-
- found = []
- for build in builder.builds:
- if build.slave.name not in builder_slaves or build.slave.name in found:
- continue
- # Only add the slave for the first completed build but still look for
- # incomplete builds.
- if build.completed:
- found.append(build.slave.name)
-
- if steps:
- if any(build.steps[step].simplified_result is False for step in steps):
- yield build
- elif build.simplified_result is False:
- yield build
-
- if len(found) == len(builder_slaves):
- # Found all the slaves, quit.
- break
-
-
-@need_buildbot
-def CMDlast_failure(parser, args):
- """Lists all slaves that failed on that step on their last build.
-
- Examples:
- To find all slaves where their last build was a compile failure,
- run with --step compile
- """
- parser.add_option(
- '-S',
- '--step',
- dest='steps',
- action='append',
- default=[],
- help='List all slaves that failed on that step on their last build')
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '-s',
- '--slave',
- dest='slaves',
- action='append',
- default=[],
- help='Slaves to filter on')
- parser.add_option(
- '-n',
- '--no_cache',
- action='store_true',
- help="Don't load all builds at once")
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- print_builders = not options.quiet and len(options.builders) != 1
- last_builder = None
- for build in last_failure(
- buildbot,
- builders=options.builders,
- slaves=options.slaves,
- steps=options.steps,
- no_cache=options.no_cache):
-
- if print_builders and last_builder != build.builder:
- print(build.builder.name)
- last_builder = build.builder
-
- if options.quiet:
- if options.slaves:
- print('%s: %s' % (build.builder.name, build.slave.name))
- else:
- print(build.slave.name)
- else:
- out = '%d on %s: blame:%s' % (build.number, build.slave.name, ', '.join(
- build.blame))
- if print_builders:
- out = ' ' + out
- print(out)
-
- if len(options.steps) != 1:
- for step in build.steps:
- if step.simplified_result is False:
- # Assume the first line is the text name anyway.
- summary = ', '.join(step.data['text'][1:])[:40]
- out = ' %s: "%s"' % (step.data['name'], summary)
- if print_builders:
- out = ' ' + out
- print(out)
- return 0
-
-
-@need_buildbot
-def CMDcurrent(parser, args):
- """Lists current jobs."""
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '--blame', action='store_true', help='Only print the blame list')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.builders:
- options.builders = buildbot.builders.keys
-
- if options.blame:
- blame = set()
- for builder in options.builders:
- for build in buildbot.builders[builder].current_builds:
- if build.blame:
- for blamed in build.blame:
- blame.add(blamed)
- print('\n'.join(blame))
- return 0
-
- for builder in options.builders:
- builder = buildbot.builders[builder]
- if not options.quiet and builder.current_builds:
- print(builder.name)
- for build in builder.current_builds:
- if options.quiet:
- print(build.slave.name)
- else:
- out = '%4d: slave=%10s' % (build.number, build.slave.name)
- out += ' duration=%5d' % (build.duration or 0)
- if build.eta:
- out += ' eta=%5.0f' % build.eta
- else:
- out += ' '
- if build.blame:
- out += ' blame=' + ', '.join(build.blame)
- print(out)
-
- return 0
-
-
-@need_buildbot
-def CMDbuilds(parser, args):
- """Lists all builds.
-
- Examples:
- To find all builds on a single slave, run with -b bar -s foo.
- """
- parser.add_option(
- '-r', '--result', type='int', help='Build result to filter on')
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- parser.add_option(
- '-s',
- '--slave',
- dest='slaves',
- action='append',
- default=[],
- help='Slaves to filter on')
- parser.add_option(
- '-n',
- '--no_cache',
- action='store_true',
- help="Don't load all builds at once")
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- builders = options.builders or buildbot.builders.keys
- for builder in builders:
- builder = buildbot.builders[builder]
- for build in builder.builds:
- if not options.slaves or build.slave.name in options.slaves:
- if options.quiet:
- out = ''
- if options.builders:
- out += '%s/' % builder.name
- if len(options.slaves) != 1:
- out += '%s/' % build.slave.name
- out += '%d revision:%s result:%s blame:%s' % (
- build.number, build.revision, build.result, ','.join(build.blame))
- print(out)
- else:
- print(build)
- return 0
-
-
-@need_buildbot
-def CMDcount(parser, args):
- """Count the number of builds that occured during a specific period."""
- parser.add_option(
- '-o', '--over', type='int', help='Number of seconds to look for')
- parser.add_option(
- '-b',
- '--builder',
- dest='builders',
- action='append',
- default=[],
- help='Builders to filter on')
- options, args, buildbot = parser.parse_args(args)
- if args:
- parser.error('Unrecognized parameters: %s' % ' '.join(args))
- if not options.over:
- parser.error(
- 'Specify the number of seconds, e.g. --over 86400 for the last 24 '
- 'hours')
- builders = options.builders or buildbot.builders.keys
- counts = {}
- since = time.time() - options.over
- for builder in builders:
- builder = buildbot.builders[builder]
- counts[builder.name] = 0
- if not options.quiet:
- print(builder.name)
- for build in builder.builds.iterall():
- try:
- start_time = build.start_time
- except urllib.error.HTTPError:
- # The build was probably trimmed.
- print(
- 'Failed to fetch build %s/%d' % (builder.name, build.number),
- file=sys.stderr)
- continue
- if start_time >= since:
- counts[builder.name] += 1
- else:
- break
- if not options.quiet:
- print('.. %d' % counts[builder.name])
-
- align_name = max(len(b) for b in counts)
- align_number = max(len(str(c)) for c in counts.values())
- for builder in sorted(counts):
- print('%*s: %*d' % (align_name, builder, align_number, counts[builder]))
- print('Total: %d' % sum(counts.values()))
- return 0
-
-
-def gen_parser():
- """Returns an OptionParser instance with default options.
-
- It should be then processed with gen_usage() before being used.
- """
- parser = optparse.OptionParser(version=__version__)
- # Remove description formatting
- parser.format_description = lambda x: parser.description
- # Add common parsing.
- old_parser_args = parser.parse_args
-
- def Parse(*args, **kwargs):
- options, args = old_parser_args(*args, **kwargs)
- if options.verbose >= 2:
- logging.basicConfig(level=logging.DEBUG)
- elif options.verbose:
- logging.basicConfig(level=logging.INFO)
- else:
- logging.basicConfig(level=logging.WARNING)
- return options, args
-
- parser.parse_args = Parse
-
- parser.add_option(
- '-v',
- '--verbose',
- action='count',
- help='Use multiple times to increase logging leve')
- parser.add_option(
- '-q',
- '--quiet',
- action='store_true',
- help='Reduces the output to be parsed by scripts, independent of -v')
- parser.add_option(
- '--throttle',
- type='float',
- help='Minimum delay to sleep between requests')
- return parser
-
-
-# Generic subcommand handling code
-
-
-def Command(name):
- return getattr(sys.modules[__name__], 'CMD' + name, None)
-
-
-@usage('<command>')
-def CMDhelp(parser, args):
- """Print list of commands or use 'help <command>'."""
- _, args = parser.parse_args(args)
- if len(args) == 1:
- return main(args + ['--help'])
- parser.print_help()
- return 0
-
-
-def gen_usage(parser, command):
- """Modifies an OptionParser object with the command's documentation.
-
- The documentation is taken from the function's docstring.
- """
- obj = Command(command)
- more = getattr(obj, 'func_usage_more')
- # OptParser.description prefer nicely non-formatted strings.
- parser.description = obj.__doc__ + '\n'
- parser.set_usage('usage: %%prog %s %s' % (command, more))
-
-
-def main(args=None):
- # Do it late so all commands are listed.
- # pylint: disable=E1101
- CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join(
- ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0])
- for fn in dir(sys.modules[__name__])
- if fn.startswith('CMD'))
-
- parser = gen_parser()
- if args is None:
- args = sys.argv[1:]
- if args:
- command = Command(args[0])
- if command:
- # "fix" the usage and the description now that we know the subcommand.
- gen_usage(parser, args[0])
- return command(parser, args[1:])
-
- # Not a known command. Default to help.
- gen_usage(parser, 'help')
- return CMDhelp(parser, args)
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/cros_utils/buildbot_utils_unittest.py b/cros_utils/buildbot_utils_unittest.py
index afbdfae..c615c95 100755
--- a/cros_utils/buildbot_utils_unittest.py
+++ b/cros_utils/buildbot_utils_unittest.py
@@ -21,18 +21,6 @@
class TrybotTest(unittest.TestCase):
"""Test for CommandExecuter class."""
- old_tryjob_out = (
- 'Verifying patches...\n'
- 'Submitting tryjob...\n'
- 'Successfully sent PUT request to [buildbucket_bucket:master.chromiumos.t'
- 'ryserver] with [config:success-build] [buildbucket_id:895272114382368817'
- '6].\n'
- 'Tryjob submitted!\n'
- 'To view your tryjobs, visit:\n'
- ' http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbuck'
- 'etId=8952721143823688176\n'
- ' https://uberchromegw.corp.google.com/i/chromiumos.tryserver/waterfall?'
- 'committer=laszio@chromium.org&builder=etc\n')
tryjob_out = (
'[{"buildbucket_id": "8952721143823688176", "build_config": '
'"cave-llvm-toolchain-tryjob", "url": '
