blob: 7640312bf62ae342a173fac65c38c9ae4687768a [file] [log] [blame] [edit]
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""RDB utilities.
Do not import rdb or autotest modules here to avoid cyclic dependencies.
"""
import collections
import common
from autotest_lib.client.common_lib import priorities
from autotest_lib.client.common_lib import utils
try:
from chromite.lib import metrics
except ImportError:
metrics = utils.metrics_mock
RDB_STATS_KEY = 'rdb'
class RDBException(Exception):
"""Generic RDB exception."""
def wire_format(self, **kwargs):
"""Convert the exception to a format better suited to an rpc response.
"""
return str(self)
class CacheMiss(RDBException):
"""Generic exception raised for a cache miss in the rdb."""
pass
class LabelIterator(object):
"""An Iterator for labels.
Within the rdb any label/dependency comparisons are performed based on label
ids. However, the host object returned needs to contain label names instead.
This class returns label ids for iteration, but a list of all label names
when accessed through get_label_names.
"""
def __init__(self, labels):
self.labels = labels
def __iter__(self):
return iter(label.id for label in self.labels)
def get_label_names(self):
"""Get all label names of the labels associated with this class.
@return: A list of label names.
"""
return [label.name for label in self.labels]
class RequestAccountant(object):
"""A helper class that count requests and manages min_duts requirement.
On initialization, this object takes a list of host requests.
It will batch the requests by grouping similar requests together
and generate a mapping from unique request-> count of the request.
It will also generates a mapping from suite_job_id -> min_duts.
RDB does a two-round of host aquisition. The first round attempts
to get min_duts for each suite. The second round attemps to satisfy
the rest requests. RDB calls get_min_duts and get_rest to
figure out how many duts it should attempt to get for a unique
request in the first and second round respectively.
Assume we have two distinct requests
R1 (parent_job_id: 10, need hosts: 2)
R2 (parent_job_id: 10, need hosts: 4)
And parent job P (job_id:10) has min dut requirement of 3. So we got
requests_to_counts = {R1: 2, R2: 4}
min_duts_map = {P: 3}
First round acquiring:
Call get_min_duts(R1)
return 2, because P hasn't reach its min dut limit (3) yet
requests_to_counts -> {R1: 2-2=0, R2: 4}
min_duts_map -> {P: 3-2=1}
Call get_min_duts(R2)
return 1, because although R2 needs 4 duts, P's min dut limit is now 1
requests_to_counts -> {R1: 0, R2: 4-1=3}
min_duts_map -> {P: 1-1=0}
Second round acquiring:
Call get_rest(R1):
return 0, requests_to_counts[R1]
Call get_rest(R2):
return 3, requests_to_counts[R2]
Note it is possible that in the first round acquiring, although
R1 requested 2 duts, it may only get 1 or None. However get_rest
doesn't need to care whether the first round succeeded or not, as
in the case when the first round failed, regardless how many duts
get_rest requests, it will not be fullfilled anyway.
"""
_host_ratio_metric = metrics.Float(
'chromeos/autotest/scheduler/rdb/host_acquisition_ratio')
def __init__(self, host_requests):
"""Initialize.
@param host_requests: A list of request to acquire hosts.
"""
self.requests_to_counts = {}
# The order matters, it determines which request got fullfilled first.
self.requests = []
for request, count in self._batch_requests(host_requests):
self.requests.append(request)
self.requests_to_counts[request] = count
self.min_duts_map = dict(
(r.parent_job_id, r.suite_min_duts)
for r in self.requests_to_counts.iterkeys() if r.parent_job_id)
@classmethod
def _batch_requests(cls, requests):
""" Group similar requests, sort by priority and parent_job_id.
@param requests: A list or unsorted, unordered requests.
@return: A list of tuples of the form (request, number of occurances)
formed by counting the number of requests with the same acls/deps/
priority in the input list of requests, and sorting by priority.
The order of this list ensures against priority inversion.
"""
sort_function = lambda request: (request[0].priority,
-request[0].parent_job_id)
return sorted(collections.Counter(requests).items(), key=sort_function,
reverse=True)
def get_min_duts(self, host_request):
"""Given a distinct host request figure out min duts to request for.
@param host_request: A request.
@returns: The minimum duts that should be requested.
"""
parent_id = host_request.parent_job_id
count = self.requests_to_counts[host_request]
if parent_id:
min_duts = self.min_duts_map.get(parent_id, 0)
to_acquire = min(count, min_duts)
self.min_duts_map[parent_id] = max(0, min_duts - to_acquire)
else:
to_acquire = 0
self.requests_to_counts[host_request] -= to_acquire
return to_acquire
def get_duts(self, host_request):
"""Return the number of duts host_request still need.
@param host_request: A request.
@returns: The number of duts need to be requested.
"""
return self.requests_to_counts[host_request]
# TODO(akeshet): Possibly this code is dead, see crbug.com/738508 for
# context.
def record_acquire_min_duts(cls, host_request, hosts_required,
acquired_host_count):
"""Send stats about host acquisition.
@param host_request: A request.
@param hosts_required: Number of hosts required to satisfy request.
@param acquired_host_count: Number of host acquired.
"""
try:
priority = priorities.Priority.get_string(host_request.priority)
except ValueError:
return
cls._host_ratio_metric.set(acquired_host_count/float(hosts_required))