fetch partial builder status from the cq master
This CL causes the CQ master to fetch not only the final build status,
but also the partial 'unittest' statuses.
BUG=chromium:310668
TEST=Unit tests pass.
Change-Id: I171e62d23a3bed9cec54b7d69659dd4effa7d0d6
Reviewed-on: https://chromium-review.googlesource.com/174433
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
Tested-by: Aviv Keshet <akeshet@chromium.org>
Commit-Queue: Aviv Keshet <akeshet@chromium.org>
diff --git a/buildbot/cbuildbot_stages.py b/buildbot/cbuildbot_stages.py
index 1ac99f0..6a63cc1 100644
--- a/buildbot/cbuildbot_stages.py
+++ b/buildbot/cbuildbot_stages.py
@@ -1212,7 +1212,7 @@
super(LKGMCandidateSyncCompletionStage, self).__init__(*args, **kwargs)
self._slave_statuses = {}
- def _FetchSlaveStatuses(self):
+ def _FetchSlaveStatuses(self, step_name=None, wait_for_status=True):
"""Fetch and return build status for this build and any of its slaves."""
if self._options.debug:
# In debug mode, nothing is uploaded to Google Storage, so we bypass
@@ -1223,18 +1223,23 @@
elif not self._build_config['master']:
# Slaves only need to look at their own status.
return ManifestVersionedSyncStage.manifest_manager.GetBuildersStatus(
- [self._bot_id])
+ [self._bot_id], step_name=step_name,
+ wait_for_status=wait_for_status)
else:
builders = self._GetSlavesForMaster(self._build_config)
manager = ManifestVersionedSyncStage.manifest_manager
sub_manager = LKGMCandidateSyncStage.sub_manager
if sub_manager:
public_builders = [b['name'] for b in builders if not b['internal']]
- statuses = sub_manager.GetBuildersStatus(public_builders)
+ statuses = sub_manager.GetBuildersStatus(public_builders,
+ step_name=step_name, wait_for_status=wait_for_status)
private_builders = [b['name'] for b in builders if b['internal']]
- statuses.update(manager.GetBuildersStatus(private_builders))
+ statuses.update(manager.GetBuildersStatus(private_builders,
+ step_name=step_name, wait_for_status=wait_for_status))
else:
- statuses = manager.GetBuildersStatus([b['name'] for b in builders])
+ statuses = manager.GetBuildersStatus([b['name'] for b in builders],
+ step_name=step_name,
+ wait_for_status=wait_for_status)
return statuses
def _AbortCQHWTests(self):
@@ -1301,6 +1306,14 @@
self._AbortCQHWTests()
statuses = self._FetchSlaveStatuses()
+
+ # disable the unused-variable warning
+ # pylint: disable-msg=W0612
+ unittest_statuses = self._FetchSlaveStatuses(step_name='unittest',
+ wait_for_status=False)
+ # TODO: Handle the case where statuses shows full build failures, but
+ # unittest_statuses shows that all slave builder unit tests passed.
+ # In this case, submit and uprev the non-unit-test CLs only.
self._slave_statuses = statuses
failing_build_dict, inflight_build_dict = {}, {}
for builder, status in statuses.iteritems():
diff --git a/buildbot/lkgm_manager.py b/buildbot/lkgm_manager.py
index 064f9dd..dea8673 100755
--- a/buildbot/lkgm_manager.py
+++ b/buildbot/lkgm_manager.py
@@ -156,7 +156,11 @@
self.rel_working_dir = self.LKGM_SUBDIR
def _RunLambdaWithTimeout(self, function_to_run, use_long_timeout=False):
- """Runs function_to_run until it returns a value or timeout is reached."""
+ """Runs function_to_run until it returns a value or timeout is reached.
+
+ Returns:
+ The value returned by the final call to function_to_run.
+ """
function_success = False
start_time = time.time()
max_timeout = self.MAX_TIMEOUT_SECONDS
@@ -380,22 +384,32 @@
else:
return None
- def GetBuildersStatus(self, builders_array):
+ def GetBuildersStatus(self, builders_array, step_name=None,
+ wait_for_status=True):
"""Returns a build-names->status dictionary of build statuses.
Args:
builders_array: A list of the names of the builders to check.
+ step_name: The step name to get a partial builder status for.
+ Defaults to None, in which case the full builder
+ status is fetched.
+ wait_for_status: Defaults to True, in which case the call will
+ wait until status files appear (with a default
+ timeout). If False, will not wait for files to
+ appear.
"""
builders_completed = set()
builder_statuses = {}
def _CheckStatusOfBuildersArray():
- """Helper function that iterates through current statuses."""
+ """Helper function that iterates through current statuses.
+ """
for b in builders_array:
cached_status = builder_statuses.get(b)
if not cached_status or not cached_status.Completed():
logging.debug("Checking for builder %s's status", b)
- builder_status = self.GetBuildStatus(b, self.current_version)
+ builder_status = self.GetBuildStatus(b, self.current_version,
+ step_name=step_name)
builder_statuses[b] = builder_status
if builder_status is None:
logging.warn('No status found for builder %s.', b)
@@ -414,10 +428,14 @@
return 'Builds completed.'
# Check for build completion until all builders report in.
- builds_succeeded = self._RunLambdaWithTimeout(_CheckStatusOfBuildersArray,
- use_long_timeout=True)
+ if wait_for_status:
+ builds_succeeded = self._RunLambdaWithTimeout(_CheckStatusOfBuildersArray,
+ use_long_timeout=True)
+ else:
+ builds_succeeded = _CheckStatusOfBuildersArray()
+
if not builds_succeeded:
- logging.error('Not all builds finished before MAX_TIMEOUT reached.')
+ logging.error('Not all build statuses available or MAX_TIMEOUT reached.')
return builder_statuses
diff --git a/buildbot/lkgm_manager_unittest.py b/buildbot/lkgm_manager_unittest.py
index 0cf69fe..eba376b 100755
--- a/buildbot/lkgm_manager_unittest.py
+++ b/buildbot/lkgm_manager_unittest.py
@@ -341,12 +341,13 @@
osutils.Touch(manifest)
return manifest, dir_pfx
- def _GetBuildersStatus(self, builders, status_runs):
+ def _GetBuildersStatus(self, builders, status_runs, step_name=None):
"""Test a call to LKGMManager.GetBuildersStatus.
Args:
builders: List of builders to get status for.
status_runs: List of expected (builder, status) tuples.
+ step_name: Builder step to test getting status for. Default: None
"""
self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'GetBuildStatus')
for builder, status in status_runs:
@@ -355,7 +356,7 @@
if status is not None:
status = manifest_version.BuilderStatus(status, None)
lkgm_manager.LKGMManager.GetBuildStatus(
- builder, mox.IgnoreArg()).AndReturn(status)
+ builder, mox.IgnoreArg(), step_name=step_name).AndReturn(status)
self.mox.ReplayAll()
statuses = self.manager.GetBuildersStatus(builders)
diff --git a/buildbot/manifest_version.py b/buildbot/manifest_version.py
index a66dbfb..9d85221 100644
--- a/buildbot/manifest_version.py
+++ b/buildbot/manifest_version.py
@@ -535,7 +535,8 @@
return self._latest_status and self._latest_status.Failed()
@staticmethod
- def GetBuildStatus(builder, version, retries=NUM_RETRIES):
+ def GetBuildStatus(builder, version, retries=NUM_RETRIES,
+ step_name=None):
"""Returns a BuilderStatus instance for the given the builder.
Args:
@@ -547,7 +548,7 @@
A BuilderStatus instance containing the builder status and any optional
message associated with the status passed by the builder.
"""
- url = BuildSpecsManager._GetStatusUrl(builder, version)
+ url = BuildSpecsManager._GetStatusUrl(builder, version, step_name)
ctx = gs.GSContext(retries=retries)
try:
output = ctx.Cat(url).output
