bluetooth: Add functionality to flag system failures in BT tests
Some failures, such as USB disconnects or bluetoothd crashes will cause
tests to fail, and require us to manually investigate the logs to
identify these root causes. This change wraps all QuickSanity tests in a
log checker, such that if a common failure is detected in system logs,
it is appended to the failure reason reported to stainless. This should
imrove our ability to correlate these root causes to the reported error
in an automated way.
BUG=b:170221594
TEST=QuickSanity and AdvSanity on Hatch
Change-Id: If0342598129752bf08de47dc6974092dd8afccbb
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/2454191
Reviewed-by: Shijin Abraham <shijinabraham@google.com>
Reviewed-by: Shyh-In Hwang <josephsih@chromium.org>
Commit-Queue: Daniel Winkler <danielwinkler@google.com>
Tested-by: Daniel Winkler <danielwinkler@google.com>
diff --git a/server/cros/bluetooth/bluetooth_adapter_tests.py b/server/cros/bluetooth/bluetooth_adapter_tests.py
index 8c2f692..340b613 100644
--- a/server/cros/bluetooth/bluetooth_adapter_tests.py
+++ b/server/cros/bluetooth/bluetooth_adapter_tests.py
@@ -96,6 +96,11 @@
'BLUETOOTH_AUDIO': lambda chameleon: chameleon.get_bluetooth_audio,
}
+COMMON_FAILURES = {
+ 'Freeing adapter /org/bluez/hci': 'adapter_freed',
+ '/var/spool/crash/bluetoothd': 'bluetoothd_crashed',
+}
+
# TODO(b/150898182) - Don't run some tests on tablet form factors
# This list was generated by looking for tablet models on Goldeneye and removing
# the ones that were not launched
@@ -386,6 +391,22 @@
return truthiness_of_result or result in legal_falsy_values
+def _flag_common_failures(instance):
+ """Checks if a common failure has occurred during the test run
+
+ Scans system logs for known signs of failure. If a failure is discovered,
+ it is added to the test results, to make it easier to identify common root
+ causes from Stainless
+ """
+
+ for fail_tag, fail_log in COMMON_FAILURES.items():
+ if instance.bluetooth_facade.messages_find(fail_tag):
+ logging.error('Detected failure tag: %s', fail_tag)
+ # We mark this instance's results with the discovered failure
+ if type(instance.results) is dict:
+ instance.results[fail_log] = True
+
+
def fix_serial_device(btpeer, device, operation='reset'):
"""Fix the serial device.
@@ -525,16 +546,22 @@
instance.last_test_method = test_method.__name__
try:
+ # Grab /var/log/messages output during test run
+ instance.bluetooth_facade.messages_start()
if callable(test_method_or_retry_flag
) or test_method_or_retry_flag:
test_result = retry(test_method, instance, *args, **kwargs)
else:
test_result = test_method(instance, *args, **kwargs)
+ syslog_captured = instance.bluetooth_facade.messages_stop()
+
if test_result:
logging.info('[*** passed: {}]'.format(
test_method.__name__))
else:
+ if syslog_captured:
+ _flag_common_failures(instance)
fail_msg = '[--- failed: {} ({})]'.format(
test_method.__name__, str(instance.results))
logging.error(fail_msg)
