| # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """ |
| Repair actions and verifiers relating to CrOS firmware. |
| |
| This contains the repair actions and verifiers need to find problems |
| with the firmware installed on Chrome OS DUTs, and when necessary, to |
| fix problems by updating or re-installing the firmware. |
| |
| The operations in the module support two distinct use cases: |
| * DUTs used for FAFT tests can in some cases have problems with |
| corrupted firmware. The module supplies `FirmwareStatusVerifier` |
| to check for corruption, and supplies `FirmwareRepair` to re-install |
| firmware via servo when needed. |
| * DUTs used for general testing normally should be running a |
| designated "stable" firmware version. This module supplies |
| `FirmwareVersionVerifier` to detect and automatically update |
| firmware that is out-of-date from the designated version. |
| |
| For purposes of the operations in the module, we distinguish three kinds |
| of DUT, based on pool assignments: |
| * DUTs used for general testing. These DUTs automatically check for |
| and install the stable firmware using `FirmwareVersionVerifier`. |
| * DUTs in pools used for FAFT testing. These check for bad firmware |
| builds with `FirmwareStatusVerifier`, and will fix problems using |
| `FirmwareRepair`. These DUTs don't check for or install the |
| stable firmware. |
| * DUTs not in general pools, and not used for FAFT. These DUTs |
| are expected to be managed by separate processes and are excluded |
| from all of the verification and repair code in this module. |
| """ |
| |
| # pylint: disable=missing-docstring |
| |
| import logging |
| import re |
| |
| import common |
| from autotest_lib.client.common_lib import global_config |
| from autotest_lib.client.common_lib import hosts |
| from autotest_lib.server import afe_utils |
| from autotest_lib.server.hosts import repair_utils |
| |
| |
| # _FIRMWARE_REPAIR_POOLS - The set of pools that should be |
| # managed by `FirmwareStatusVerifier` and `FirmwareRepair`. |
| # |
| _FIRMWARE_REPAIR_POOLS = set( |
| global_config.global_config.get_config_value( |
| 'CROS', |
| 'pools_support_firmware_repair', |
| type=str).split(',')) |
| |
| |
| def _is_firmware_repair_supported(host): |
| """ |
| Check if a host supports firmware repair. |
| |
| When this function returns true, the DUT should be managed by |
| `FirmwareStatusVerifier` and `FirmwareRepair`, but not |
| `FirmwareVersionVerifier`. In general, this applies to DUTs |
| used for firmware testing. |
| |
| @return A true value if the host should use `FirmwareStatusVerifier` |
| and `FirmwareRepair`; a false value otherwise. |
| """ |
| info = host.host_info_store.get() |
| return bool(info.pools & _FIRMWARE_REPAIR_POOLS) |
| |
| |
| def _is_firmware_update_supported(host): |
| """ |
| Return whether a DUT should be running the standard firmware. |
| |
| In the test lab, DUTs used for general testing, (e.g. the `bvt` |
| pool) need their firmware kept up-to-date with |
| `FirmwareVersionVerifier`. However, some pools have alternative |
| policies for firmware management. This returns whether a given DUT |
| should be updated via the standard stable version update, or |
| managed by some other procedure. |
| |
| @param host The host to be checked for update policy. |
| @return A true value if the host should use |
| `FirmwareVersionVerifier`; a false value otherwise. |
| """ |
| return not _is_firmware_repair_supported(host) |
| |
| |
| def _get_firmware_version(output): |
| """Parse the output and get the firmware version. |
| |
| @param output The standard output of chromeos-firmwareupdate script. |
| @return Firmware version if found, else, None. |
| """ |
| # At one point, the chromeos-firmwareupdate script was updated to |
| # add "RW" version fields. The old string, "BIOS version:" still |
| # appears in the new output, however it now refers to the RO |
| # firmware version. Therefore, we try searching for the new string |
| # first, "BIOS (RW) version". If that string isn't found, we then |
| # fallback to searching for old string. |
| version = re.search(r'BIOS \(RW\) version:\s*(?P<version>.*)', output) |
| |
| if not version: |
| version = re.search(r'BIOS version:\s*(?P<version>.*)', output) |
| |
| if version is not None: |
| return version.group('version') |
| |
| return None |
| |
| |
| def _get_available_firmware(host, model): |
| """Get the available firmware version given the model. |
| |
| @param host The host to get available firmware for. |
| @param model The model name to get corresponding firmware version. |
| @return The available firmware version if found, else, None. |
| """ |
| result = host.run('chromeos-firmwareupdate -V', ignore_status=True) |
| |
| if result.exit_status == 0: |
| unibuild = False |
| paragraphs = result.stdout.split('\n\n') |
| for p in paragraphs: |
| match = re.search(r'Model:\s*(?P<model>.*)', p) |
| if match: |
| unibuild = True |
| if model == match.group('model'): |
| return _get_firmware_version(p) |
| |
| if not unibuild: |
| return _get_firmware_version(result.stdout) |
| |
| return None |
| |
| |
| class FirmwareStatusVerifier(hosts.Verifier): |
| """ |
| Verify that a host's firmware is in a good state. |
| |
| For DUTs that run firmware tests, it's possible that the firmware |
| on the DUT can get corrupted. This verifier checks whether it |
| appears that firmware should be re-flashed using servo. |
| """ |
| |
| def verify(self, host): |
| if not _is_firmware_repair_supported(host): |
| return |
| try: |
| # Read the AP firmware and dump the sections that we're |
| # interested in. |
| cmd = ('mkdir /tmp/verify_firmware; ' |
| 'cd /tmp/verify_firmware; ' |
| 'for section in VBLOCK_A VBLOCK_B FW_MAIN_A FW_MAIN_B; ' |
| 'do flashrom -p host -r -i $section:$section; ' |
| 'done') |
| host.run(cmd) |
| |
| # Verify the firmware blocks A and B. |
| cmd = ('vbutil_firmware --verify /tmp/verify_firmware/VBLOCK_%c' |
| ' --signpubkey /usr/share/vboot/devkeys/root_key.vbpubk' |
| ' --fv /tmp/verify_firmware/FW_MAIN_%c') |
| for c in ('A', 'B'): |
| rv = host.run(cmd % (c, c), ignore_status=True) |
| if rv.exit_status: |
| raise hosts.AutoservVerifyError( |
| 'Firmware %c is in a bad state.' % c) |
| finally: |
| # Remove the temporary files. |
| host.run('rm -rf /tmp/verify_firmware') |
| |
| @property |
| def description(self): |
| return 'Firmware on this DUT is clean' |
| |
| |
| class FirmwareRepair(hosts.RepairAction): |
| """ |
| Reinstall the firmware image using servo. |
| |
| This repair function attempts to use servo to install the DUT's |
| designated "stable firmware version". |
| |
| This repair method only applies to DUTs used for FAFT. |
| """ |
| |
| def repair(self, host): |
| if not _is_firmware_repair_supported(host): |
| raise hosts.AutoservRepairError( |
| 'Firmware repair is not applicable to host %s.' % |
| host.hostname, 'not_applicable') |
| repair_utils.require_servo(host) |
| host.firmware_install() |
| |
| @property |
| def description(self): |
| return 'Re-install the stable firmware via servo' |
| |
| |
| class FirmwareVersionVerifier(hosts.Verifier): |
| """ |
| Check for a firmware update, and apply it if appropriate. |
| |
| This verifier checks to ensure that either the firmware on the DUT |
| is up-to-date, or that the target firmware can be installed from the |
| currently running build. |
| |
| Failure occurs when all of the following apply: |
| 1. The DUT is not excluded from updates. For example, DUTs used |
| for FAFT testing use `FirmwareRepair` instead. |
| 2. The DUT's board has an assigned stable firmware version. |
| 3. The DUT is not running the assigned stable firmware. |
| 4. The firmware supplied in the running OS build is not the |
| assigned stable firmware. |
| |
| If the DUT needs an upgrade and the currently running OS build |
| supplies the necessary firmware, the verifier installs the new |
| firmware using `chromeos-firmwareupdate`. Failure to install will |
| cause the verifier to fail. |
| |
| This verifier nominally breaks the rule that "verifiers must succeed |
| quickly", since it can invoke `reboot()` during the success code |
| path. We're doing it anyway for two reasons: |
| * The time between updates will typically be measured in months, |
| so the amortized cost is low. |
| * The reason we distinguish repair from verify is to allow |
| rescheduling work immediately while the expensive repair happens |
| out-of-band. But a firmware update will likely hit all DUTs at |
| once, so it's pointless to pass the buck to repair. |
| |
| N.B. This verifier is a trigger for all repair actions that install |
| the stable repair image. If the firmware is out-of-date, but the |
| stable repair image does *not* contain the proper firmware version, |
| _the target DUT will fail repair, and will be unable to fix itself_. |
| """ |
| |
| @staticmethod |
| def _get_rw_firmware(host): |
| result = host.run('crossystem fwid', ignore_status=True) |
| if result.exit_status == 0: |
| return result.stdout |
| else: |
| return None |
| |
| @staticmethod |
| def _check_hardware_match(version_a, version_b): |
| """ |
| Check that two firmware versions identify the same hardware. |
| |
| Firmware version strings look like this: |
| Google_Gnawty.5216.239.34 |
| The part before the numbers identifies the hardware for which |
| the firmware was built. This function checks that the hardware |
| identified by `version_a` and `version_b` is the same. |
| |
| This is a sanity check to protect us from installing the wrong |
| firmware on a DUT when a board label has somehow gone astray. |
| |
| @param version_a First firmware version for the comparison. |
| @param version_b Second firmware version for the comparison. |
| """ |
| hardware_a = version_a.split('.')[0] |
| hardware_b = version_b.split('.')[0] |
| if hardware_a != hardware_b: |
| message = 'Hardware/Firmware mismatch updating %s to %s' |
| raise hosts.AutoservVerifyError( |
| message % (version_a, version_b)) |
| |
| def verify(self, host): |
| # Test 1 - The DUT is not excluded from updates. |
| if not _is_firmware_update_supported(host): |
| return |
| # Test 2 - The DUT has an assigned stable firmware version. |
| info = host.host_info_store.get() |
| if info.model is None: |
| raise hosts.AutoservVerifyError( |
| 'Can not verify firmware version. ' |
| 'No model label value found') |
| |
| stable_firmware = None |
| try: |
| stable_firmware = afe_utils.get_stable_firmware_version(info.model) |
| except Exception as e: |
| logging.exception('Failed lookup to AFE for stable fw version ' |
| ' with exception: %s', e) |
| |
| if stable_firmware is None: |
| # This DUT doesn't have a firmware update target |
| return |
| |
| # For tests 3 and 4: If the output from `crossystem` or |
| # `chromeos-firmwareupdate` isn't what we expect, we log an |
| # error, but don't fail: We don't want DUTs unable to test a |
| # build merely because of a bug or change in either of those |
| # commands. |
| |
| # Test 3 - The DUT is not running the target stable firmware. |
| current_firmware = self._get_rw_firmware(host) |
| if current_firmware is None: |
| logging.error('DUT firmware version can\'t be determined.') |
| return |
| if current_firmware == stable_firmware: |
| return |
| # Test 4 - The firmware supplied in the running OS build is not |
| # the assigned stable firmware. |
| available_firmware = _get_available_firmware(host, info.model) |
| if available_firmware is None: |
| logging.error('Supplied firmware version in OS can\'t be ' |
| 'determined.') |
| return |
| if available_firmware != stable_firmware: |
| raise hosts.AutoservVerifyError( |
| 'DUT firmware requires update from %s to %s' % |
| (current_firmware, stable_firmware)) |
| # Time to update the firmware. |
| logging.info('Updating firmware from %s to %s', |
| current_firmware, stable_firmware) |
| self._check_hardware_match(current_firmware, stable_firmware) |
| try: |
| host.run('chromeos-firmwareupdate --mode=autoupdate') |
| host.reboot() |
| except Exception as e: |
| message = ('chromeos-firmwareupdate failed: from ' |
| '%s to %s') |
| logging.exception(message, current_firmware, stable_firmware) |
| raise hosts.AutoservVerifyError( |
| message % (current_firmware, stable_firmware)) |
| final_firmware = self._get_rw_firmware(host) |
| if final_firmware != stable_firmware: |
| message = ('chromeos-firmwareupdate failed: tried upgrade ' |
| 'to %s, now running %s instead') |
| raise hosts.AutoservVerifyError( |
| message % (stable_firmware, final_firmware)) |
| |
| @property |
| def description(self): |
| return 'The firmware on this DUT is up-to-date' |
