| #!/usr/bin/python2 |
| # Copyright 2016 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """ |
| Automatically update the afe_stable_versions table. |
| |
| This command updates the stable repair version for selected boards |
| in the lab. For each board, if the version that Omaha is serving |
| on the Beta channel for the board is more recent than the current |
| stable version in the AFE database, then the AFE is updated to use |
| the version on Omaha. |
| |
| The upgrade process is applied to every "managed board" in the test |
| lab. Generally, a managed board is a board with both spare and |
| critical scheduling pools. |
| |
| See `autotest_lib.site_utils.lab_inventory` for the full definition |
| of "managed board". |
| |
| The command supports a `--dry-run` option that reports changes that |
| would be made, without making the actual RPC calls to change the |
| database. |
| |
| """ |
| |
| import argparse |
| import logging |
| |
| import common |
| from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
| from autotest_lib.site_utils import lab_inventory |
| from autotest_lib.site_utils import loglib |
| from autotest_lib.site_utils.stable_images import build_data |
| from chromite.lib import ts_mon_config |
| from chromite.lib import metrics |
| |
| |
| # _DEFAULT_BOARD - The distinguished board name used to identify a |
| # stable version mapping that is used for any board without an explicit |
| # mapping of its own. |
| # |
| # _DEFAULT_VERSION_TAG - A string used to signify that there is no |
| # mapping for a board, in other words, the board is mapped to the |
| # default version. |
| # |
| _DEFAULT_BOARD = 'DEFAULT' |
| _DEFAULT_VERSION_TAG = '(default)' |
| |
| _METRICS_PREFIX = 'chromeos/autotest/assign_stable_images' |
| |
| |
| class _VersionUpdater(object): |
| """ |
| Class to report and apply version changes. |
| |
| This class is responsible for the low-level logic of applying |
| version upgrades and reporting them as command output. |
| |
| This class exists to solve two problems: |
| 1. To distinguish "normal" vs. "dry-run" modes. Each mode has a |
| subclass; methods that perform actual AFE updates are |
| implemented for the normal mode subclass only. |
| 2. To provide hooks for unit tests. The unit tests override both |
| the reporting and modification behaviors, in order to test the |
| higher level logic that decides what changes are needed. |
| |
| Methods meant merely to report changes to command output have names |
| starting with "report" or "_report". Methods that are meant to |
| change the AFE in normal mode have names starting with "_do" |
| """ |
| |
| def __init__(self, afe, dry_run): |
| """Initialize us. |
| |
| @param afe: A frontend.AFE object. |
| @param dry_run: A boolean indicating whether to execute in dry run mode. |
| No updates are persisted to the afe in dry run. |
| """ |
| self._dry_run = dry_run |
| image_types = [afe.CROS_IMAGE_TYPE, afe.FIRMWARE_IMAGE_TYPE] |
| self._version_maps = { |
| image_type: afe.get_stable_version_map(image_type) |
| for image_type in image_types |
| } |
| self._cros_map = self._version_maps[afe.CROS_IMAGE_TYPE] |
| self._selected_map = None |
| |
| def select_version_map(self, image_type): |
| """ |
| Select an AFE version map object based on `image_type`. |
| |
| This creates and remembers an AFE version mapper object to be |
| used for making changes in normal mode. |
| |
| @param image_type Image type parameter for the version mapper |
| object. |
| """ |
| self._selected_map = self._version_maps[image_type] |
| return self._selected_map.get_all_versions() |
| |
| def report_default_changed(self, old_default, new_default): |
| """ |
| Report that the default version mapping is changing. |
| |
| This merely reports a text description of the pending change |
| without executing it. |
| |
| @param old_default The original default version. |
| @param new_default The new default version to be applied. |
| """ |
| logging.debug('Default %s -> %s', old_default, new_default) |
| |
| def _report_board_changed(self, board, old_version, new_version): |
| """ |
| Report a change in one board's assigned version mapping. |
| |
| This merely reports a text description of the pending change |
| without executing it. |
| |
| @param board The board with the changing version. |
| @param old_version The original version mapped to the board. |
| @param new_version The new version to be applied to the board. |
| """ |
| logging.debug(' %-22s %s -> %s', board, old_version, new_version) |
| |
| def report_board_unchanged(self, board, old_version): |
| """ |
| Report that a board's version mapping is unchanged. |
| |
| This reports that a board has a non-default mapping that will be |
| unchanged. |
| |
| @param board The board that is not changing. |
| @param old_version The board's version mapping. |
| """ |
| self._report_board_changed(board, '(no change)', old_version) |
| |
| def _do_set_mapping(self, board, new_version): |
| """ |
| Change one board's assigned version mapping. |
| |
| @param board The board with the changing version. |
| @param new_version The new version to be applied to the board. |
| """ |
| if self._dry_run: |
| logging.info('DRYRUN: Would have set %s version to %s', |
| board, new_version) |
| else: |
| self._selected_map.set_version(board, new_version) |
| |
| def _do_delete_mapping(self, board): |
| """ |
| Delete one board's assigned version mapping. |
| |
| @param board The board with the version to be deleted. |
| """ |
| if self._dry_run: |
| logging.info('DRYRUN: Would have deleted version for %s', board) |
| else: |
| self._selected_map.delete_version(board) |
| |
| def set_mapping(self, board, old_version, new_version): |
| """ |
| Change and report a board version mapping. |
| |
| @param board The board with the changing version. |
| @param old_version The original version mapped to the board. |
| @param new_version The new version to be applied to the board. |
| """ |
| self._report_board_changed(board, old_version, new_version) |
| self._do_set_mapping(board, new_version) |
| |
| def upgrade_default(self, new_default): |
| """ |
| Apply a default version change. |
| |
| @param new_default The new default version to be applied. |
| """ |
| self._do_set_mapping(_DEFAULT_BOARD, new_default) |
| |
| def delete_mapping(self, board, old_version): |
| """ |
| Delete a board version mapping, and report the change. |
| |
| @param board The board with the version to be deleted. |
| @param old_version The board's verson prior to deletion. |
| """ |
| assert board != _DEFAULT_BOARD |
| self._report_board_changed(board, |
| old_version, |
| _DEFAULT_VERSION_TAG) |
| self._do_delete_mapping(board) |
| |
| |
| def _get_upgrade_versions(cros_versions, omaha_versions, boards): |
| """ |
| Get the new stable versions to which we should update. |
| |
| The new versions are returned as a tuple of a dictionary mapping |
| board names to versions, plus a new default board setting. The |
| new default is determined as the most commonly used version |
| across the given boards. |
| |
| The new dictionary will have a mapping for every board in `boards`. |
| That mapping will be taken from `cros_versions`, unless the board has |
| a mapping in `omaha_versions` _and_ the omaha version is more recent |
| than the AFE version. |
| |
| @param cros_versions The current board->version mappings in the |
| AFE. |
| @param omaha_versions The current board->version mappings from |
| Omaha for the Beta channel. |
| @param boards Set of boards to be upgraded. |
| @return Tuple of (mapping, default) where mapping is a dictionary |
| mapping boards to versions, and default is a version string. |
| """ |
| upgrade_versions = {} |
| version_counts = {} |
| afe_default = cros_versions[_DEFAULT_BOARD] |
| for board in boards: |
| version = build_data.get_omaha_upgrade( |
| omaha_versions, board, |
| cros_versions.get(board, afe_default)) |
| upgrade_versions[board] = version |
| version_counts.setdefault(version, 0) |
| version_counts[version] += 1 |
| return (upgrade_versions, |
| max(version_counts.items(), key=lambda x: x[1])[0]) |
| |
| |
| def _get_firmware_upgrades(cros_versions): |
| """ |
| Get the new firmware versions to which we should update. |
| |
| @param cros_versions Current board->cros version mappings in the |
| AFE. |
| @return A dictionary mapping boards/models to firmware upgrade versions. |
| If the build is unibuild, the key is a model name; else, the key |
| is a board name. |
| """ |
| firmware_upgrades = {} |
| for board, version in cros_versions.iteritems(): |
| firmware_upgrades.update( |
| build_data.get_firmware_versions(board, version)) |
| return firmware_upgrades |
| |
| |
| def _apply_cros_upgrades(updater, old_versions, new_versions, |
| new_default): |
| """ |
| Change CrOS stable version mappings in the AFE. |
| |
| The input `old_versions` dictionary represents the content of the |
| `afe_stable_versions` database table; it contains mappings for a |
| default version, plus exceptions for boards with non-default |
| mappings. |
| |
| The `new_versions` dictionary contains a mapping for every board, |
| including boards that will be mapped to the new default version. |
| |
| This function applies the AFE changes necessary to produce the new |
| AFE mappings indicated by `new_versions` and `new_default`. The |
| changes are ordered so that at any moment, every board is mapped |
| either according to the old or the new mapping. |
| |
| @param updater Instance of _VersionUpdater responsible for |
| making the actual database changes. |
| @param old_versions The current board->version mappings in the |
| AFE. |
| @param new_versions New board->version mappings obtained by |
| applying Beta channel upgrades from Omaha. |
| @param new_default The new default build for the AFE. |
| """ |
| old_default = old_versions[_DEFAULT_BOARD] |
| if old_default != new_default: |
| updater.report_default_changed(old_default, new_default) |
| logging.info('Applying stable version changes:') |
| default_count = 0 |
| for board, new_build in new_versions.items(): |
| if new_build == new_default: |
| default_count += 1 |
| elif board in old_versions and new_build == old_versions[board]: |
| updater.report_board_unchanged(board, new_build) |
| else: |
| old_build = old_versions.get(board) |
| if old_build is None: |
| old_build = _DEFAULT_VERSION_TAG |
| updater.set_mapping(board, old_build, new_build) |
| if old_default != new_default: |
| updater.upgrade_default(new_default) |
| for board, new_build in new_versions.items(): |
| if new_build == new_default and board in old_versions: |
| updater.delete_mapping(board, old_versions[board]) |
| logging.info('%d boards now use the default mapping', default_count) |
| |
| |
| def _apply_firmware_upgrades(updater, old_versions, new_versions): |
| """ |
| Change firmware version mappings in the AFE. |
| |
| The input `old_versions` dictionary represents the content of the |
| firmware mappings in the `afe_stable_versions` database table. |
| There is no default version; missing boards simply have no current |
| version. |
| |
| This function applies the AFE changes necessary to produce the new |
| AFE mappings indicated by `new_versions`. |
| |
| TODO(jrbarnette) This function ought to remove any mapping not found |
| in `new_versions`. However, in theory, that's only needed to |
| account for boards that are removed from the lab, and that hasn't |
| happened yet. |
| |
| @param updater Instance of _VersionUpdater responsible for |
| making the actual database changes. |
| @param old_versions The current board->version mappings in the |
| AFE. |
| @param new_versions New board->version mappings obtained by |
| applying Beta channel upgrades from Omaha. |
| """ |
| unchanged = 0 |
| no_version = 0 |
| for board, new_firmware in new_versions.items(): |
| if new_firmware is None: |
| no_version += 1 |
| elif board not in old_versions: |
| updater.set_mapping(board, '(nothing)', new_firmware) |
| else: |
| old_firmware = old_versions[board] |
| if new_firmware != old_firmware: |
| updater.set_mapping(board, old_firmware, new_firmware) |
| else: |
| unchanged += 1 |
| logging.info('%d boards have no firmware mapping', no_version) |
| logging.info('%d boards are unchanged', unchanged) |
| |
| |
| def _assign_stable_images(arguments): |
| afe = frontend_wrappers.RetryingAFE(server=arguments.web) |
| updater = _VersionUpdater(afe, dry_run=arguments.dry_run) |
| |
| cros_versions = updater.select_version_map(afe.CROS_IMAGE_TYPE) |
| omaha_versions = build_data.get_omaha_version_map() |
| upgrade_versions, new_default = ( |
| _get_upgrade_versions(cros_versions, omaha_versions, |
| lab_inventory.get_managed_boards(afe))) |
| _apply_cros_upgrades(updater, cros_versions, |
| upgrade_versions, new_default) |
| |
| logging.info('Applying firmware updates.') |
| fw_versions = updater.select_version_map(afe.FIRMWARE_IMAGE_TYPE) |
| firmware_upgrades = _get_firmware_upgrades(upgrade_versions) |
| _apply_firmware_upgrades(updater, fw_versions, firmware_upgrades) |
| |
| |
| def main(): |
| """Standard main routine.""" |
| parser = argparse.ArgumentParser( |
| description='Update the stable repair version for all ' |
| 'boards') |
| parser.add_argument('-n', '--dry-run', |
| action='store_true', |
| help='print changes without executing them') |
| loglib.add_logging_options(parser) |
| # TODO(crbug/888046) Make these arguments required once puppet is updated to |
| # pass them in. |
| parser.add_argument('--web', |
| default='cautotest', |
| help='URL to the AFE to update.') |
| |
| arguments = parser.parse_args() |
| loglib.configure_logging_with_args(parser, arguments) |
| |
| tsmon_args = { |
| 'service_name': parser.prog, |
| 'indirect': False, |
| 'auto_flush': False, |
| } |
| if arguments.dry_run: |
| logging.info('DRYRUN: No changes will be made.') |
| # metrics will be logged to logging stream anyway. |
| tsmon_args['debug_file'] = '/dev/null' |
| |
| try: |
| with ts_mon_config.SetupTsMonGlobalState(**tsmon_args): |
| with metrics.SuccessCounter(_METRICS_PREFIX + '/tick', |
| fields={'afe': arguments.web}): |
| _assign_stable_images(arguments) |
| finally: |
| metrics.Flush() |
| |
| if __name__ == '__main__': |
| main() |