| # -*- coding: utf-8 -*- |
| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Merge multiple package status CSV files into one csv file. |
| |
| This simplifies uploading to a Google Docs spreadsheet. |
| """ |
| |
| from __future__ import print_function |
| |
| import os |
| import re |
| import sys |
| |
| from chromite.lib import commandline |
| from chromite.lib import operation |
| from chromite.lib import table |
| from chromite.lib import upgrade_table as utable |
| |
| |
| assert sys.version_info >= (3, 6), 'This module requires Python 3.6+' |
| |
| |
| COL_PACKAGE = utable.UpgradeTable.COL_PACKAGE |
| COL_SLOT = utable.UpgradeTable.COL_SLOT |
| COL_TARGET = utable.UpgradeTable.COL_TARGET |
| COL_OVERLAY = utable.UpgradeTable.COL_OVERLAY |
| ID_COLS = [COL_PACKAGE, COL_SLOT] |
| |
| oper = operation.Operation('merge_package_status') |
| |
| # A bit of hard-coding with knowledge of how cros targets work. |
| CHROMEOS_TARGET_ORDER = [ |
| 'virtual/target-os', |
| 'virtual/target-os-dev', |
| 'virtual/target-os-test', |
| ] |
| |
| |
| def _GetCrosTargetRank(target): |
| """Hard-coded ranking of known/expected OS root targets for sorting. |
| |
| The lower the ranking, the earlier in the target list it falls by |
| convention. In other words, in the typical target combination |
| "virtual/target-os virtual/target-os-dev", "virtual/target-os" has |
| a lower ranking than "virtual/target-os-dev". |
| |
| All valid rankings are greater than zero. |
| |
| Returns: |
| Valid ranking for target or a false value if target is unrecognized. |
| """ |
| for ix, targ in enumerate(CHROMEOS_TARGET_ORDER): |
| if target == targ: |
| return ix + 1 # Avoid a 0 (non-true) result |
| return None |
| |
| |
| def ProcessTargets(targets, reverse_cros=False): |
| """Process a list of |targets| to smaller, sorted list. |
| |
| For example: |
| virtual/target-os virtual/target-os-dev -> virtual/target-os-dev |
| virtual/target-os virtual/target-os-dev world -> virtual/target-os-dev world |
| world virtual/target-sdk -> virtual/target-sdk world |
| |
| The one virtual/target-os target always comes back first, with targets |
| otherwise sorted alphabetically. The virtual/target-os target that is |
| kept will be the one with the highest 'ranking', as decided |
| by _GetCrosTargetRank. To reverse the ranking sense, specify |
| |reverse_cros| as True. |
| |
| These rules are specific to how we want the information to appear |
| in the final spreadsheet. |
| """ |
| if targets: |
| # Sort cros targets according to "rank". |
| cros_targets = [t for t in targets if _GetCrosTargetRank(t)] |
| cros_targets.sort(key=_GetCrosTargetRank, reverse=reverse_cros) |
| |
| # Don't condense non-cros targets. |
| other_targets = [t for t in targets if not _GetCrosTargetRank(t)] |
| other_targets.sort() |
| |
| # Assemble final target list, with single cros target first. |
| final_targets = [] |
| if cros_targets: |
| final_targets.append(cros_targets[-1]) |
| if other_targets: |
| final_targets.extend(other_targets) |
| |
| return final_targets |
| |
| |
| def LoadTable(filepath): |
| """Load the csv file at |filepath| into a table.Table object.""" |
| table_name = os.path.basename(filepath) |
| if table_name.endswith('.csv'): |
| table_name = table_name[:-4] |
| return table.Table.LoadFromCSV(filepath, name=table_name) |
| |
| |
| def MergeTables(tables): |
| """Merge all |tables| into one merged table. Return table.""" |
| def TargetMerger(_col, val, other_val): |
| """Function to merge two values in Root Target column from two tables.""" |
| targets = [] |
| if val: |
| targets.extend(val.split()) |
| if other_val: |
| targets.extend(other_val.split()) |
| |
| processed_targets = ProcessTargets(targets, reverse_cros=True) |
| return ' '.join(processed_targets) |
| |
| def DefaultMerger(col, val, other_val): |
| """Merge |val| and |other_val| in column |col| for some row.""" |
| # This function is registered as the default merge function, |
| # so verify that the column is a supported one. |
| prfx = utable.UpgradeTable.COL_DEPENDS_ON.replace('ARCH', '') |
| if col.startswith(prfx): |
| # Merge dependencies by taking the superset. |
| return MergeToSuperset(col, val, other_val) |
| |
| prfx = utable.UpgradeTable.COL_USED_BY.replace('ARCH', '') |
| if col.startswith(prfx): |
| # Merge users by taking the superset. |
| return MergeToSuperset(col, val, other_val) |
| |
| regexp = utable.UpgradeTable.COL_UPGRADED.replace('ARCH', r'\S+') |
| if re.search(regexp, col): |
| return MergeWithAND(col, val, other_val) |
| |
| # For any column, if one value is missing just accept the other value. |
| # For example, when one table has an entry for 'arm version' but |
| # the other table does not. |
| if val == table.Table.EMPTY_CELL and other_val != table.Table.EMPTY_CELL: |
| return other_val |
| if other_val == table.Table.EMPTY_CELL and val != table.Table.EMPTY_CELL: |
| return val |
| |
| # Raise a generic ValueError, which MergeTable function will clarify. |
| # The effect should be the same as having no merge_rule for this column. |
| raise ValueError |
| |
| def MergeToSuperset(_col, val, other_val): |
| """Merge |col| values as superset of tokens in |val| and |other_val|.""" |
| tokens = set(val.split()) |
| other_tokens = set(other_val.split()) |
| all_tokens = tokens.union(other_tokens) |
| return ' '.join(sorted(tok for tok in all_tokens)) |
| |
| # This is only needed because the automake-wrapper package is coming from |
| # different overlays for different boards right now! |
| def MergeWithAND(_col, val, other_val): |
| """For merging columns that might have differences but should not!.""" |
| if not val: |
| return '"" AND ' + other_val |
| if not other_val + ' AND ""': |
| return val |
| return val + ' AND ' + other_val |
| |
| # Prepare merge_rules with the defined functions. |
| merge_rules = {COL_TARGET: TargetMerger, |
| COL_OVERLAY: MergeWithAND, |
| '__DEFAULT__': DefaultMerger} |
| |
| # Merge each table one by one. |
| csv_table = tables[0] |
| if len(tables) > 1: |
| oper.Notice('Merging tables into one.') |
| for tmp_table in tables[1:]: |
| oper.Notice('Merging "%s" and "%s".' % |
| (csv_table.GetName(), tmp_table.GetName())) |
| csv_table.MergeTable(tmp_table, ID_COLS, |
| merge_rules=merge_rules, allow_new_columns=True) |
| |
| # Sort the table by package name, then slot. |
| def IdSort(row): |
| return tuple(row[col] for col in ID_COLS) |
| csv_table.Sort(IdSort) |
| |
| return csv_table |
| |
| |
| def LoadAndMergeTables(args): |
| """Load all csv files in |args| into one merged table. Return table.""" |
| tables = [] |
| for arg in args: |
| oper.Notice('Loading csv table from "%s".' % arg) |
| tables.append(LoadTable(arg)) |
| |
| return MergeTables(tables) |
| |
| |
| # Used by upload_package_status. |
| def FinalizeTable(csv_table): |
| """Process the table to prepare it for upload to online spreadsheet.""" |
| oper.Notice('Processing final table to prepare it for upload.') |
| |
| col_ver = utable.UpgradeTable.COL_CURRENT_VER |
| col_arm_ver = utable.UpgradeTable.GetColumnName(col_ver, 'arm') |
| col_x86_ver = utable.UpgradeTable.GetColumnName(col_ver, 'x86') |
| |
| # Insert new columns |
| col_cros_target = 'ChromeOS Root Target' |
| col_host_target = 'Host Root Target' |
| col_cmp_arch = 'Comparing arm vs x86 Versions' |
| csv_table.AppendColumn(col_cros_target) |
| csv_table.AppendColumn(col_host_target) |
| csv_table.AppendColumn(col_cmp_arch) |
| |
| # Row by row processing |
| for row in csv_table: |
| # If the row is not unique when just the package |
| # name is considered, then add a ':<slot>' suffix to the package name. |
| id_values = {COL_PACKAGE: row[COL_PACKAGE]} |
| matching_rows = csv_table.GetRowsByValue(id_values) |
| if len(matching_rows) > 1: |
| for mr in matching_rows: |
| mr[COL_PACKAGE] += ':' + mr[COL_SLOT] |
| |
| # Split target column into cros_target and host_target columns |
| target_str = row.get(COL_TARGET, None) |
| if target_str: |
| targets = target_str.split() |
| cros_targets = [] |
| host_targets = [] |
| for target in targets: |
| if _GetCrosTargetRank(target): |
| cros_targets.append(target) |
| else: |
| host_targets.append(target) |
| |
| row[col_cros_target] = ' '.join(cros_targets) |
| row[col_host_target] = ' '.join(host_targets) |
| |
| # Compare x86 vs. arm version, add result to col_cmp_arch. |
| x86_ver = row.get(col_x86_ver) |
| arm_ver = row.get(col_arm_ver) |
| if x86_ver and arm_ver: |
| if x86_ver != arm_ver: |
| row[col_cmp_arch] = 'different' |
| else: |
| row[col_cmp_arch] = 'same' |
| |
| |
| def WriteTable(csv_table, outpath): |
| """Write |csv_table| out to |outpath| as csv.""" |
| try: |
| fh = open(outpath, 'w') |
| csv_table.WriteCSV(fh) |
| oper.Notice('Wrote merged table to "%s"' % outpath) |
| except IOError as ex: |
| oper.Error('Unable to open %s for write: %s' % (outpath, ex)) |
| raise |
| |
| |
| def GetParser(): |
| """Return a command line parser.""" |
| parser = commandline.ArgumentParser(description=__doc__) |
| parser.add_argument('--out', dest='outpath', type='path', |
| required=True, |
| help='File to write merged results to') |
| parser.add_argument('input_csv_files', nargs='+') |
| return parser |
| |
| |
| def main(argv): |
| """Main function.""" |
| parser = GetParser() |
| options = parser.parse_args(argv) |
| |
| csv_table = LoadAndMergeTables(options.input_csv_files) |
| |
| WriteTable(csv_table, options.outpath) |