| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Synchronize issues in Package Status spreadsheet with Issue Tracker.""" |
| |
| from __future__ import print_function |
| |
| import optparse |
| import os |
| import sys |
| |
| from chromite.lib import cros_build_lib |
| from chromite.lib import gdata_lib |
| from chromite.lib import operation |
| from chromite.lib import upgrade_table as utable |
| from chromite.scripts import upload_package_status as ups |
| |
| |
| TRACKER_PROJECT_NAME = 'chromium' |
| |
| PKGS_WS_NAME = 'Packages' |
| |
| CR_ORG = 'chromium.org' |
| CHROMIUMOS_SITE = 'http://www.%s/chromium-os' % CR_ORG |
| PKG_UPGRADE_PAGE = '%s/gentoo-package-upgrade-process' % CHROMIUMOS_SITE |
| DOCS_SITE = 'https://docs.google.com/a' |
| |
| COL_PACKAGE = gdata_lib.PrepColNameForSS(utable.UpgradeTable.COL_PACKAGE) |
| COL_TEAM = gdata_lib.PrepColNameForSS('Team/Lead') |
| COL_OWNER = gdata_lib.PrepColNameForSS('Owner') |
| COL_TRACKER = gdata_lib.PrepColNameForSS('Tracker') |
| |
| ARCHES = ('amd64', 'arm', 'x86') |
| |
| DISABLING_TRACKER_VALUES = set(['n/a', 'disable', 'disabled']) |
| |
| oper = operation.Operation('sync_package_status') |
| |
| |
| def _GetPkgSpreadsheetURL(ss_key): |
| return '%s/%s/spreadsheet/ccc?key=%s' % (DOCS_SITE, CR_ORG, ss_key) |
| |
| |
| class SyncError(RuntimeError): |
| """Extend RuntimeError for use in this module.""" |
| |
| |
| class PackageBlacklisted(RuntimeError): |
| """Raised when package has disabled automatic tracker syncing.""" |
| |
| |
| class Syncer(object): |
| """Class to manage synchronizing between spreadsheet and Tracker.""" |
| |
| # Map spreadsheet team names to Tracker team labels. |
| VALID_TEAMS = {'build': 'Build', |
| 'kernel': 'Cr-OS-Kernel', |
| 'security': 'Security', |
| 'system': 'Cr-OS-Systems', |
| 'ui': 'Cr-UI'} |
| UPGRADE_STATES = set( |
| [utable.UpgradeTable.STATE_NEEDS_UPGRADE, |
| utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED, |
| utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_DUPLICATED]) |
| |
| __slots__ = ( |
| 'default_owner', # Default owner to use when creating issues |
| 'owners', # Set of owners to select (None means no filter) |
| 'pretend', # If True, make no real changes |
| 'scomm', # SpreadsheetComm |
| 'tcomm', # TrackerComm |
| 'teams', # Set of teams to select (None means no filter) |
| 'tracker_col_ix', # Index of Tracker column in spreadsheet |
| 'verbose', # Verbose boolean |
| ) |
| |
| def __init__(self, tcomm, scomm, pretend=False, verbose=False): |
| self.tcomm = tcomm |
| self.scomm = scomm |
| |
| self.tracker_col_ix = None |
| |
| self.teams = None |
| self.owners = None |
| self.default_owner = None |
| |
| self.pretend = pretend |
| self.verbose = verbose |
| |
| def _ReduceTeamName(self, team): |
| """Translate |team| from spreadsheet/commandline name to short name. |
| |
| For example: build/bdavirro --> build, build --> build |
| """ |
| if team: |
| return team.lower().split('/')[0] |
| return None |
| |
| def SetTeamFilter(self, teamarg): |
| """Set team filter using colon-separated team names in |teamarg| |
| |
| Resulting filter in self.teams is set of "reduced" team names. |
| Examples: |
| 'build:system:ui' --> set(['build', 'system', 'ui']) |
| 'Build:system:UI' --> set(['build', 'system', 'ui']) |
| |
| If an invalid team name is given oper.Die is called with explanation. |
| """ |
| if teamarg: |
| teamlist = [] |
| for team in teamarg.split(':'): |
| t = self._ReduceTeamName(team) |
| if t in self.VALID_TEAMS: |
| teamlist.append(t) |
| else: |
| oper.Die('Invalid team name "%s". Choose from: %s' % |
| (team, ','.join(self.VALID_TEAMS.keys()))) |
| self.teams = set(teamlist) |
| else: |
| self.teams = None |
| |
| def _ReduceOwnerName(self, owner): |
| """Translate |owner| from spreadsheet/commandline name to short name. |
| |
| For example: joe@chromium.org -> joe, joe --> joe |
| """ |
| if owner: |
| return owner.lower().split('@')[0] |
| return None |
| |
| def SetOwnerFilter(self, ownerarg): |
| """Set owner filter using colon-separated owner names in |ownerarg|.""" |
| if ownerarg: |
| self.owners = set([self._ReduceOwnerName(o) for o in ownerarg.split(':')]) |
| else: |
| self.owners = None |
| |
| def SetDefaultOwner(self, default_owner): |
| """Use |default_owner| as isue owner when none set in spreadsheet.""" |
| if default_owner and default_owner == 'me': |
| self.default_owner = os.environ['USER'] |
| else: |
| self.default_owner = default_owner |
| |
| def _RowPassesFilters(self, row): |
| """Return true if |row| passes any team/owner filters.""" |
| if self.teams: |
| team = self._ReduceTeamName(row[COL_TEAM]) |
| if team not in self.teams: |
| return False |
| |
| if self.owners: |
| owner = self._ReduceOwnerName(row[COL_OWNER]) |
| if owner not in self.owners: |
| return False |
| |
| return True |
| |
| def Sync(self): |
| """Do synchronization between Spreadsheet and Tracker.""" |
| self.tracker_col_ix = self.scomm.GetColumnIndex(COL_TRACKER) |
| if None is self.tracker_col_ix: |
| raise SyncError('Unable to find "Tracker" column in spreadsheet') |
| |
| errors = [] |
| |
| # Go over each row in Spreadsheet. Row index starts at 2 |
| # because spreadsheet rows start at 1 and we don't want the header row. |
| rows = self.scomm.GetRows() |
| for rowIx, row in enumerate(rows, start=self.scomm.ROW_NUMBER_OFFSET): |
| if not self._RowPassesFilters(row): |
| oper.Info('\nSkipping row %d, pkg: %r (team=%s, owner=%s) ...' % |
| (rowIx, row[COL_PACKAGE], row[COL_TEAM], row[COL_OWNER])) |
| continue |
| |
| oper.Info('\nProcessing row %d, pkg: %r (team=%s, owner=%s) ...' % |
| (rowIx, row[COL_PACKAGE], row[COL_TEAM], row[COL_OWNER])) |
| |
| try: |
| new_issue = self._GenIssueForRow(row) |
| old_issue_id = self._GetRowTrackerId(row) |
| |
| if new_issue and not old_issue_id: |
| self._CreateRowIssue(rowIx, row, new_issue) |
| elif not new_issue and old_issue_id: |
| self._ClearRowIssue(rowIx, row) |
| else: |
| # Nothing to do for this package. |
| reason = 'already has issue' if old_issue_id else 'no upgrade needed' |
| oper.Notice('Nothing to do for row %d, package %r: %s.' % |
| (rowIx, row[COL_PACKAGE], reason)) |
| except PackageBlacklisted: |
| oper.Notice('Tracker sync disabled for row %d, package %r: skipped.' % |
| (rowIx, row[COL_PACKAGE])) |
| except SyncError: |
| errors.append('Error processing row %d, pkg: %r. See above.' % |
| (rowIx, row[COL_PACKAGE])) |
| |
| if errors: |
| raise SyncError('\n'.join(errors)) |
| |
| def _GetRowValue(self, row, colName, arch=None): |
| """Get value from |row| at |colName|, adjusted for |arch|""" |
| if arch: |
| colName = utable.UpgradeTable.GetColumnName(colName, arch=arch) |
| colName = gdata_lib.PrepColNameForSS(colName) |
| return row[colName] |
| |
| def _GenIssueForRow(self, row): |
| """Generate an Issue object for |row| if applicable""" |
| # Row needs an issue if it "needs upgrade" on any platform. |
| statuses = {} |
| needs_issue = False |
| for arch in ARCHES: |
| state = self._GetRowValue(row, utable.UpgradeTable.COL_STATE, arch) |
| statuses[arch] = state |
| if state in self.UPGRADE_STATES: |
| needs_issue = True |
| |
| if not needs_issue: |
| return None |
| |
| pkg = row[COL_PACKAGE] |
| team = self._ReduceTeamName(row[COL_TEAM]) |
| if not team: |
| oper.Error('Unable to create Issue for package "%s" because no ' |
| 'team value is specified.' % pkg) |
| raise SyncError() |
| |
| labels = ['Type-Bug', |
| 'OS-Chrome', |
| 'Cr-OS-Packages', |
| 'Pri-2', |
| self.VALID_TEAMS[team]] |
| |
| owner = self._ReduceOwnerName(row[COL_OWNER]) |
| status = 'Untriaged' |
| if owner: |
| owner = owner + '@chromium.org' |
| status = 'Available' |
| elif self.default_owner: |
| owner = self.default_owner + '@chromium.org.' |
| else: |
| owner = None # Rather than '' |
| |
| title = '%s package needs upgrade from upstream Portage' % pkg |
| |
| lines = ['The %s package can be upgraded from upstream Portage' % pkg, |
| '', |
| 'At this moment the status on each arch is as follows:'] |
| |
| for arch in sorted(statuses): |
| arch_status = statuses[arch] |
| if arch_status: |
| # Get all versions for this arch right now. |
| curr_ver_col = utable.UpgradeTable.COL_CURRENT_VER |
| curr_ver = self._GetRowValue(row, curr_ver_col, arch) |
| stable_upst_ver_col = utable.UpgradeTable.COL_STABLE_UPSTREAM_VER |
| stable_upst_ver = self._GetRowValue(row, stable_upst_ver_col, arch) |
| latest_upst_ver_col = utable.UpgradeTable.COL_LATEST_UPSTREAM_VER |
| latest_upst_ver = self._GetRowValue(row, latest_upst_ver_col, arch) |
| |
| arch_vers = ['Current version: %s' % curr_ver, |
| 'Stable upstream version: %s' % stable_upst_ver, |
| 'Latest upstream version: %s' % latest_upst_ver] |
| lines.append(' On %s: %s' % (arch, arch_status)) |
| lines.append(' %s' % ', '.join(arch_vers)) |
| else: |
| lines.append(' On %s: not used' % arch) |
| |
| lines.append('') |
| lines.append('Check the latest status for this package, including ' |
| 'which upstream versions are available, at:\n %s' % |
| _GetPkgSpreadsheetURL(self.scomm.ss_key)) |
| lines.append('For help upgrading see: %s' % PKG_UPGRADE_PAGE) |
| |
| summary = '\n'.join(lines) |
| |
| issue = gdata_lib.Issue(title=title, |
| summary=summary, |
| status=status, |
| owner=owner, |
| labels=labels,) |
| return issue |
| |
| def _GetRowTrackerId(self, row): |
| """Get the tracker issue id in |row| if it exists, return None otherwise. |
| |
| Raises: |
| PackageBlacklisted if package has Tracker column value to disable syncing. |
| """ |
| tracker_val = row[COL_TRACKER] |
| if tracker_val: |
| try: |
| return int(tracker_val) |
| except ValueError: |
| # See if the unexpected value is one that disables tracker syncing. |
| if tracker_val.replace(' ', '').lower() in DISABLING_TRACKER_VALUES: |
| raise PackageBlacklisted() |
| |
| raise |
| |
| return None |
| |
| def _CreateRowIssue(self, rowIx, row, issue): |
| """Create a Tracker issue for |issue|, insert into |row| at |rowIx|""" |
| |
| pkg = row[COL_PACKAGE] |
| if not self.pretend: |
| oper.Info('Creating Tracker issue for package %s with details:\n%s' % |
| (pkg, issue)) |
| |
| # Before actually creating the Tracker issue, confirm that writing |
| # to this spreadsheet row is going to work. |
| try: |
| self.scomm.ClearCellValue(rowIx, self.tracker_col_ix) |
| except gdata_lib.SpreadsheetError as ex: |
| oper.Error('Unable to write to row %d, package %r. Aborting issue' |
| ' creation. Error was:\n%s' % (rowIx, pkg, ex)) |
| raise SyncError |
| |
| try: |
| issue_id = self.tcomm.CreateTrackerIssue(issue) |
| except gdata_lib.TrackerInvalidUserError as ex: |
| oper.Warning('%s. Ignoring owner field for issue %d, package %r.' % |
| (ex, rowIx, pkg)) |
| issue.summary += ('\n\nNote that the row for this package in' |
| ' the spreadsheet at go/crospkgs has an "owner"\n' |
| 'value that is not a valid Tracker user: "%s".' % |
| issue.owner) |
| issue.owner = None |
| issue_id = self.tcomm.CreateTrackerIssue(issue) |
| |
| oper.Info('Inserting new Tracker issue %d for package %s' % |
| (issue_id, pkg)) |
| ss_issue_val = self._GenSSLinkToIssue(issue_id) |
| |
| # This really should not fail since write access was checked before. |
| try: |
| self.scomm.ReplaceCellValue(rowIx, self.tracker_col_ix, ss_issue_val) |
| oper.Notice('Created Tracker issue %d for row %d, package %r' % |
| (issue_id, rowIx, pkg)) |
| except gdata_lib.SpreadsheetError as ex: |
| oper.Error('Failed to write link to new issue %d into' |
| ' row %d, package %r:\n%s' % |
| (issue_id, rowIx, pkg, ex)) |
| oper.Error('This means that the spreadsheet will have no record of' |
| ' this Tracker Issue and will create one again next time' |
| ' unless the spreadsheet is edited by hand!') |
| raise SyncError |
| |
| else: |
| oper.Notice('Would create and insert issue for row %d, package %r' % |
| (rowIx, pkg)) |
| oper.Info('Issue would be as follows:\n%s' % issue) |
| |
| def _GenSSLinkToIssue(self, issue_id): |
| """Create the spreadsheet hyperlink format for |issue_id|""" |
| return '=hyperlink("crbug.com/%d";"%d")' % (issue_id, issue_id) |
| |
| def _ClearRowIssue(self, rowIx, row): |
| """Clear the Tracker cell for row at |rowIx|""" |
| |
| pkg = row[COL_PACKAGE] |
| if not self.pretend: |
| try: |
| self.scomm.ClearCellValue(rowIx, self.tracker_col_ix) |
| oper.Notice('Cleared Tracker issue from row %d, package %r' % |
| (rowIx, pkg)) |
| except gdata_lib.SpreadsheetError as ex: |
| oper.Error('Error while clearing Tracker issue for' |
| ' row %d, package %r:\n%s' % (rowIx, pkg, ex)) |
| raise SyncError |
| |
| else: |
| oper.Notice('Would clear Tracker issue from row %d, package %r' % |
| (rowIx, pkg)) |
| |
| |
| def PrepareCreds(cred_file, token_file, email): |
| """Return a Creds object from given credentials. |
| |
| If |email| is given, the Creds object will contain that |email| |
| and a password entered at a prompt. |
| |
| Otherwise, if |token_file| is given then the Creds object will have |
| the auth_token from that file. |
| |
| Otherwise, if |cred_file| is given then the Creds object will have |
| the email/password from that file. |
| """ |
| |
| creds = gdata_lib.Creds() |
| |
| if email: |
| creds.SetCreds(email) |
| elif token_file and os.path.exists(token_file): |
| creds.LoadAuthToken(token_file) |
| elif cred_file and os.path.exists(cred_file): |
| creds.LoadCreds(cred_file) |
| |
| return creds |
| |
| |
| def _CreateOptParser(): |
| """Create the optparser.parser object for command-line args.""" |
| usage = 'Usage: %prog [options]' |
| epilog = (""" |
| Use this script to synchronize tracker issues between the package status |
| spreadsheet and the chromium-os Tracker. It uses the "Tracker" column of the |
| package spreadsheet. If a package needs an upgrade and has no tracker issue in |
| that column then a tracker issue is created. If it does not need an upgrade |
| then that column is cleared. |
| |
| Credentials must be specified using --auth-token-file, --cred-file or --email. |
| The first two have default values which you can rely on if valid, the latter |
| will prompt for your password. If you specify --email you will be given a |
| chance to save your email/password out as a credentials file for next time. |
| |
| Uses spreadsheet key %(ss_key)s, worksheet "%(ws_name)s". |
| (if --test-spreadsheet is set then spreadsheet |
| %(test_ss_key)s is used). |
| |
| Use the --team and --owner options to operate only on packages assigned to |
| particular owners or teams. Generally, running without a team or owner filter |
| is not intended, so use --team=all and/or --owner=all. |
| |
| Issues will be assigned to the owner of the package in the spreadsheet, if |
| available. If not, the owner defaults to value given to --default-owner. |
| |
| The --owner and --default-owner options accept "me" as an argument, which is |
| only useful if your username matches your chromium account name. |
| |
| """ % |
| {'ss_key': ups.REAL_SS_KEY, 'ws_name': PKGS_WS_NAME, |
| 'test_ss_key': ups.TEST_SS_KEY}) |
| |
| class MyOptParser(optparse.OptionParser): |
| """Override default epilog formatter, which strips newlines.""" |
| |
| def format_epilog(self, formatter): |
| return self.epilog |
| |
| teamhelp = '[%s]' % ', '.join(Syncer.VALID_TEAMS.keys()) |
| |
| parser = MyOptParser(usage=usage, epilog=epilog) |
| parser.add_option('--auth-token-file', dest='token_file', type='string', |
| action='store', default=gdata_lib.TOKEN_FILE, |
| help='File for reading/writing Docs auth token.' |
| ' [default: "%default"]') |
| parser.add_option('--cred-file', dest='cred_file', type='string', |
| action='store', default=gdata_lib.CRED_FILE, |
| help='Path to gdata credentials file [default: "%default"]') |
| parser.add_option('--email', dest='email', type='string', |
| action='store', default=None, |
| help='Email for Google Doc/Tracker user') |
| parser.add_option('--pretend', dest='pretend', action='store_true', |
| default=False, |
| help='Do not make any actual changes.') |
| parser.add_option('--team', dest='team', type='string', action='store', |
| default=None, |
| help='Filter by team; colon-separated %s' % teamhelp) |
| parser.add_option('--default-owner', dest='default_owner', |
| type='string', action='store', default=None, |
| help='Specify issue owner to use when package has no owner') |
| parser.add_option('--owner', dest='owner', type='string', action='store', |
| default=None, |
| help='Filter by package owner;' |
| ' colon-separated chromium.org accounts') |
| parser.add_option('--test-spreadsheet', dest='test_ss', |
| action='store_true', default=False, |
| help='Sync to the testing spreadsheet (implies --pretend).') |
| parser.add_option('--verbose', dest='verbose', action='store_true', |
| default=False, |
| help='Enable verbose output (for debugging)') |
| |
| return parser |
| |
| |
| def _CheckOptions(options): |
| """Vet the options.""" |
| me = os.environ['USER'] |
| |
| if not options.email and not os.path.exists(options.cred_file): |
| options.email = me |
| oper.Notice('Assuming your chromium email is %s@chromium.org.' |
| ' Override with --email.' % options.email) |
| |
| if not options.team and not options.owner: |
| oper.Notice('Without --owner or --team filters this will run for all' |
| ' packages in the spreadsheet (same as --team=all).') |
| if not cros_build_lib.BooleanPrompt( |
| 'Are you sure you want to run for all packages?', False): |
| sys.exit(0) |
| |
| if options.team and options.team == 'all': |
| options.team = None |
| |
| if options.owner and options.owner == 'all': |
| options.owner = None |
| |
| if options.owner and options.owner == 'me': |
| options.owner = me |
| oper.Notice('Using %r for owner filter (from $USER envvar)' % options.owner) |
| |
| if options.test_ss and not options.pretend: |
| oper.Notice('Running in --pretend mode because of --test-spreadsheet') |
| options.pretend = True |
| |
| |
| def main(argv): |
| """Main function.""" |
| parser = _CreateOptParser() |
| (options, _args) = parser.parse_args(argv) |
| |
| oper.verbose = options.verbose |
| |
| _CheckOptions(options) |
| |
| ss_key = ups.TEST_SS_KEY if options.test_ss else ups.REAL_SS_KEY |
| |
| # Prepare credentials for Docs and Tracker access. |
| creds = PrepareCreds(options.cred_file, options.token_file, options.email) |
| |
| scomm = gdata_lib.SpreadsheetComm() |
| scomm.Connect(creds, ss_key, PKGS_WS_NAME, source='Sync Package Status') |
| tcomm = gdata_lib.TrackerComm() |
| tcomm.Connect(creds, TRACKER_PROJECT_NAME, source='Sync Package Status') |
| |
| oper.Notice('Syncing between Tracker and spreadsheet %s' % ss_key) |
| syncer = Syncer(tcomm, scomm, |
| pretend=options.pretend, verbose=options.verbose) |
| |
| if options.team: |
| syncer.SetTeamFilter(options.team) |
| if options.owner: |
| syncer.SetOwnerFilter(options.owner) |
| if options.default_owner: |
| syncer.SetDefaultOwner(options.default_owner) |
| |
| try: |
| syncer.Sync() |
| except SyncError as ex: |
| oper.Die(str(ex)) |
| |
| # If --email, which is only effective when run interactively (because the |
| # password must be entered), give the option of saving to a creds file for |
| # next time. |
| if options.email and options.cred_file: |
| prompt = ('Do you want to save credentials for next time to %r?' % |
| options.cred_file) |
| if cros_build_lib.BooleanPrompt(prompt, False): |
| creds.StoreCreds(options.cred_file) |
| oper.Notice('Be sure to save the creds file to the same location' |
| ' outside your chroot so it will also be used with' |
| ' future chroots.') |
