#!/usr/bin/python2.6
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Synchronize issues in Package Status spreadsheet with Issue Tracker."""

import optparse
import os
import sys
import urllib
import xml.dom.minidom

import gdata.projecthosting.client
import gdata.spreadsheet.service

sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
import chromite.lib.cros_build_lib as cros_lib
import chromite.lib.gdata_lib as gdata_lib
import chromite.lib.operation as operation
import chromite.lib.upgrade_table as utable

# pylint: disable=W0201,R0904

PROJECT_NAME = 'chromium-os'

SS_KEY = '0AsXDKtaHikmcdEp1dVN1SG1yRU1xZEw1Yjhka2dCSUE'
PKGS_WS_NAME = 'Packages'

CROS_ORG = 'chromium.org'
CHROMIUMOS_SITE = 'http://www.%s/%s' % (CROS_ORG, PROJECT_NAME)
PKG_UPGRADE_PAGE = '%s/gentoo-package-upgrade-process' % CHROMIUMOS_SITE
DOCS_SITE = 'https://docs.google.com/a'
PKG_STATUS_PAGE = '%s/%s/spreadsheet/ccc?key=%s' % (DOCS_SITE, CROS_ORG, SS_KEY)

COL_PACKAGE = gdata_lib.PrepColNameForSS(utable.UpgradeTable.COL_PACKAGE)
COL_TEAM = gdata_lib.PrepColNameForSS('Team/Lead')
COL_OWNER = gdata_lib.PrepColNameForSS('Owner')
COL_TRACKER = gdata_lib.PrepColNameForSS('Tracker')

ARCHES = ('amd64', 'arm', 'x86')

oper = operation.Operation('sync_package_status')

class SyncError(RuntimeError):
  """Extend RuntimeError for use in this module."""

class IssueComment(object):
  """Represent a Tracker issue comment."""

  __slots__ = ['title', 'text']

  def __init__(self, title, text):
    self.title = title
    self.text = text

  def __str__(self):
    text = '\n  '.join(self.text.split('\n'))
    return '%s:\n  %s' % (self.title, text)

class Issue(object):
  """Represents one Tracker Issue."""

  SlotDefaults = {
    'comments': [], # List of IssueComment objects
    'id': 0,        # Issue id number (int)
    'labels': [],   # List of text labels
    'owner': None,  # Current owner (text, chromium.org account)
    'status': None, # Current issue status (text) (e.g. Assigned)
    'summary': None,# Issue summary (first comment)
    'title': None,  # Title text
    }

  __slots__ = SlotDefaults.keys()

  def __init__(self, tracker_issue=None, **kwargs):
    """Init for one Issue object.

    |tracker_issue| - Optional, Tracker issue object to use to
    initialize attributes.
    |kwargs| - key/value arguments to give initial values to
    any additional attributes on |self|.
    """
    # Start every attribute with a default value.
    for slot in self.__slots__:
      setattr(self, slot, self.SlotDefaults[slot])

    # Initialize from real Tracker issue if given.
    if tracker_issue:
      self.InitFromTracker(tracker_issue)

    # Optional overwrite of any attribute using kwargs.
    for slot in self.__slots__:
      if slot in kwargs:
        setattr(self, slot, kwargs[slot])

  def __str__(self):
    """Pretty print of issue."""
    lines = ['Issue %d - %s' % (self.id, self.title),
             'Status: %s, Owner: %s' % (self.status, self.owner),
             'Labels: %s' % ', '.join(self.labels),
             ]

    if self.summary:
      lines.append('Summary: %s' % self.summary)

    if self.comments:
      for comment in self.comments:
        lines.append('%s' % comment)

    return '\n'.join(lines)

  def InitFromTracker(self, t_issue):
    """Initialize |self| from tracker issue |t_issue|"""

    self.id = int(t_issue.id.text.split('/')[-1])
    self.labels = [label.text for label in t_issue.label]
    if t_issue.owner:
      self.owner = t_issue.owner.username.text
    self.status = t_issue.status.text
    self.summary = t_issue.content.text
    self.title = t_issue.title.text
    self.comments = self.GetTrackerIssueComments(self.id)

  def GetTrackerIssueComments(self, issue_id):
    """Retrieve comments for |issue_id| from comments URL"""
    comments = []

    feeds = 'http://code.google.com/feeds'
    url = '%s/issues/p/%s/issues/%d/comments/full' % (feeds, PROJECT_NAME,
                                                      issue_id)
    doc = xml.dom.minidom.parse(urllib.urlopen(url))
    entries = doc.getElementsByTagName('entry')
    for entry in entries:
      title = entry.getElementsByTagName('title')[0].firstChild.nodeValue
      text = entry.getElementsByTagName('content')[0].firstChild.nodeValue
      comments.append(IssueComment(title, text))

    return comments

class TrackerComm(object):
  """Class to manage communication with Tracker."""

  __slots__ = [
    'creds',       # gdata_lib.Creds object
    'it_client',   # Issue Tracker client
    ]

  def __init__(self, creds):
    self.creds = creds

    self.it_client = gdata.projecthosting.client.ProjectHostingClient()
    self.it_client.source = 'package_status_upgrade'
    self.it_client.ClientLogin(creds.user, creds.password,
                               source=self.it_client.source,
                               service='code',
                               account_type='GOOGLE')

  def GetTrackerIssueById(self, tid):
    """Get tracker issue given |tid| number.  Return Issue object if found."""

    query = gdata.projecthosting.client.Query(issue_id=str(tid))
    feed = self.it_client.get_issues('chromium-os', query=query)

    if feed.entry:
      return Issue(feed.entry[0])
    return None

  def CreateTrackerIssue(self, issue):
    """Create a new issue in Tracker according to |issue|."""
    created = self.it_client.add_issue(project_name=PROJECT_NAME,
                                       title=issue.title,
                                       content=issue.summary,
                                       author=self.creds.user,
                                       status=issue.status,
                                       owner=issue.owner,
                                       labels=issue.labels)
    issue.id = int(created.id.text.split('/')[-1])
    return issue.id

  def AppendTrackerIssueById(self, issue_id, comment):
    """Append |comment| to issue |issue_id| in Tracker"""
    self.it_client.update_issue(project_name=PROJECT_NAME,
                                issue_id=issue_id,
                                author=self.creds.user,
                                comment=comment)
    return issue_id

# TODO(mtennant): Remove this class;  Use refactored version in gdata_lib.
class SpreadsheetComm(object):
  """Class to manage communication with one Google Spreadsheet worksheet."""

  __slots__ = [
    'columns',     # List of column names
    'creds',       # gdata_lib.Creds object
    'gd_client',   # Google Data client
    'ss_key',      # Spreadsheet key
    'ws_key',      # Worksheet key
    ]

  def __init__(self, creds, ss_key, ws_name):
    self.creds = creds
    self._LoginWithUserPassword(creds.user, creds.password)

    self.ss_key = ss_key
    self.ws_key = self._GetWorksheetKey(ss_key, ws_name)

    self.columns = self._GetColumns()

  def _LoginWithUserPassword(self, user, password):
    """Set up and connect the Google Doc client using email/password."""
    gd_client = gdata_lib.RetrySpreadsheetsService()

    gd_client.source = 'Sync Package Status'
    gd_client.email = user
    gd_client.password = password
    gd_client.ProgrammaticLogin()
    self.gd_client = gd_client

  def _GetWorksheetKey(self, ss_key, ws_name):
    """Get the worksheet key with name |ws_name| in spreadsheet |ss_key|."""
    feed = self.gd_client.GetWorksheetsFeed(ss_key)
    # The worksheet key is the last component in the URL (after last '/')
    for entry in feed.entry:
      if ws_name == entry.title.text:
        return entry.id.text.split('/')[-1]

    oper.Die('Unable to find worksheet "%s" in spreadsheet "%s"' %
             (ws_name, ss_key))

  def _GetColumns(self):
    """Return list of column names in worksheet."""
    columns = []

    query = gdata.spreadsheet.service.CellQuery()
    query['max-row'] = '1'
    feed = self.gd_client.GetCellsFeed(self.ss_key, self.ws_key, query=query)
    for entry in feed.entry:
      columns.append(entry.content.text)

    return columns

  def GetColumnIndex(self, colName):
    """Get the column index (starting at 1) for column |colName|"""
    for ix, col in enumerate(self.columns):
      if colName == col:
        # Spreadsheet column indices start at 1.
        return ix + 1

    return None

  def GetAllRowsAsDicts(self):
    """Get every row in spreadsheet as dicts of key/value pairs."""
    feed = self.gd_client.GetListFeed(self.ss_key, self.ws_key)
    rows = []
    for entry in feed.entry:
      row = {}
      for key, val in entry.custom.items():
        row[key] = gdata_lib.ScrubValFromSS(val.text)

      rows.append(row)

    return rows

  def ReplaceCellValue(self, rowIx, colIx, val):
    """Replace cell value at |rowIx| and |colIx| with |val|"""
    self.gd_client.UpdateCell(rowIx, colIx, val, self.ss_key, self.ws_key)

  def ClearCellValue(self, rowIx, colIx):
    """Clear cell value at |rowIx| and |colIx|"""
    self.ReplaceCellValue(rowIx, colIx, None)

class Syncer(object):
  """Class to manage synchronizing between spreadsheet and Tracker."""

  # Map spreadsheet team names to Tracker team values.
  VALID_TEAMS = {'build': 'BuildRelease',
                 'kernel': 'Kernel',
                 'security': 'Security',
                 'system': 'Systems',
                 'ui': 'UI',
                 }
  UPGRADE_STATES = set([utable.UpgradeTable.STATE_NEEDS_UPGRADE,
                        utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED,
                        utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_DUPLICATED,
                        ])

  __slots__ = [
    'owners',         # Set of owners to select (None means no filter)
    'pretend',        # If True, make no real changes
    'scomm',          # SpreadsheetComm
    'tcomm',          # TrackerComm
    'teams',          # Set of teams to select (None means no filter)
    'tracker_col_ix', # Index of Tracker column in spreadsheet
    'verbose',        # Verbose boolean
    ]

  def __init__(self, tcomm, scomm, pretend=False, verbose=False):
    self.tcomm = tcomm
    self.scomm = scomm

    self.tracker_col_ix = scomm.GetColumnIndex('Tracker')

    self.teams = None
    self.owners = None

    self.pretend = pretend
    self.verbose = verbose

  def _ReduceTeamName(self, team):
    """Translate |team| from spreadsheet/commandline name to short name.

    For example:  build/bdavirro --> build, build --> build
    """
    if team:
      return team.lower().split('/')[0]
    return None

  def SetTeamFilter(self, teamarg):
    """Set team filter using colon-separated team names in |teamarg|

    Resulting filter in self.teams is set of "reduced" team names.
    Examples:
      'build:system:ui' --> set(['build', 'system', 'ui'])
      'Build:system:UI' --> set(['build', 'system', 'ui'])

    If an invalid team name is given oper.Die is called with explanation.
    """
    if teamarg:
      teamlist = []
      for team in teamarg.split(':'):
        t = self._ReduceTeamName(team)
        if t in self.VALID_TEAMS:
          teamlist.append(t)
        else:
          oper.Die('Invalid team name "%s".  Choose from: %s' %
                   (team, ','.join(self.VALID_TEAMS.keys())))
      self.teams = set(teamlist)
    else:
      self.teams = None

  def _ReduceOwnerName(self, owner):
    """Translate |owner| from spreadsheet/commandline name to short name.

    For example:  joe@chromium.org -> joe, joe --> joe
    """
    if owner:
      return owner.lower().split('@')[0]
    return None

  def SetOwnerFilter(self, ownerarg):
    """Set owner filter using colon-separated owner names in |ownerarg|"""
    if ownerarg:
      self.owners = set([self._ReduceOwnerName(o) for o in ownerarg.split(':')])
    else:
      self.owners = None

  def _RowPassesFilters(self, row):
    """Return true if |row| passes any team/owner filters"""
    if self.teams:
      team = self._ReduceTeamName(row[COL_TEAM])
      if team not in self.teams:
        return False

    if self.owners:
      owner = self._ReduceOwnerName(row[COL_OWNER])
      if owner not in self.owners:
        return False

    return True

  def Sync(self):
    """Do synchronization between Spreadsheet and Tracker."""

    errors = []

    # Go over each row in Spreadsheet.
    rows = self.scomm.GetAllRowsAsDicts()
    for rowIx, row in enumerate(rows):
      # Spreadsheet row index starts at 1, and we don't count
      # the header row.  So add 2.
      rowIx += 2
      if not self._RowPassesFilters(row):
        oper.Info('\nSkipping row %d, pkg: %r (team=%s, owner=%s) ...' %
                  (rowIx, row[COL_PACKAGE], row[COL_TEAM], row[COL_OWNER]))
        continue

      oper.Info('\nProcessing row %d, pkg: %r (team=%s, owner=%s) ...' %
                (rowIx, row[COL_PACKAGE], row[COL_TEAM], row[COL_OWNER]))

      try:
        new_issue = self._GenIssueForRow(row)
        old_issue_id = self._GetRowTrackerId(row)

        if new_issue and not old_issue_id:
          self._CreateRowIssue(rowIx, row, new_issue)
        elif not new_issue and old_issue_id:
          self._ClearRowIssue(rowIx, row)
        else:
          # Nothing to do for this package.
          reason = 'already has issue' if old_issue_id else 'no upgrade needed'
          oper.Notice('Nothing to do for row %d, package %r: %s.' %
                      (rowIx, row[COL_PACKAGE], reason))
      except SyncError:
        errors.append('Error processing row %d, pkg: %r.  See above.' %
                      (rowIx, row[COL_PACKAGE]))

    if errors:
      raise SyncError('\n'.join(errors))

  def _GetRowValue(self, row, colName, arch=None):
    """Get value from |row| at |colName|, adjusted for |arch|"""
    if arch:
      colName = utable.UpgradeTable.GetColumnName(colName, arch=arch)
    colName = gdata_lib.PrepColNameForSS(colName)
    return row[colName]

  def _GenIssueForRow(self, row):
    """Generate an Issue object for |row| if applicable"""
    # Row needs an issue if it "needs upgrade" on any platform.
    statuses = {}
    needs_issue = False
    for arch in ARCHES:
      state = self._GetRowValue(row, utable.UpgradeTable.COL_STATE, arch)
      statuses[arch] = state
      if state in self.UPGRADE_STATES:
        needs_issue = True

    if not needs_issue:
      return None

    pkg = row[COL_PACKAGE]
    team = self._ReduceTeamName(row[COL_TEAM])
    if not team:
      oper.Error('Unable to create Issue for package "%s" because no '
                 'team value is specified.' % pkg)
      raise SyncError()

    team_label = self.VALID_TEAMS[team]
    labels = ['Type-Task',
              'Area-LinuxUserSpace',
              'Pri-2',
              'Team-%s' % team_label,
              ]

    owner = self._ReduceOwnerName(row[COL_OWNER])
    status = 'Untriaged'
    if owner:
      owner = owner + '@chromium.org'
      status = 'Available'
    else:
      owner = None # Rather than ''

    title = '%s package needs upgrade from upstream Portage' % pkg

    lines = ['The %s package can be upgraded from upstream Portage' % pkg,
             '',
             'At this moment the status on each arch is as follows:',
             ]

    for arch in sorted(statuses):
      arch_status = statuses[arch]
      if arch_status:
        # Get all versions for this arch right now.
        curr_ver_col = utable.UpgradeTable.COL_CURRENT_VER
        curr_ver = self._GetRowValue(row, curr_ver_col, arch)
        stable_upst_ver_col = utable.UpgradeTable.COL_STABLE_UPSTREAM_VER
        stable_upst_ver = self._GetRowValue(row, stable_upst_ver_col, arch)
        latest_upst_ver_col = utable.UpgradeTable.COL_LATEST_UPSTREAM_VER
        latest_upst_ver = self._GetRowValue(row, latest_upst_ver_col, arch)

        arch_vers = ['Current version: %s' % curr_ver,
                     'Stable upstream version: %s' % stable_upst_ver,
                     'Latest upstream version: %s' % latest_upst_ver]
        lines.append('  On %s: %s' % (arch, arch_status))
        lines.append('    %s' % ', '.join(arch_vers))
      else:
        lines.append('  On %s: not used' % arch)

    lines.append('')
    lines.append('Check the latest status for this package, including '
                 'which upstream versions are available, at:\n  %s' %
                 PKG_STATUS_PAGE)
    lines.append('For help upgrading see: %s' % PKG_UPGRADE_PAGE)

    summary = '\n'.join(lines)

    issue = Issue(title=title,
                  summary=summary,
                  status=status,
                  owner=owner,
                  labels=labels,
                  )
    return issue

  def _GetRowTrackerId(self, row):
    """Get the tracker issue id in |row| if it exists, return None otherwise."""
    tracker_val = row[COL_TRACKER]
    if tracker_val:
      return int(tracker_val)

    return None

  def _CreateRowIssue(self, rowIx, row, issue):
    """Create a Tracker issue for |issue|, insert into |row| at |rowIx|"""

    pkg = row[COL_PACKAGE]
    if not self.pretend:
      oper.Info('Creating Tracker issue for package %s with details:\n%s' %
                (pkg, issue))
      issue_id = self.tcomm.CreateTrackerIssue(issue)
      oper.Info('Inserting new Tracker issue %d for package %s' %
                (issue_id, pkg))
      ss_issue_val = self._GenSSLinkToIssue(issue_id)
      self.scomm.ReplaceCellValue(rowIx, self.tracker_col_ix, ss_issue_val)

      oper.Notice('Created Tracker issue %d for row %d, package %r' %
                  (issue_id, rowIx, pkg))
    else:
      oper.Notice('Would create and insert issue for row %d, package %r' %
                  (rowIx, pkg))
      oper.Info('Issue would be as follows:\n%s' % issue)

  def _GenSSLinkToIssue(self, issue_id):
    """Create the spreadsheet hyperlink format for |issue_id|"""
    return '=hyperlink("crosbug.com/%d";"%d")' % (issue_id, issue_id)

  def _ClearRowIssue(self, rowIx, row):
    """Clear the Tracker cell for row at |rowIx|"""

    pkg = row[COL_PACKAGE]
    if not self.pretend:
      oper.Notice('Clearing Tracker issue for package %s' % pkg)
      self.scomm.ClearCellValue(rowIx, self.tracker_col_ix)
    else:
      oper.Notice('Would clear Tracker issue from row %d, package %r' %
                  (rowIx, pkg))


def _CreateOptParser():
  """Create the optparser.parser object for command-line args."""
  usage = 'Usage: %prog [options]'
  epilog = ('\n'
            'Use this script to synchronize tracker issues between the '
            'package status spreadsheet and the chromium-os Tracker.\n'
            'It uses the "Tracker" column of the package spreadsheet. '
            'If a package needs an upgrade and has no tracker issue\n'
            'in that column then a tracker issue is created.  If it '
            'does not need an upgrade then that column is cleared.\n'
            '\n'
            'Credentials must be specified using --cred-file or '
            '--email.  The former has a default value which you can\n'
            'rely on if valid, the latter will prompt for your password.  '
            'If you specify --email you will be given a chance to save\n'
            'your email/password out as a credentials file for next time.\n'
            '\n'
            'Uses spreadsheet key %s (worksheet "%s").\n'
            '\n'
            'Use the --team and --owner options to operate only on '
            'packages assigned to particular owners or teams.\n'
            '\n' %
            (SS_KEY, PKGS_WS_NAME)
            )

  class MyOptParser(optparse.OptionParser):
    """Override default epilog formatter, which strips newlines."""
    def format_epilog(self, formatter):
      return self.epilog

  teamhelp = '[%s]' % ', '.join(Syncer.VALID_TEAMS.keys())

  parser = MyOptParser(usage=usage, epilog=epilog)
  parser.add_option('--cred-file', dest='cred_file', type='string',
                    action='store', default=gdata_lib.CRED_FILE,
                    help='Path to gdata credentials file [default: "%default"]')
  parser.add_option('--email', dest='email', type='string',
                    action='store', default=None,
                    help="Email for Google Doc/Tracker user")
  parser.add_option('--pretend', dest='pretend', action='store_true',
                    default=False,
                    help='Do not make any actual changes.')
  parser.add_option('--team', dest='team', type='string', action='store',
                    default=None,
                    help='Filter by team; colon-separated %s' % teamhelp)
  parser.add_option('--owner', dest='owner', type='string', action='store',
                    default=None,
                    help='Filter by package owner;'
                    ' colon-separated chromium.org accounts')
  parser.add_option('--verbose', dest='verbose', action='store_true',
                    default=False,
                    help='Enable verbose output (for debugging)')

  return parser

def main():
  """Main function."""
  parser = _CreateOptParser()
  (options, _args) = parser.parse_args()

  oper.verbose = options.verbose

  if not options.email and not os.path.exists(options.cred_file):
    options.email = os.environ['USER']
    oper.Notice('Assuming your chromium email is %s@chromium.org.'
                '  Override with --email.' % options.email)

  creds = gdata_lib.Creds(cred_file=options.cred_file, user=options.email)
  tcomm = TrackerComm(creds)
  scomm = SpreadsheetComm(creds, SS_KEY, PKGS_WS_NAME)

  syncer = Syncer(tcomm, scomm,
                  pretend=options.pretend, verbose=options.verbose)

  if options.team:
    syncer.SetTeamFilter(options.team)
  if options.owner:
    syncer.SetOwnerFilter(options.owner)

  try:
    syncer.Sync()
  except SyncError as ex:
    oper.Die(str(ex))

  # If --email, which is only effective when run interactively (because the
  # password must be entered), give the option of saving to a creds file for
  # next time.
  if options.email and options.cred_file:
    prompt = ('Do you want to save credentials for next time to %r?' %
              options.cred_file)
    if 'yes' == cros_lib.YesNoPrompt(default='no', prompt=prompt):
      creds.StoreCreds(options.cred_file)
      oper.Notice('Be sure to save the creds file to the same location'
                  ' outside your chroot so it will also be used with'
                  ' future chroots.')

if __name__ == '__main__':
  main()