# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Recipe module to ensure a checkout is consistent on a bot."""

from recipe_engine import recipe_api

from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2


class BotUpdateApi(recipe_api.RecipeApi):

  def __init__(self, properties, deps_revision_overrides, fail_patch, *args,
               **kwargs):
    self._deps_revision_overrides = deps_revision_overrides
    self._fail_patch = fail_patch

    self._last_returned_properties = {}
    super(BotUpdateApi, self).__init__(*args, **kwargs)

  def initialize(self):
    assert len(self.m.buildbucket.build.input.gerrit_changes) <= 1, (
        'bot_update does not support more than one '
        'buildbucket.build.input.gerrit_changes')

  def __call__(self, name, cmd, **kwargs):
    """Wrapper for easy calling of bot_update."""
    assert isinstance(cmd, (list, tuple))
    bot_update_path = self.resource('bot_update.py')
    kwargs.setdefault('infra_step', True)

    # If a Git HTTP request is constantly below GIT_HTTP_LOW_SPEED_LIMIT
    # bytes/second for GIT_HTTP_LOW_SPEED_TIME seconds then such request will be
    # aborted. Otherwise, it would wait for global timeout to be reached.
    env = {
        'GIT_HTTP_LOW_SPEED_LIMIT': '102400',  # in bytes
        'GIT_HTTP_LOW_SPEED_TIME': 300,  # in seconds
    }
    with self.m.context(env=env):
      with self.m.depot_tools.on_path():
        return self.m.python(name, bot_update_path, cmd, **kwargs)

  @property
  def last_returned_properties(self):
    return self._last_returned_properties

  def _get_commit_repo_path(self, commit, gclient_config):
    """Returns local path to the repo that the commit is associated with.

    The commit must be a self.m.buildbucket.common_pb2.GitilesCommit.
    If commit does not specify any repo, returns name of the first solution.

    Raises an InfraFailure if the commit specifies a repo unexpected by gclient.
    """
    assert gclient_config.solutions, 'gclient_config.solutions is empty'

    # if repo is not specified, choose the first solution.
    if not (commit.host and commit.project):
      return gclient_config.solutions[0].name
    assert commit.host and commit.project

    repo_url = self.m.gitiles.unparse_repo_url(commit.host, commit.project)
    repo_path = self.m.gclient.get_repo_path(
        repo_url, gclient_config=gclient_config)
    if not repo_path:
      raise self.m.step.InfraFailure(
          'invalid (host, project) pair in '
          'buildbucket.build.input.gitiles_commit: '
          '(%r, %r) does not match any of configured gclient solutions '
          'and not present in gclient_config.repo_path_map' % (
              commit.host, commit.project))

    return repo_path

  def ensure_checkout(self,
                      gclient_config=None,
                      suffix=None,
                      patch=True,
                      update_presentation=True,
                      patch_root=None,
                      with_branch_heads=False,
                      with_tags=False,
                      no_fetch_tags=False,
                      refs=None,
                      patch_oauth2=None,
                      oauth2_json=None,
                      use_site_config_creds=None,
                      clobber=False,
                      root_solution_revision=None,
                      rietveld=None,
                      issue=None,
                      patchset=None,
                      gerrit_no_reset=False,
                      gerrit_no_rebase_patch_ref=False,
                      disable_syntax_validation=False,
                      patch_refs=None,
                      ignore_input_commit=False,
                      add_blamelists=False,
                      set_output_commit=False,
                      step_test_data=None,
                      **kwargs):
    """
    Args:
      gclient_config: The gclient configuration to use when running bot_update.
        If omitted, the current gclient configuration is used.
      no_fetch_tags: When true, the root git repo being checked out will not
        fetch any tags referenced from the references being fetched. When a repo
        has many references, it can become a performance bottleneck, so avoid
        tags if the checkout will not need them present.
      disable_syntax_validation: (legacy) Disables syntax validation for DEPS.
        Needed as migration paths for recipes dealing with older revisions,
        such as bisect.
      ignore_input_commit: if True, ignore api.buildbucket.gitiles_commit.
        Exists for historical reasons. Please do not use.
      add_blamelists: if True, add blamelist pins for all of the repos that had
        revisions specified in the gclient config.
      set_output_commit: if True, mark the checked out commit as the
        primary output commit of this build, i.e. call
        api.buildbucket.set_output_gitiles_commit.
        In case of multiple repos, the repo is the one specified in
        api.buildbucket.gitiles_commit or the first configured solution.
        When sorting builds by commit position, this commit will be used.
        Requires falsy ignore_input_commit.
      step_test_data: a null function that returns test bot_update.py output.
        Use test_api.output_json to generate test data.
    """
    assert use_site_config_creds is None, "use_site_config_creds is deprecated"
    assert rietveld is None, "rietveld is deprecated"
    assert issue is None, "issue is deprecated"
    assert patchset is None, "patchset is deprecated"
    assert patch_oauth2 is None, "patch_oauth2 is deprecated"
    assert oauth2_json is None, "oauth2_json is deprecated"
    assert not (ignore_input_commit and set_output_commit)

    refs = refs or []
    # We can re-use the gclient spec from the gclient module, since all the
    # data bot_update needs is already configured into the gclient spec.
    cfg = gclient_config or self.m.gclient.c
    assert cfg is not None, (
        'missing gclient_config or forgot api.gclient.set_config(...) before?')

    # Construct our bot_update command.  This basically be inclusive of
    # everything required for bot_update to know:
    patch_root = patch_root or self.m.gclient.get_gerrit_patch_root(
        gclient_config=cfg)

    # Allow patched project's revision if necessary.
    # This is important for projects which are checked out as DEPS of the
    # gclient solution.
    self.m.gclient.set_patch_repo_revision(cfg)

    reverse_rev_map = self.m.gclient.got_revision_reverse_mapping(cfg)

    flags = [
        # What do we want to check out (spec/root/rev/reverse_rev_map).
        ['--spec-path', self.m.raw_io.input(
            self.m.gclient.config_to_pythonish(cfg))],
        ['--patch_root', patch_root],
        ['--revision_mapping_file', self.m.json.input(reverse_rev_map)],
        ['--git-cache-dir', cfg.cache_dir],
        ['--cleanup-dir', self.m.path['cleanup'].join('bot_update')],

        # Hookups to JSON output back into recipes.
        ['--output_json', self.m.json.output()],
    ]

    # How to find the patch, if any
    if patch:
      repo_url = self.m.tryserver.gerrit_change_repo_url
      fetch_ref = self.m.tryserver.gerrit_change_fetch_ref
      target_ref = self.m.tryserver.gerrit_change_target_ref
      if repo_url and fetch_ref:
        flags.append([
            '--patch_ref',
            '%s@%s:%s' % (repo_url, target_ref, fetch_ref),
        ])
      if patch_refs:
        flags.extend(
            ['--patch_ref', patch_ref]
            for patch_ref in patch_refs)

    # Compute requested revisions.
    revisions = {}
    for solution in cfg.solutions:
      if solution.revision:
        revisions[solution.name] = solution.revision

    # HACK: ensure_checkout API must be redesigned so that we don't pass such
    # parameters. Existing semantics is too opiniated.
    in_commit = self.m.buildbucket.gitiles_commit
    in_commit_rev = in_commit.id or in_commit.ref
    if not ignore_input_commit and in_commit_rev:
      # Note: this is not entirely correct. build.input.gitiles_commit
      # definition says "The Gitiles commit to run against.".
      # However, here we ignore it if the config specified a revision.
      # This is necessary because existing builders rely on this behavior,
      # e.g. they want to force refs/heads/master at the config level.
      in_commit_repo_path = self._get_commit_repo_path(in_commit, cfg)
      # The repo_path that comes back on Windows will have backslashes, which
      # won't match the paths that the gclient configs and bot_update script use
      in_commit_repo_path = in_commit_repo_path.replace(self.m.path.sep, '/')
      revisions[in_commit_repo_path] = (
          revisions.get(in_commit_repo_path) or in_commit_rev)
      parsed_solution_urls = set(
          self.m.gitiles.parse_repo_url(s.url) for s in cfg.solutions)
      if (in_commit.id and in_commit.ref
          and (in_commit.host, in_commit.project) in parsed_solution_urls):
        refs = [in_commit.ref] + refs

    # Guarantee that first solution has a revision.
    # TODO(machenbach): We should explicitly pass HEAD for ALL solutions
    # that don't specify anything else.
    first_sol = cfg.solutions[0].name
    revisions[first_sol] = revisions.get(first_sol) or 'HEAD'

    if cfg.revisions:
      # Only update with non-empty values. Some recipe might otherwise
      # overwrite the HEAD default with an empty string.
      revisions.update(
          (k, v) for k, v in cfg.revisions.items() if v)
    if cfg.solutions and root_solution_revision:
      revisions[first_sol] = root_solution_revision
    # Allow for overrides required to bisect into rolls.
    revisions.update(self._deps_revision_overrides)

    # Compute command-line parameters for requested revisions.
    # Also collect all fixed revisions to simulate them in the json output.
    # Fixed revision are the explicit input revisions of bot_update.py, i.e.
    # every command line parameter "--revision name@value".
    fixed_revisions = {}
    for name, revision in sorted(revisions.items()):
      fixed_revision = self.m.gclient.resolve_revision(revision)
      if fixed_revision:
        fixed_revisions[name] = fixed_revision
        if fixed_revision.upper() == 'HEAD':
          # Sync to correct destination ref if HEAD was specified.
          fixed_revision = self._destination_ref(cfg, name)
        # If we're syncing to a ref, we want to make sure it exists before
        # trying to check it out.
        if (fixed_revision.startswith('refs/') and
            # TODO(crbug.com/874501): fetching additional refs is currently
            # only supported for the root solution. We should investigate
            # supporting it for other dependencies.
            cfg.solutions and
            cfg.solutions[0].name == name):
          # Handle the "ref:revision" syntax, e.g.
          # refs/branch-heads/4.2:deadbeef
          refs.append(fixed_revision.split(':')[0])
        flags.append(['--revision', '%s@%s' % (name, fixed_revision)])

    for ref in refs:
      assert not ref.startswith('refs/remotes/'), (
          'The "refs/remotes/*" syntax is not supported.\n'
          'The "remotes" syntax is dependent on the way the local repo is '
          'configured, and while there are defaults that can often be '
          'assumed, there is no guarantee the mapping will always be done in '
          'a particular way.')

    # Add extra fetch refspecs.
    for ref in refs:
      flags.append(['--refs', ref])

    # Filter out flags that are None.
    cmd = [item for flag_set in flags
           for item in flag_set if flag_set[1] is not None]

    if clobber:
      cmd.append('--clobber')
    if with_branch_heads or cfg.with_branch_heads:
      cmd.append('--with_branch_heads')
    if with_tags or cfg.with_tags:
      cmd.append('--with_tags')
    if gerrit_no_reset:
      cmd.append('--gerrit_no_reset')
    if no_fetch_tags:
      cmd.append('--no_fetch_tags')
    if gerrit_no_rebase_patch_ref:
      cmd.append('--gerrit_no_rebase_patch_ref')
    if disable_syntax_validation or cfg.disable_syntax_validation:
      cmd.append('--disable-syntax-validation')

    # Inject Json output for testing.
    first_sln = cfg.solutions[0].name
    step_test_data = step_test_data or (lambda: self.test_api.output_json(
        patch_root, first_sln, reverse_rev_map, self._fail_patch,
        fixed_revisions=fixed_revisions))

    name = 'bot_update'
    if not patch:
      name += ' (without patch)'
    if suffix:
      name += ' - %s' % suffix

    # Ah hah! Now that everything is in place, lets run bot_update!
    step_result = None
    try:
      # 87 and 88 are the 'patch failure' codes for patch download and patch
      # apply, respectively. We don't actually use the error codes, and instead
      # rely on emitted json to determine cause of failure.
      step_result = self(
           name, cmd, step_test_data=step_test_data,
           ok_ret=(0, 87, 88), **kwargs)
    except self.m.step.StepFailure as f:
      step_result = f.result
      raise
    finally:
      if step_result and step_result.json.output:
        result = step_result.json.output
        self._last_returned_properties = result.get('properties', {})

        if update_presentation:
          # Set properties such as got_revision.
          for prop_name, prop_value in (
              self.last_returned_properties.items()):
            step_result.presentation.properties[prop_name] = prop_value

        # Add helpful step description in the step UI.
        if 'step_text' in result:
          step_text = result['step_text']
          step_result.presentation.step_text = step_text

        if add_blamelists and 'manifest' in result:
          blamelist_pins = []
          for name in revisions:
            m = result['manifest'][name]
            pin = {'id': m['revision']}
            pin['host'], pin['project'] = (
                self.m.gitiles.parse_repo_url(m['repository']))
            blamelist_pins.append(pin)

          result.blamelist_pins = blamelist_pins
          self.m.milo.show_blamelist_for(blamelist_pins)

        # Set output commit of the build.
        if (set_output_commit and
            'got_revision' in self._last_returned_properties):
          # As of April 2019, got_revision describes the output commit,
          # the same commit that Build.output.gitiles_commit describes.
          # In particular, users tend to set got_revision to make Milo display
          # it. Derive output commit from got_revision.
          out_commit = common_pb2.GitilesCommit(
              id=self._last_returned_properties['got_revision'],
          )

          out_solution = reverse_rev_map['got_revision']
          out_manifest = result['manifest'][out_solution]
          assert out_manifest['revision'] == out_commit.id, (
              out_manifest, out_commit.id)

          out_commit.host, out_commit.project = (
              self.m.gitiles.parse_repo_url(out_manifest['repository'])
          )

          # Determine the output ref.
          got_revision_cp = self._last_returned_properties.get('got_revision_cp')
          in_rev = revisions.get(out_solution)
          if got_revision_cp:
            # If commit position string is available, read the ref from there.
            out_commit.ref, out_commit.position = (
                self.m.commit_position.parse(got_revision_cp))
          elif in_rev.startswith('refs/'):
            # If we were asked to check out a specific ref, use it as output
            # ref.
            out_commit.ref = in_rev
          elif in_rev == 'HEAD':
            # bot_update.py interprets HEAD as refs/heads/master
            out_commit.ref = 'refs/heads/master'
          elif out_commit.id == in_commit.id and in_commit.ref:
            # Derive output ref from the input ref.
            out_commit.ref = in_commit.ref
          else: # pragma: no cover
            assert False, (
                'Unsupposed case. '
                'Call buildbucket.set_output_gitiles_commit directly.'
            )
          self.m.buildbucket.set_output_gitiles_commit(out_commit)

        # Set the "checkout" path for the main solution.
        # This is used by the Chromium module to figure out where to look for
        # the checkout.
        # If there is a patch failure, emit another step that said things
        # failed.
        if result.get('patch_failure'):
          return_code = result.get('patch_apply_return_code')
          patch_body = result.get('failed_patch_body')
          try:
            if return_code == 3:
              # This is download failure, hence an infra failure.
              with self.m.context(infra_steps=True):
                self.m.python.failing_step(
                    'Patch failure', 'Git reported a download failure')
            else:
              # This is actual patch failure.
              self.m.tryserver.set_patch_failure_tryjob_result()
              self.m.cq.set_do_not_retry_build()
              self.m.python.failing_step(
                  'Patch failure', 'See attached log. Try rebasing?')
          except self.m.step.StepFailure as e:
            if patch_body:
              e.result.presentation.logs['patch error'] = (
                  patch_body.splitlines())
            raise e

        # bot_update actually just sets root to be the folder name of the
        # first solution.
        if (result.get('did_run')
            and 'checkout' not in self.m.path
            and 'root' in result):
          co_root = result['root']
          cwd = self.m.context.cwd or self.m.path['start_dir']
          self.m.path['checkout'] = cwd.join(*co_root.split(self.m.path.sep))

    return step_result

  def _destination_ref(self, cfg, path):
    """Returns the ref branch of a CL for the matching project if available or
    HEAD otherwise.

    If there's no Gerrit CL associated with the run, returns 'HEAD'.
    Otherwise this queries Gerrit for the correct destination ref, which
    might differ from refs/heads/master.

    Args:
      cfg: The used gclient config.
      path: The DEPS path of the project this prefix is for. E.g. 'src' or
          'src/v8'. The query will only be made for the project that matches
          the CL's project.
    Returns:
        A destination ref as understood by bot_update.py if available
        and if different from refs/heads/master, returns 'HEAD' otherwise.
    """
    # Ignore project paths other than the one belonging to the current CL.
    patch_path = self.m.gclient.get_gerrit_patch_root(gclient_config=cfg)
    if patch_path:
      patch_path = patch_path.replace(self.m.path.sep, '/')
    if not patch_path or path != patch_path:
      return 'HEAD'

    target_ref = self.m.tryserver.gerrit_change_target_ref
    if target_ref == 'refs/heads/master':
      return 'HEAD'

    return target_ref

  def resolve_fixed_revision(self, bot_update_json, name):
    """Set a fixed revision for a single dependency using project revision
    properties.
    """

    rev_properties = self.get_project_revision_properties(name)
    self.m.gclient.c.revisions = {
      name: bot_update_json['properties'][rev_properties[0]]
    }

  def _resolve_fixed_revisions(self, bot_update_json):
    """Set all fixed revisions from the first sync to their respective
    got_X_revision values.

    If on the first sync, a revision was requested to be HEAD, this avoids
    using HEAD potentially resolving to a different revision on the second
    sync. Instead, we sync explicitly to whatever was checked out the first
    time.

    Example (chromium trybot used with v8 patch):

    First sync was called with
    bot_update.py --revision src@abc --revision src/v8@HEAD
    Fixed revisions are: src, src/v8
    Got_revision_mapping: src->got_revision, src/v8->got_v8_revision
    got_revision = abc, got_v8_revision = deadbeef
    Second sync will be called with
    bot_update.py --revision src@abc --revision src/v8@deadbeef

    Example (chromium trybot used with chromium DEPS change, changing v8 from
    "v8_before" to "v8_after"):

    First sync was called with
    bot_update.py --revision src@abc
    Fixed revisions are: src
    Got_revision_mapping: src->got_revision, src/v8->got_v8_revision
    got_revision = abc, got_v8_revision = v8_after
    Second sync will be called with
    bot_update.py --revision src@abc
    When deapplying the patch, v8 will be synced to v8_before.
    """
    for name in bot_update_json.get('fixed_revisions', {}):
      rev_properties = self.get_project_revision_properties(name)
      if (rev_properties and
          bot_update_json['properties'].get(rev_properties[0])):
        self.m.gclient.c.revisions[name] = str(
            bot_update_json['properties'][rev_properties[0]])

  # TODO(machenbach): Replace usages of this method eventually by direct calls
  # to the manifest output.
  def get_project_revision_properties(self, project_name, gclient_config=None):
    """Returns all property names used for storing the checked-out revision of
    a given project.

    Args:
      project_name (str): The name of a checked-out project as deps path, e.g.
          src or src/v8.
      gclient_config: The gclient configuration to use. If omitted, the current
          gclient configuration is used.

    Returns (list of str): All properties that'll hold the checked-out revision
        of the given project. An empty list if no such properties exist.
    """
    cfg = gclient_config or self.m.gclient.c
    # Sort for determinism. We might have several properties for the same
    # project, e.g. got_revision and got_webrtc_revision.
    rev_reverse_map = self.m.gclient.got_revision_reverse_mapping(cfg)
    return sorted(
        prop
        for prop, project in rev_reverse_map.items()
        if project == project_name
    )

  def deapply_patch(self, bot_update_step):
    """Deapplies a patch, taking care of DEPS and solution revisions properly.
    """
    bot_update_json = bot_update_step.json.output
    # We only override first solution here to make sure that we correctly revert
    # changes to DEPS file, which is particularly important for auto-rolls. It
    # is also imporant that we do not assume that corresponding revision is
    # stored in the 'got_revision' as some gclient configs change the default
    # mapping for their own purposes.
    first_solution_name = self.m.gclient.c.solutions[0].name
    rev_property = self.get_project_revision_properties(first_solution_name)[0]
    self.m.gclient.c.revisions[first_solution_name] = str(
        bot_update_json['properties'][rev_property])
    self._resolve_fixed_revisions(bot_update_json)

    self.ensure_checkout(
        patch=False, no_fetch_tags=True, update_presentation=False)