# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Configuration options for various cbuildbot builders."""

from __future__ import print_function

import copy
import json

from chromite.cbuildbot import constants
from chromite.lib import osutils

GS_PATH_DEFAULT = 'default' # Means gs://chromeos-image-archive/ + bot_id

# Contains the valid build config suffixes in the order that they are dumped.
CONFIG_TYPE_PRECQ = 'pre-cq'
CONFIG_TYPE_PALADIN = 'paladin'
CONFIG_TYPE_RELEASE = 'release'
CONFIG_TYPE_FULL = 'full'
CONFIG_TYPE_FIRMWARE = 'firmware'
CONFIG_TYPE_FACTORY = 'factory'
CONFIG_TYPE_RELEASE_AFDO = 'release-afdo'

# This is only used for unitests... find a better solution?
CONFIG_TYPE_DUMP_ORDER = (
    CONFIG_TYPE_PALADIN,
    constants.PRE_CQ_GROUP_CONFIG,
    CONFIG_TYPE_PRECQ,
    constants.PRE_CQ_LAUNCHER_CONFIG,
    'incremental',
    'telemetry',
    CONFIG_TYPE_FULL,
    'full-group',
    CONFIG_TYPE_RELEASE,
    'release-group',
    'release-afdo',
    'release-afdo-generate',
    'release-afdo-use',
    'sdk',
    'chromium-pfq',
    'chromium-pfq-informational',
    'chrome-perf',
    'chrome-pfq',
    'chrome-pfq-informational',
    'pre-flight-branch',
    CONFIG_TYPE_FACTORY,
    CONFIG_TYPE_FIRMWARE,
    'toolchain-major',
    'toolchain-minor',
    'asan',
    'asan-informational',
    'refresh-packages',
    'test-ap',
    'test-ap-group',
    constants.BRANCH_UTIL_CONFIG,
    constants.PAYLOADS_TYPE,
)


# In the Json, this special build config holds the default values for all
# other configs.
DEFAULT_BUILD_CONFIG = '_default'


def IsPFQType(b_type):
  """Returns True if this build type is a PFQ."""
  return b_type in (constants.PFQ_TYPE, constants.PALADIN_TYPE,
                    constants.CHROME_PFQ_TYPE)


def IsCQType(b_type):
  """Returns True if this build type is a Commit Queue."""
  return b_type == constants.PALADIN_TYPE


def IsCanaryType(b_type):
  """Returns True if this build type is a Canary."""
  return b_type == constants.CANARY_TYPE


class BuildConfig(dict):
  """Dictionary of explicit configuration settings for a cbuildbot config

  Each dictionary entry is in turn a dictionary of config_param->value.

  See _settings for details on known configurations, and their documentation.
  """

  _delete_key_sentinel = object()

  @classmethod
  def delete_key(cls):
    """Used to remove the given key from inherited config.

    Usage:
      new_config = base_config.derive(foo=delete_key())
    """
    return cls._delete_key_sentinel

  @classmethod
  def delete_keys(cls, keys):
    """Used to remove a set of keys from inherited config.

    Usage:
      new_config = base_config.derive(delete_keys(set_of_keys))
    """
    return {k: cls._delete_key_sentinel for k in keys}

  def __getattr__(self, name):
    """Support attribute-like access to each dict entry."""
    if name in self:
      return self[name]

    # Super class (dict) has no __getattr__ method, so use __getattribute__.
    return super(BuildConfig, self).__getattribute__(name)

  def GetBotId(self, remote_trybot=False):
    """Get the 'bot id' of a particular bot.

    The bot id is used to specify the subdirectory where artifacts are stored
    in Google Storage. To avoid conflicts between remote trybots and regular
    bots, we add a 'trybot-' prefix to any remote trybot runs.

    Args:
      remote_trybot: Whether this run is a remote trybot run.
    """
    return 'trybot-%s' % self.name if remote_trybot else self.name

  def deepcopy(self):
    """Create a deep copy of this object.

    This is a specialized version of copy.deepcopy() for BuildConfig objects. It
    speeds up deep copies by 10x because we know in advance what is stored
    inside a BuildConfig object and don't have to do as much introspection. This
    function is called a lot during setup of the config objects so optimizing it
    makes a big difference. (It saves seconds off the load time of this module!)
    """
    new_config = BuildConfig(self)
    for k, v in self.iteritems():
      # type(v) is faster than isinstance.
      if type(v) is list:
        new_config[k] = v[:]

    if new_config.get('child_configs'):
      new_config['child_configs'] = [
          x.deepcopy() for x in new_config['child_configs']]

    if new_config.get('hw_tests'):
      new_config['hw_tests'] = [copy.copy(x) for x in new_config['hw_tests']]

    return new_config

  def derive(self, *args, **kwargs):
    """Create a new config derived from this one.

    Note: If an override is callable, it will be called and passed the prior
    value for the given key (or None) to compute the new value.

    Args:
      args: Mapping instances to mixin.
      kwargs: Settings to inject; see _settings for valid values.

    Returns:
      A new _config instance.
    """
    inherits = list(args)
    inherits.append(kwargs)
    new_config = self.deepcopy()

    for update_config in inherits:
      for k, v in update_config.iteritems():
        if callable(v):
          new_config[k] = v(new_config.get(k))
        else:
          new_config[k] = v

      keys_to_delete = [k for k in new_config if
                        new_config[k] is self._delete_key_sentinel]

      for k in keys_to_delete:
        new_config.pop(k, None)

    return new_config

  def HideDefaults(self, default):
    """Create a duplicate BuildConfig with default values missing.

    Args:
      default: Dictionary of key/value pairs to remove, if present.

    Returns:
      A copy BuildConfig, that only contains values different from defaults.
    """
    d = BuildConfig()
    for k, v in self.iteritems():
      # Recurse to children.
      if k == 'child_configs':
        v = [child.HideDefaults(default) for child in v]

      # Only store a value if it's different from default.
      if k not in default or default[k] != v:
        d[k] = v

    return d


class HWTestConfig(object):
  """Config object for hardware tests suites.

  Members:
    suite: Name of the test suite to run.
    timeout: Number of seconds to wait before timing out waiting for
             results.
    pool: Pool to use for hw testing.
    blocking: Suites that set this true run sequentially; each must pass
              before the next begins.  Tests that set this false run in
              parallel after all blocking tests have passed.
    async: Fire-and-forget suite.
    warn_only: Failure on HW tests warns only (does not generate error).
    critical: Usually we consider structural failures here as OK.
    priority:  Priority at which tests in the suite will be scheduled in
               the hw lab.
    file_bugs: Should we file bugs if a test fails in a suite run.
    num: Maximum number of DUTs to use when scheduling tests in the hw lab.
    minimum_duts: minimum number of DUTs required for testing in the hw lab.
    retry: Whether we should retry tests that fail in a suite run.
    max_retries: Integer, maximum job retries allowed at suite level.
                 None for no max.
    suite_min_duts: Preferred minimum duts. Lab will prioritize on getting such
                    number of duts even if the suite is competing with
                    other suites that have higher priority.

  Some combinations of member settings are invalid:
    * A suite config may not specify both blocking and async.
    * A suite config may not specify both retry and async.
    * A suite config may not specify both warn_only and critical.
  """
  # This timeout is larger than it needs to be because of autotest overhead.
  # TODO(davidjames): Reduce this timeout once http://crbug.com/366141 is fixed.
  DEFAULT_HW_TEST_TIMEOUT = 60 * 220
  BRANCHED_HW_TEST_TIMEOUT = 10 * 60 * 60

  def __init__(self, suite, num=constants.HWTEST_DEFAULT_NUM,
               pool=constants.HWTEST_MACH_POOL, timeout=DEFAULT_HW_TEST_TIMEOUT,
               async=False, warn_only=False, critical=False, blocking=False,
               file_bugs=False, priority=constants.HWTEST_BUILD_PRIORITY,
               retry=True, max_retries=10, minimum_duts=0, suite_min_duts=0):
    """Constructor -- see members above."""
    assert not async or (not blocking and not retry)
    assert not warn_only or not critical
    self.suite = suite
    self.num = num
    self.pool = pool
    self.timeout = timeout
    self.blocking = blocking
    self.async = async
    self.warn_only = warn_only
    self.critical = critical
    self.file_bugs = file_bugs
    self.priority = priority
    self.retry = retry
    self.max_retries = max_retries
    self.minimum_duts = minimum_duts
    self.suite_min_duts = suite_min_duts

  def SetBranchedValues(self):
    """Changes the HW Test timeout/priority values to branched values."""
    self.timeout = max(HWTestConfig.BRANCHED_HW_TEST_TIMEOUT, self.timeout)

    # Set minimum_duts default to 0, which means that lab will not check the
    # number of available duts to meet the minimum requirement before creating
    # a suite job for branched build.
    self.minimum_duts = 0

    # Only reduce priority if it's lower.
    new_priority = constants.HWTEST_DEFAULT_PRIORITY
    if (constants.HWTEST_PRIORITIES_MAP[self.priority] >
        constants.HWTEST_PRIORITIES_MAP[new_priority]):
      self.priority = new_priority

  @property
  def timeout_mins(self):
    return int(self.timeout / 60)

  def __eq__(self, other):
    return self.__dict__ == other.__dict__


class SiteConfig(dict):
  """This holds a set of named BuildConfig values."""

  def __init__(self, defaults=None):
    """Init."""
    super(SiteConfig, self).__init__()
    self._defaults = {} if defaults is None else defaults

  def GetDefault(self):
    """Create the cannonical default build configuration."""
    # Enumeration of valid settings; any/all config settings must be in this.
    # All settings must be documented.
    return BuildConfig(**self._defaults)

  #
  # Methods for searching a SiteConfig's contents.
  #
  def FindFullConfigsForBoard(self, board=None):
    """Returns full builder configs for a board.

    Args:
      board: The board to match. By default, match all boards.

    Returns:
      A tuple containing a list of matching external configs and a list of
      matching internal release configs for a board.
    """
    ext_cfgs = []
    int_cfgs = []

    for name, c in self.iteritems():
      if c['boards'] and (board is None or board in c['boards']):
        if (name.endswith('-%s' % CONFIG_TYPE_RELEASE) and
            c['internal']):
          int_cfgs.append(c.deepcopy())
        elif (name.endswith('-%s' % CONFIG_TYPE_FULL) and
              not c['internal']):
          ext_cfgs.append(c.deepcopy())

    return ext_cfgs, int_cfgs

  def FindCanonicalConfigForBoard(self, board, allow_internal=True):
    """Get the canonical cbuildbot builder config for a board."""
    ext_cfgs, int_cfgs = self.FindFullConfigsForBoard(board)
    # If both external and internal builds exist for this board, prefer the
    # internal one unless instructed otherwise.
    both = (int_cfgs if allow_internal else []) + ext_cfgs

    if not both:
      raise ValueError('Invalid board specified: %s.' % board)
    return both[0]

  def GetSlavesForMaster(self, master_config, options=None):
    """Gets the important slave builds corresponding to this master.

    A slave config is one that matches the master config in build_type,
    chrome_rev, and branch.  It also must be marked important.  For the
    full requirements see the logic in code below.

    The master itself is eligible to be a slave (of itself) if it has boards.

    TODO(dgarrett): Replace this with explicit master/slave defitions to make
    the concept less Chrome OS specific. crbug.com/492382.

    Args:
      master_config: A build config for a master builder.
      options: The options passed on the commandline. This argument is optional,
               and only makes sense when called from cbuildbot.

    Returns:
      A list of build configs corresponding to the slaves for the master
        represented by master_config.

    Raises:
      AssertionError if the given config is not a master config or it does
        not have a manifest_version.
    """
    assert master_config['manifest_version']
    assert master_config['master']

    slave_configs = []
    if options is not None and options.remote_trybot:
      return slave_configs

    # TODO(davidjames): In CIDB the master isn't considered a slave of itself,
    # so we probably shouldn't consider it a slave here either.
    for build_config in self.itervalues():
      if (build_config['important'] and
          build_config['manifest_version'] and
          (not build_config['master'] or build_config['boards']) and
          build_config['build_type'] == master_config['build_type'] and
          build_config['chrome_rev'] == master_config['chrome_rev'] and
          build_config['branch'] == master_config['branch']):
        slave_configs.append(build_config)

    return slave_configs

  #
  # Methods used when creating a Config programatically.
  #

  def AddConfig(self, config, name, *args, **kwargs):
    """Derive and add the config to cbuildbot's usable config targets

    Args:
      config: BuildConfig to derive the new config from.
      name: The name to label this configuration; this is what cbuildbot
            would see.
      args: See the docstring of derive.
      kwargs: See the docstring of derive.

    Returns:
      See the docstring of derive.
    """
    inherits, overrides = args, kwargs
    overrides['name'] = name

    # Add ourselves into the global dictionary, adding in the defaults.
    new_config = config.derive(*inherits, **overrides)
    self[name] = self.GetDefault().derive(config, new_config)

    # Return a BuildConfig object without the defaults, so that other objects
    # can derive from us without inheriting the defaults.
    return new_config

  def AddRawConfig(self, name, *args, **kwargs):
    """Add a config containing only explicitly listed values (no defaults)."""
    return self.AddConfig(BuildConfig(), name, *args, **kwargs)

  def AddGroup(self, name, *args, **kwargs):
    """Create a new group of build configurations.

    Args:
      name: The name to label this configuration; this is what cbuildbot
            would see.
      args: Configurations to build in this group. The first config in
            the group is considered the primary configuration and is used
            for syncing and creating the chroot.
      kwargs: Override values to use for the parent config.

    Returns:
      A new BuildConfig instance.
    """
    child_configs = [self.GetDefault().derive(x, grouped=True) for x in args]
    return self.AddConfig(args[0], name, child_configs=child_configs, **kwargs)

  def SaveConfigToFile(self, config_file):
    """Save this Config to a Json file.

    Args:
      config_file: The file to write too.
    """
    json_string = self.SaveConfigToString()
    osutils.WriteFile(config_file, json_string)

  def SaveConfigToString(self):
    """Save this Config object to a Json format string."""
    default = self.GetDefault()

    config_dict = {}
    for k, v in self.iteritems():
      config_dict[k] = v.HideDefaults(default)

    config_dict['_default'] = default

    class _JSONEncoder(json.JSONEncoder):
      """Json Encoder that encodes objects as their dictionaries."""
      # pylint: disable=E0202
      def default(self, obj):
        return self.encode(obj.__dict__)

    return json.dumps(config_dict, cls=_JSONEncoder,
                      sort_keys=True, indent=4, separators=(',', ': '))

#
# Methods related to loading/saving Json.
#

def CreateConfigFromFile(config_file):
  """Load a Config a Json encoded file."""
  json_string = osutils.ReadFile(config_file)
  return CreateConfigFromString(json_string)


def CreateConfigFromString(json_string):
  """Load a cbuildbot config from it's Json encoded string."""
  config_dict = json.loads(json_string, object_hook=_DecodeDict)

  # default is a dictionary of default build configuration values.
  defaults = config_dict.pop(DEFAULT_BUILD_CONFIG)

  defaultBuildConfig = BuildConfig(**defaults)

  builds = {n: _CreateBuildConfig(defaultBuildConfig, v)
            for n, v in config_dict.iteritems()}

  # config is the struct that holds the complete cbuildbot config.
  result = SiteConfig(defaults=defaults)
  result.update(builds)

  return result

# TODO(dgarrett): Remove Decode methods when we prove unicde strings work.
def _DecodeList(data):
  """Convert a JSON result list from unicode to utf-8."""
  rv = []
  for item in data:
    if isinstance(item, unicode):
      item = item.encode('utf-8')
    elif isinstance(item, list):
      item = _DecodeList(item)
    elif isinstance(item, dict):
      item = _DecodeDict(item)

    # Other types (None, int, float, etc) are stored unmodified.
    rv.append(item)
  return rv


def _DecodeDict(data):
  """Convert a JSON result dict from unicode to utf-8."""
  rv = {}
  for key, value in data.iteritems():
    if isinstance(key, unicode):
      key = key.encode('utf-8')

    if isinstance(value, unicode):
      value = value.encode('utf-8')
    elif isinstance(value, list):
      value = _DecodeList(value)
    elif isinstance(value, dict):
      value = _DecodeDict(value)

    # Other types (None, int, float, etc) are stored unmodified.
    rv[key] = value
  return rv


def _CreateHwTestConfig(jsonString):
  """Create a HWTestConfig object from a JSON string."""
  # Each HW Test is dumped as a json string embedded in json.
  hw_test_config = json.loads(jsonString, object_hook=_DecodeDict)
  return HWTestConfig(**hw_test_config)


def _CreateBuildConfig(default, build_dict):
  """Create a BuildConfig object from it's parsed JSON dictionary encoding."""
  # These build config values need special handling.
  hwtests = build_dict.pop('hw_tests', None)
  child_configs = build_dict.pop('child_configs', None)

  result = default.derive(**build_dict)

  if hwtests is not None:
    result['hw_tests'] = [_CreateHwTestConfig(hwtest) for hwtest in hwtests]

  if child_configs is not None:
    result['child_configs'] = [_CreateBuildConfig(default, child)
                               for child in child_configs]

  return result