# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import datetime
import logging
import threading

import common
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import global_config
from autotest_lib.server import site_utils
from autotest_lib.server.cros import provision
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers, reporting
from chromite.lib import metrics


CONFIG = global_config.global_config

JOB_MAX_RUNTIME_MINS_DEFAULT = CONFIG.get_config_value(
        'AUTOTEST_WEB', 'job_max_runtime_mins_default', type=int, default=72*60)

# Minimum RPC timeout setting for calls expected to take long time, e.g.,
# create_suite_job. If default socket time (socket.getdefaulttimeout()) is
# None or greater than this value, the default will be used.
# The value here is set to be the same as the timeout for the RetryingAFE object
# so long running RPCs can wait long enough before being aborted.
_MIN_RPC_TIMEOUT = 600

# Number of days back to search for existing job.
SEARCH_JOB_MAX_DAYS = 14

# Number of minutes to increase the value of DedupingScheduler.delay_minutes.
# This allows all suite jobs created in the same event to start provision jobs
# at different time. 5 minutes allows 40 boards to have provision jobs started
# with in about 200 minutes. That way, we don't add too much delay on test jobs
# and do not keep suite jobs running for too long. Note that suite jobs created
# by suite scheduler does not wait for test job to finish. That helps to reduce
# the load on drone.
DELAY_MINUTES_INTERVAL = 5
# Set maximum delay minutes to 24 hours. This is to prevent suite jobs from
# running for too long. Nightly and new_build tasks won't create that many
# suites that need such a long delay. However, weekly tasks can create several
# hundreds of suites as most of them requires to run on all branches.
MAX_DELAY_MINUTES = 1440

class DedupingSchedulerException(Exception):
    """Base class for exceptions from this module."""
    pass


class ScheduleException(DedupingSchedulerException):
    """Raised when an error is returned from the AFE during scheduling."""
    pass


class DedupException(DedupingSchedulerException):
    """Raised when an error occurs while checking for duplicate jobs."""
    pass


class DedupingScheduler(object):
    """A class that will schedule suites to run on a given board, build.

    Includes logic to check whether or not a given (suite, board, build)
    has already been run.  If so, it will skip scheduling that suite.

    @var _afe: a frontend.AFE instance used to talk to autotest.
    """

    _SUITE_SCHEDULER_SUITE_COUNT = metrics.Counter(
            'chromeos/autotest/suite_scheduler/suite/created')

    def __init__(self, afe=None, file_bug=False):
        """Constructor

        @param afe: an instance of AFE as defined in server/frontend.py.
                    Defaults to a frontend_wrappers.RetryingAFE instance.
        """
        self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30,
                                                         delay_sec=10,
                                                         debug=False)
        self._file_bug = file_bug

        # Number of minutes to delay a suite job from creating test jobs.
        self.delay_minutes = 0
        # Number of minutes to increase of decrease self.delay_minutes. When
        # self.delay_minutes reaches MAX_DELAY_MINUTES, it should wind down
        # to allow even distribution of test job creation.
        self.delay_minutes_interval = DELAY_MINUTES_INTERVAL
        # Lock to make sure each suite created with different delay_minutes.
        self._lock = threading.Lock()


    def _ShouldScheduleSuite(self, suite, board, test_source_build):
        """Return True if |suite| has not yet been run for |build| on |board|.

        True if |suite| has not been run for |build| on |board|, and
        the lab is open for this particular request.  False otherwise.

        @param suite: the name of the suite to run, e.g. 'bvt'
        @param board: the board to run the suite on, e.g. x86-alex
        @param test_source_build: Build with the source of tests.

        @return False if the suite was already scheduled, True if not
        @raise DedupException if the AFE raises while searching for jobs.

        """
        try:
            site_utils.check_lab_status(test_source_build)
        except site_utils.TestLabException as ex:
            logging.debug('Skipping suite %s, board %s, build %s:  %s',
                          suite, board, test_source_build, str(ex))
            return False
        try:
            start_time = str(datetime.datetime.now() -
                             datetime.timedelta(days=SEARCH_JOB_MAX_DAYS))
            return not self._afe.get_jobs(
                    name__istartswith=test_source_build,
                    name__iendswith='control.'+suite,
                    created_on__gte=start_time,
                    min_rpc_timeout=_MIN_RPC_TIMEOUT)
        except Exception as e:
            raise DedupException(e)


    def _Schedule(self, suite, board, build, pool, num, priority, timeout,
                  file_bugs=False, firmware_rw_build=None,
                  firmware_ro_build=None, test_source_build=None,
                  job_retry=False, launch_control_build=None,
                  run_prod_code=False, testbed_dut_count=None, no_delay=False):
        """Schedule |suite|, if it hasn't already been run.

        @param suite: the name of the suite to run, e.g. 'bvt'
        @param board: the board to run the suite on, e.g. x86-alex
        @param build: the ChromeOS build to install e.g.
                      x86-alex-release/R18-1655.0.0-a1-b1584.
        @param pool: the pool of machines to use for scheduling purposes.
                     Default: None
        @param num: the number of devices across which to shard the test suite.
                    Type: integer or None
                    Default: None (uses sharding factor in global_config.ini).
        @param priority: One of the values from
                         client.common_lib.priorities.Priority.
        @param timeout: The max lifetime of the suite in hours.
        @param file_bugs: True if bug filing is desired for this suite.
        @param firmware_rw_build: Firmware build to update RW firmware. Default
                                  to None.
        @param firmware_ro_build: Firmware build to update RO firmware. Default
                                  to None.
        @param test_source_build: Build that contains the server-side test code.
                                  Default to None to use the ChromeOS build
                                  (defined by `build`).
        @param job_retry: Set to True to enable job-level retry. Default is
                          False.
        @param launch_control_build: Name of a Launch Control build, e.g.,
                                     'git_mnc_release/shamu-eng/123'
        @param run_prod_code: If True, the suite will run the test code that
                              lives in prod aka the test code currently on the
                              lab servers. If False, the control files and test
                              code for this suite run will be retrieved from the
                              build artifacts. Default is False.
        @param testbed_dut_count: Number of duts to test when using a testbed.
        @param no_delay: Set to True to allow suite to be created without
                         configuring delay_minutes. Default is False.

        @return True if the suite got scheduled
        @raise ScheduleException if an error occurs while scheduling.

        """
        try:
            if build:
                builds = {provision.CROS_VERSION_PREFIX: build}
            if firmware_rw_build:
                builds[provision.FW_RW_VERSION_PREFIX] = firmware_rw_build
            if firmware_ro_build:
                builds[provision.FW_RO_VERSION_PREFIX] = firmware_ro_build
            if launch_control_build:
                if testbed_dut_count is None:
                    builds = {provision.ANDROID_BUILD_VERSION_PREFIX:
                              launch_control_build}
                else:
                    builds = {provision.TESTBED_BUILD_VERSION_PREFIX:
                              launch_control_build}

            # Suite scheduler handles all boards in parallel, to guarantee each
            # call of `create_suite_job` use different value of delay_minutes,
            # we need a lock around get/set attempts of self.delay_minutes.
            # To prevent suite jobs from running too long, the value for
            # self.delay_minutes is limited between 0 and MAX_DELAY_MINUTES (4
            # hours). The value starts at 0 and is increased by
            # DELAY_MINUTES_INTERVAL, when it reaches MAX_DELAY_MINUTES, the
            # logic here allows its value to step back by DELAY_MINUTES_INTERVAL
            # at each call of this method. When the value drops back to 0, it
            # will increase again in the next call of this method.
            # Such logic allows the values of delay_minutes for all calls
            # of `create_suite_job` running in parallel to be evenly distributed
            # between 0 and MAX_DELAY_MINUTES.
            delay_minutes = 0
            if not no_delay:
                with self._lock:
                    delay_minutes = self.delay_minutes
                    if ((self.delay_minutes < MAX_DELAY_MINUTES and
                         self.delay_minutes_interval > 0) or
                        (self.delay_minutes >= DELAY_MINUTES_INTERVAL and
                         self.delay_minutes_interval < 0)):
                        self.delay_minutes += self.delay_minutes_interval
                    else:
                        limit = ('Maximum' if self.delay_minutes_interval > 0
                                 else 'Minimum')
                        logging.info(
                                '%s delay minutes reached when scheduling '
                                '%s on %s against %s (pool: %s)',
                                limit, suite, builds, board, pool)
                        self.delay_minutes_interval = (
                                -self.delay_minutes_interval)

            # Update timeout settings for the suite job with delay_minutes.
            # `timeout` is in hours.
            if not timeout:
                timeout = JOB_MAX_RUNTIME_MINS_DEFAULT / 60.0
            timeout += delay_minutes / 60.0
            max_runtime_mins = JOB_MAX_RUNTIME_MINS_DEFAULT + delay_minutes
            timeout_mins = JOB_MAX_RUNTIME_MINS_DEFAULT + delay_minutes

            logging.info('Scheduling %s on %s against %s (pool: %s)',
                         suite, builds, board, pool)
            if self._afe.run('create_suite_job', name=suite, board=board,
                             builds=builds, check_hosts=False, num=num,
                             pool=pool, priority=priority, timeout=timeout,
                             max_runtime_mins=max_runtime_mins,
                             timeout_mins=timeout_mins,
                             file_bugs=file_bugs,
                             wait_for_results=file_bugs,
                             test_source_build=test_source_build,
                             job_retry=job_retry,
                             delay_minutes=delay_minutes,
                             run_prod_code=run_prod_code,
                             min_rpc_timeout=_MIN_RPC_TIMEOUT) is not None:
                # Report data to metrics.
                fields = {'suite': suite,
                          'board': board,
                          'pool': pool,
                          'priority': str(priority)}
                self._SUITE_SCHEDULER_SUITE_COUNT.increment(fields=fields)
                return True
            else:
                raise ScheduleException(
                        "Can't schedule %s for %s." % (suite, builds))
        except (error.ControlFileNotFound, error.ControlFileEmpty,
                error.ControlFileMalformed, error.NoControlFileList) as e:
            if self._file_bug:
                # File bug on test_source_build if it's specified.
                b = reporting.SuiteSchedulerBug(
                        suite, test_source_build or build, board, e)
                # If a bug has filed with the same <suite, build, error type>
                # will not file again, but simply gets the existing bug id.
                bid, _ = reporting.Reporter().report(
                        b, ignore_duplicate=True)
                if bid is not None:
                    return False
            # Raise the exception if not filing a bug or failed to file bug.
            raise ScheduleException(e)
        except Exception as e:
            raise ScheduleException(e)


    def ScheduleSuite(self, suite, board, build, pool, num, priority, timeout,
                      force=False, file_bugs=False, firmware_rw_build=None,
                      firmware_ro_build=None, test_source_build=None,
                      job_retry=False, launch_control_build=None,
                      run_prod_code=False, testbed_dut_count=None,
                      no_delay=False):
        """Schedule |suite|, if it hasn't already been run.

        If |suite| has not already been run against |build| on |board|,
        schedule it and return True.  If it has, return False.

        @param suite: the name of the suite to run, e.g. 'bvt'
        @param board: the board to run the suite on, e.g. x86-alex
        @param build: the ChromeOS build to install e.g.
                      x86-alex-release/R18-1655.0.0-a1-b1584.
        @param pool: the pool of machines to use for scheduling purposes.
        @param num: the number of devices across which to shard the test suite.
                    Type: integer or None
        @param priority: One of the values from
                         client.common_lib.priorities.Priority.
        @param timeout: The max lifetime of the suite in hours.
        @param force: Always schedule the suite.
        @param file_bugs: True if bug filing is desired for this suite.
        @param firmware_rw_build: Firmware build to update RW firmware. Default
                                  to None.
        @param firmware_ro_build: Firmware build to update RO firmware. Default
                                  to None.
        @param test_source_build: Build with the source of tests. Default to
                                  None to use the ChromeOS build.
        @param job_retry: Set to True to enable job-level retry. Default is
                          False.
        @param launch_control_build: Name of a Launch Control build, e.g.,
                                     'git_mnc_release/shamu-eng/123'
        @param run_prod_code: If True, the suite will run the test code that
                              lives in prod aka the test code currently on the
                              lab servers. If False, the control files and test
                              code for this suite run will be retrieved from the
                              build artifacts. Default is False.
        @param testbed_dut_count: Number of duts to test when using a testbed.
        @param no_delay: Set to True to allow suite to be created without
                configuring delay_minutes. Default is False.

        @return True if the suite got scheduled, False if not
        @raise DedupException if we can't check for dups.
        @raise ScheduleException if the suite cannot be scheduled.

        """
        if (force or self._ShouldScheduleSuite(
                suite, board,
                test_source_build or build or launch_control_build)):
            return self._Schedule(suite, board, build, pool, num, priority,
                                  timeout, file_bugs=file_bugs,
                                  firmware_rw_build=firmware_rw_build,
                                  firmware_ro_build=firmware_ro_build,
                                  test_source_build=test_source_build,
                                  job_retry=job_retry,
                                  launch_control_build=launch_control_build,
                                  run_prod_code=run_prod_code,
                                  testbed_dut_count=testbed_dut_count,
                                  no_delay=no_delay)
        return False


    def CheckHostsExist(self, *args, **kwargs):
        """Forward a request to check if hosts matching args, kwargs exist."""
        try:
            kwargs['min_rpc_timeout'] = _MIN_RPC_TIMEOUT
            return self._afe.get_hostnames(*args, **kwargs)
        except error.TimeoutException as e:
            logging.exception(e)
            return []