# -*- coding: utf-8 -*-
# Copyright 2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unittests for Mob* Monitor checkfile manager."""

from __future__ import print_function

import imp
import mock
import os
import subprocess
import time
import threading

from cherrypy.process import plugins
from chromite.lib import cros_test_lib
from chromite.mobmonitor.checkfile import manager

# Test health check and related attributes
class TestHealthCheck(object):
  """Test health check."""

  def Check(self):
    """Stub Check."""
    return 0

  def Diagnose(self, _errcode):
    """Stub Diagnose."""
    return ('Unknown Error.', [])


class TestHealthCheckHasAttributes(object):
  """Test health check with attributes."""

  CHECK_INTERVAL_SEC = 10

  def Check(self):
    """Stub Check."""
    return 0

  def Diagnose(self, _errcode):
    """Stub Diagnose."""
    return ('Unknown Error.', [])


class TestHealthCheckUnhealthy(object):
  """Unhealthy test health check."""

  def __init__(self):
    self.x = -1

  def Check(self):
    """Stub Check."""
    return self.x

  def Diagnose(self, errcode):
    """Stub Diagnose."""
    if errcode == -1:
      return ('Stub Error.', [self.Repair])
    return ('Unknown Error.', [])

  def Repair(self):
    self.x = 0


class TestHealthCheckMultipleActions(object):
  """Unhealthy check with many actions that have different parameters."""

  def __init__(self):
    self.x = -1

  def Check(self):
    """Stub Check."""
    return self.x

  def Diagnose(self, errcode):
    """Stub Diagnose."""
    if errcode == -1:
      return ('Stub Error.', [self.NoParams, self.PositionalParams,
                              self.DefaultParams, self.MixedParams])
    return ('Unknown Error.', [])

  def NoParams(self):
    """NoParams Action."""
    self.x = 0

  # pylint: disable=unused-argument
  def PositionalParams(self, x, y, z):
    """PositionalParams Action."""
    self.x = 0

  def DefaultParams(self, x=1, y=2, z=3):
    """DefaultParams Action."""
    self.x = 0

  def MixedParams(self, x, y, z=1):
    """MixedParams Action."""
    self.x = 0
  # pylint: enable=unused-argument


class TestHealthCheckQuasihealthy(object):
  """Quasi-healthy test health check."""

  def Check(self):
    """Stub Check."""
    return 1

  def Diagnose(self, errcode):
    """Stub Diagnose."""
    if errcode == 1:
      return ('Stub Error.', [self.RepairStub])
    return ('Unknown Error.', [])

  def RepairStub(self):
    """Stub repair action."""


class TestHealthCheckBroken(object):
  """Broken test health check."""

  def Check(self):
    """Stub Check."""
    raise ValueError()

  def Diagnose(self, _errcode):
    """A broken Diagnose function. A proper return should be a pair."""
    raise ValueError()


def TestAction():
  return True


TEST_SERVICE_NAME = 'test-service'
TEST_MTIME = 100
TEST_EXEC_TIME = 400
CHECKDIR = '.'

# Strings that are used to mock actual check modules.
CHECKFILE_MANY_SIMPLE = '''
SERVICE = 'test-service'

class MyHealthCheck2(object):
  def Check(self):
    return 0

  def Diagnose(self, errcode):
    return ('Unknown error.', [])

class MyHealthCheck3(object):
  def Check(self):
    return 0

  def Diagnose(self, errcode):
    return ('Unknown error.', [])

class MyHealthCheck4(object):
  def Check(self):
    return 0

  def Diagnose(self, errcode):
    return ('Unknown error.', [])
'''

CHECKFILE_MANY_SIMPLE_ONE_BAD = '''
SERVICE = 'test-service'

class MyHealthCheck(object):
  def Check(self):
    return 0

  def Diagnose(self, errcode):
    return ('Unknown error.', [])

class NotAHealthCheck(object):
  def Diagnose(self, errcode):
    return ('Unknown error.', [])

class MyHealthCheck2(object):
  def Check(self):
    return 0

  def Diagnose(self, errcode):
    return ('Unknown error.', [])
'''

NOT_A_CHECKFILE = '''
class NotAHealthCheck(object):
  def NotCheckNorDiagnose(self):
    return -1
'''

ANOTHER_NOT_A_CHECKFILE = '''
class AnotherNotAHealthCheck(object):
  def AnotherNotCheckNorDiagnose(self):
    return -2
'''

ACTION_FILE = '''
def TestAction():
  return True

def AnotherAction():
  return False
'''


class RunCommand(threading.Thread):
  """Helper class for executing the Mob* Monitor with a timeout."""

  def __init__(self, cmd, timeout):
    threading.Thread.__init__(self)
    self.cmd = cmd
    self.timeout = timeout
    self.p = None

    self.proc_stdout = None
    self.proc_stderr = None

  def run(self):
    self.p = subprocess.Popen(self.cmd, stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT)
    self.proc_stdout, self.proc_stderr = self.p.communicate()

  def Stop(self):
    self.join(self.timeout)

    if self.is_alive():
      self.p.terminate()
      self.join(self.timeout)

      if self.is_alive():
        self.p.kill()
        self.join(self.timeout)

    return self.proc_stdout


class CheckFileManagerHelperTest(cros_test_lib.MockTestCase):
  """Unittests for CheckFileManager helper functions."""

  def testMapHealthcheckStatusToDict(self):
    """Test mapping a manager.HEALTHCHECK_STATUS to a dict."""
    def _func():
      pass

    status = manager.HEALTHCHECK_STATUS('test', False, 'desc', [_func])
    expect = {'name': 'test', 'health': False, 'description': 'desc',
              'actions': ['_func']}
    self.assertEquals(expect, manager.MapHealthcheckStatusToDict(status))

  def testMapServiceStatusToDict(self):
    """Test mapping a manager.SERVICE_STATUS to a dict."""
    def _func():
      pass

    hcstatus = manager.HEALTHCHECK_STATUS('test', False, 'desc', [_func])
    hcexpect = {'name': 'test', 'health': False, 'description': 'desc',
                'actions': ['_func']}
    status = manager.SERVICE_STATUS('test-service', False, [hcstatus])
    expect = {'service': 'test-service', 'health': False,
              'healthchecks': [hcexpect]}
    self.assertEquals(expect, manager.MapServiceStatusToDict(status))

  def testMapActionInfoToDict(self):
    """Test mapping a manager.ACTION_INFO to a dict."""
    actioninfo = manager.ACTION_INFO('test', 'test', [1], {'a': 1})
    expect = {'action': 'test', 'info': 'test', 'args': [1],
              'kwargs': {'a': 1}}
    self.assertEquals(expect, manager.MapActionInfoToDict(actioninfo))

  def testIsHealthcheckHealthy(self):
    """Test checking whether health check statuses are healthy."""
    # Test a healthy health check.
    hch = manager.HEALTHCHECK_STATUS('healthy', True, manager.NULL_DESCRIPTION,
                                     manager.EMPTY_ACTIONS)
    self.assertTrue(manager.isHealthcheckHealthy(hch))

    # Test a quasi-healthy health check.
    hcq = manager.HEALTHCHECK_STATUS('quasi-healthy', True, 'Quasi-Healthy',
                                     ['QuasiAction'])
    self.assertFalse(manager.isHealthcheckHealthy(hcq))

    # Test an unhealthy health check.
    hcu = manager.HEALTHCHECK_STATUS('unhealthy', False, 'Unhealthy',
                                     ['UnhealthyAction'])
    self.assertFalse(manager.isHealthcheckHealthy(hcu))

    # Test an object that is not a health check status.
    s = manager.SERVICE_STATUS('service_status', True, [])
    self.assertFalse(manager.isHealthcheckHealthy(s))

  def testIsServiceHealthy(self):
    """Test checking whether service statuses are healthy."""
    # Define some health check statuses.
    hch = manager.HEALTHCHECK_STATUS('healthy', True, manager.NULL_DESCRIPTION,
                                     manager.EMPTY_ACTIONS)
    hcq = manager.HEALTHCHECK_STATUS('quasi-healthy', True, 'Quasi-Healthy',
                                     ['QuasiAction'])
    hcu = manager.HEALTHCHECK_STATUS('unhealthy', False, 'Unhealthy',
                                     ['UnhealthyAction'])

    # Test a healthy service.
    s = manager.SERVICE_STATUS('healthy', True, [])
    self.assertTrue(manager.isServiceHealthy(s))

    # Test a quasi-healthy service.
    s = manager.SERVICE_STATUS('quasi-healthy', True, [hch, hcq])
    self.assertFalse(manager.isServiceHealthy(s))

    # Test an unhealthy service.
    s = manager.SERVICE_STATUS('unhealthy', False, [hcu])
    self.assertFalse(manager.isServiceHealthy(s))

    # Test an object that is not a service status.
    self.assertFalse(manager.isServiceHealthy(hch))

  def testDetermineHealthcheckStatusHealthy(self):
    """Test DetermineHealthCheckStatus on a healthy check."""
    hcname = TestHealthCheck.__name__
    testhc = TestHealthCheck()
    expected = manager.HEALTHCHECK_STATUS(hcname, True,
                                          manager.NULL_DESCRIPTION,
                                          manager.EMPTY_ACTIONS)
    self.assertEquals(expected,
                      manager.DetermineHealthcheckStatus(hcname, testhc))

  def testDeterminHealthcheckStatusUnhealthy(self):
    """Test DetermineHealthcheckStatus on an unhealthy check."""
    hcname = TestHealthCheckUnhealthy.__name__
    testhc = TestHealthCheckUnhealthy()
    desc, actions = testhc.Diagnose(testhc.Check())
    expected = manager.HEALTHCHECK_STATUS(hcname, False, desc, actions)
    self.assertEquals(expected,
                      manager.DetermineHealthcheckStatus(hcname, testhc))

  def testDetermineHealthcheckStatusQuasihealth(self):
    """Test DetermineHealthcheckStatus on a quasi-healthy check."""
    hcname = TestHealthCheckQuasihealthy.__name__
    testhc = TestHealthCheckQuasihealthy()
    desc, actions = testhc.Diagnose(testhc.Check())
    expected = manager.HEALTHCHECK_STATUS(hcname, True, desc, actions)
    self.assertEquals(expected,
                      manager.DetermineHealthcheckStatus(hcname, testhc))

  def testDetermineHealthcheckStatusBrokenCheck(self):
    """Test DetermineHealthcheckStatus raises on a broken health check."""
    hcname = TestHealthCheckBroken.__name__
    testhc = TestHealthCheckBroken()
    result = manager.DetermineHealthcheckStatus(hcname, testhc)

    self.assertEquals(hcname, result.name)
    self.assertFalse(result.health)
    self.assertFalse(result.actions)

  def testIsHealthCheck(self):
    """Test that IsHealthCheck properly asserts the health check interface."""

    class NoAttrs(object):
      """Test health check missing 'check' and 'diagnose' methods."""

    class NoCheckAttr(object):
      """Test health check missing 'check' method."""
      def Diagnose(self, errcode):
        pass

    class NoDiagnoseAttr(object):
      """Test health check missing 'diagnose' method."""
      def Check(self):
        pass

    class GoodHealthCheck(object):
      """Test health check that implements 'check' and 'diagnose' methods."""
      def Check(self):
        pass

      def Diagnose(self, errcode):
        pass

    self.assertFalse(manager.IsHealthCheck(NoAttrs()))
    self.assertFalse(manager.IsHealthCheck(NoCheckAttr()))
    self.assertFalse(manager.IsHealthCheck(NoDiagnoseAttr()))
    self.assertTrue(manager.IsHealthCheck(GoodHealthCheck()))

  def testApplyHealthCheckAttributesNoAttrs(self):
    """Test that we can apply attributes to a health check."""
    testhc = TestHealthCheck()
    result = manager.ApplyHealthCheckAttributes(testhc)
    self.assertEquals(result.CHECK_INTERVAL_SEC,
                      manager.CHECK_INTERVAL_DEFAULT_SEC)

  def testApplyHealthCheckAttributesHasAttrs(self):
    """Test that we do not override an acceptable attribute."""
    testhc = TestHealthCheckHasAttributes()
    check_interval = testhc.CHECK_INTERVAL_SEC
    result = manager.ApplyHealthCheckAttributes(testhc)
    self.assertEquals(result.CHECK_INTERVAL_SEC, check_interval)

  def testImportFileAllHealthChecks(self):
    """Test that health checks and service name are collected."""
    self.StartPatcher(mock.patch('os.path.splitext'))
    os.path.splitext.return_value = '/path/to/test_check.py'

    self.StartPatcher(mock.patch('os.path.getmtime'))
    os.path.getmtime.return_value = TEST_MTIME

    checkmodule = imp.new_module('test_check')
    # pylint: disable=exec-used
    exec CHECKFILE_MANY_SIMPLE in checkmodule.__dict__
    self.StartPatcher(mock.patch('imp.load_source'))
    imp.load_source.return_value = checkmodule

    healthchecks, mtime = manager.ImportFile(TEST_SERVICE_NAME, '/')

    self.assertEquals(len(healthchecks), 3)
    self.assertEquals(mtime, TEST_MTIME)

  def testImportFileSomeHealthChecks(self):
    """Test importing when not all classes are actually health checks."""
    self.StartPatcher(mock.patch('os.path.splitext'))
    os.path.splitext.return_value = '/path/to/test_check.py'

    self.StartPatcher(mock.patch('os.path.getmtime'))
    os.path.getmtime.return_value = TEST_MTIME

    checkmodule = imp.new_module('test_check')
    # pylint: disable=exec-used
    exec CHECKFILE_MANY_SIMPLE_ONE_BAD in checkmodule.__dict__
    self.StartPatcher(mock.patch('imp.load_source'))
    imp.load_source.return_value = checkmodule

    healthchecks, mtime = manager.ImportFile(TEST_SERVICE_NAME, '/')

    self.assertEquals(len(healthchecks), 2)
    self.assertEquals(mtime, TEST_MTIME)


class CheckFileManagerTest(cros_test_lib.MockTestCase):
  """Unittests for CheckFileManager."""

  def testCollectionExecutionCallbackCheckfiles(self):
    """Test the CollectionExecutionCallback on collecting checkfiles."""
    self.StartPatcher(mock.patch('os.walk'))
    os.walk.return_value = iter([[CHECKDIR, [TEST_SERVICE_NAME], []]])

    self.StartPatcher(mock.patch('os.listdir'))
    os.listdir.return_value = ['test_check.py']

    self.StartPatcher(mock.patch('os.path.isfile'))
    os.path.isfile.return_value = True

    self.StartPatcher(mock.patch('imp.find_module'))
    imp.find_module.return_value = (None, None, None)
    self.StartPatcher(mock.patch('imp.load_module'))

    myobj = TestHealthCheck()
    manager.ImportFile = mock.Mock(return_value=[[myobj], TEST_MTIME])
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.CollectionExecutionCallback()

    manager.ImportFile.assert_called_once_with(
        TEST_SERVICE_NAME, './%s/test_check.py' % TEST_SERVICE_NAME)

    self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks)
    self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME],
                      {myobj.__class__.__name__: (TEST_MTIME, myobj)})

  def testCollectionExecutionCallbackNoChecks(self):
    """Test the CollectionExecutionCallback with no valid check files."""
    self.StartPatcher(mock.patch('os.walk'))
    os.walk.return_value = iter([['/checkdir/', [], ['test.py']]])

    manager.ImportFile = mock.Mock(return_value=None)
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.CollectionExecutionCallback()

    self.assertFalse(manager.ImportFile.called)

    self.assertFalse(TEST_SERVICE_NAME in cfm.service_checks)

  def testStartCollectionExecution(self):
    """Test the StartCollectionExecution method."""
    plugins.Monitor = mock.Mock()

    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.StartCollectionExecution()

    self.assertTrue(plugins.Monitor.called)

  def testUpdateExistingHealthCheck(self):
    """Test update when a health check exists and is not stale."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    myobj = TestHealthCheck()

    cfm.service_checks[TEST_SERVICE_NAME] = {myobj.__class__.__name__:
                                             (TEST_MTIME, myobj)}

    myobj2 = TestHealthCheck()
    cfm.Update(TEST_SERVICE_NAME, [myobj2], TEST_MTIME)
    self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks)
    self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME],
                      {myobj.__class__.__name__: (TEST_MTIME, myobj)})

  def testUpdateNonExistingHealthCheck(self):
    """Test adding a new health check to the manager."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.service_checks = {}

    myobj = TestHealthCheck()
    cfm.Update(TEST_SERVICE_NAME, [myobj], TEST_MTIME)

    self.assertTrue(TEST_SERVICE_NAME in cfm.service_checks)
    self.assertEquals(cfm.service_checks[TEST_SERVICE_NAME],
                      {myobj.__class__.__name__: (TEST_MTIME, myobj)})

  def testExecuteFresh(self):
    """Test executing a health check when the result is still fresh."""
    self.StartPatcher(mock.patch('time.time'))
    exec_time_offset = TestHealthCheckHasAttributes.CHECK_INTERVAL_SEC / 2
    time.time.return_value = TEST_EXEC_TIME + exec_time_offset

    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.service_checks = {TEST_SERVICE_NAME:
                          {TestHealthCheckHasAttributes.__name__:
                           (TEST_MTIME, TestHealthCheckHasAttributes())}}
    cfm.service_check_results = {
        TEST_SERVICE_NAME: {TestHealthCheckHasAttributes.__name__:
                            (manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME,
                             None)}}

    cfm.Execute()

    _, exec_time, _ = cfm.service_check_results[TEST_SERVICE_NAME][
        TestHealthCheckHasAttributes.__name__]

    self.assertEquals(exec_time, TEST_EXEC_TIME)

  def testExecuteStale(self):
    """Test executing a health check when the result is stale."""
    self.StartPatcher(mock.patch('time.time'))
    exec_time_offset = TestHealthCheckHasAttributes.CHECK_INTERVAL_SEC * 2
    time.time.return_value = TEST_EXEC_TIME + exec_time_offset

    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.service_checks = {TEST_SERVICE_NAME:
                          {TestHealthCheckHasAttributes.__name__:
                           (TEST_MTIME, TestHealthCheckHasAttributes())}}
    cfm.service_check_results = {
        TEST_SERVICE_NAME: {TestHealthCheckHasAttributes.__name__:
                            (manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME,
                             None)}}

    cfm.Execute()

    _, exec_time, _ = cfm.service_check_results[TEST_SERVICE_NAME][
        TestHealthCheckHasAttributes.__name__]

    self.assertEquals(exec_time, TEST_EXEC_TIME + exec_time_offset)

  def testExecuteNonExistent(self):
    """Test executing a health check when the result is nonexistent."""
    self.StartPatcher(mock.patch('time.time'))
    time.time.return_value = TEST_EXEC_TIME

    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.service_checks = {TEST_SERVICE_NAME:
                          {TestHealthCheck.__name__:
                           (TEST_MTIME, TestHealthCheck())}}

    cfm.Execute()

    resultsdict = cfm.service_check_results.get(TEST_SERVICE_NAME)
    self.assertTrue(resultsdict is not None)

    exec_status, exec_time, _ = resultsdict.get(TestHealthCheck.__name__,
                                                (None, None, None))
    self.assertTrue(exec_status is not None)
    self.assertTrue(exec_time is not None)

    self.assertEquals(exec_status, manager.HCEXECUTION_COMPLETED)
    self.assertEquals(exec_time, TEST_EXEC_TIME)

  def testExecuteForce(self):
    """Test executing a health check by ignoring the check interval."""
    self.StartPatcher(mock.patch('time.time'))
    exec_time_offset = TestHealthCheckHasAttributes.CHECK_INTERVAL_SEC / 2
    time.time.return_value = TEST_EXEC_TIME + exec_time_offset

    cfm = manager.CheckFileManager(checkdir=CHECKDIR)
    cfm.service_checks = {TEST_SERVICE_NAME:
                          {TestHealthCheckHasAttributes.__name__:
                           (TEST_MTIME, TestHealthCheckHasAttributes())}}
    cfm.service_check_results = {
        TEST_SERVICE_NAME: {TestHealthCheckHasAttributes.__name__:
                            (manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME,
                             None)}}

    cfm.Execute(force=True)

    _, exec_time, _ = cfm.service_check_results[TEST_SERVICE_NAME][
        TestHealthCheckHasAttributes.__name__]

    self.assertEquals(exec_time, TEST_EXEC_TIME + exec_time_offset)

  def testConsolidateServiceStatesUnhealthy(self):
    """Test consolidating state for a service with unhealthy checks."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    # Setup some test check results
    hcname = TestHealthCheck.__name__
    statuses = [
        manager.HEALTHCHECK_STATUS(hcname, False, 'Failed', ['Repair']),
        manager.HEALTHCHECK_STATUS(hcname, True, 'Quasi', ['RepairQuasi']),
        manager.HEALTHCHECK_STATUS(hcname, True, '', [])]

    cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {})
    for i, status in enumerate(statuses):
      name = '%s_%s' % (hcname, i)
      cfm.service_check_results[TEST_SERVICE_NAME][name] = (
          manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME,
          status)

    # Run and check the results.
    cfm.ConsolidateServiceStates()
    self.assertTrue(TEST_SERVICE_NAME in cfm.service_states)

    _, health, healthchecks = cfm.service_states[TEST_SERVICE_NAME]
    self.assertFalse(health)
    self.assertEquals(2, len(healthchecks))
    self.assertTrue(all([x in healthchecks for x in statuses[:2]]))

  def testConsolidateServiceStatesQuasiHealthy(self):
    """Test consolidating state for a service with quasi-healthy checks."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    # Setup some test check results
    hcname = TestHealthCheck.__name__
    statuses = [
        manager.HEALTHCHECK_STATUS(hcname, True, 'Quasi', ['RepairQuasi']),
        manager.HEALTHCHECK_STATUS(hcname, True, '', [])]

    cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {})
    for i, status in enumerate(statuses):
      name = '%s_%s' % (hcname, i)
      cfm.service_check_results[TEST_SERVICE_NAME][name] = (
          manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME,
          status)

    # Run and check the results.
    cfm.ConsolidateServiceStates()
    self.assertTrue(TEST_SERVICE_NAME in cfm.service_states)

    _, health, healthchecks = cfm.service_states[TEST_SERVICE_NAME]
    self.assertTrue(health)
    self.assertEquals(1, len(healthchecks))
    self.assertTrue(statuses[0] in healthchecks)

  def testConsolidateServiceStatesHealthy(self):
    """Test consolidating state for a healthy service."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    # Setup some test check results
    hcname = TestHealthCheck.__name__
    hcname2 = '%s_2' % hcname
    statuses = [
        manager.HEALTHCHECK_STATUS(hcname, True, '', []),
        manager.HEALTHCHECK_STATUS(hcname2, True, '', [])]

    cfm.service_check_results.setdefault(TEST_SERVICE_NAME, {})
    cfm.service_check_results[TEST_SERVICE_NAME][hcname] = (
        manager.HCEXECUTION_COMPLETED, TEST_EXEC_TIME, statuses[0])
    cfm.service_check_results[TEST_SERVICE_NAME][hcname2] = (
        manager.HCEXECUTION_IN_PROGRESS, TEST_EXEC_TIME, statuses[1])

    # Run and check.
    cfm.ConsolidateServiceStates()

    self.assertTrue(TEST_SERVICE_NAME in cfm.service_states)

    _, health, healthchecks = cfm.service_states.get(TEST_SERVICE_NAME)
    self.assertTrue(health)
    self.assertEquals(0, len(healthchecks))

  def testGetServiceList(self):
    """Test the GetServiceList RPC response."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    self.assertEquals([], cfm.GetServiceList())

    status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, [])
    cfm.service_states[TEST_SERVICE_NAME] = status

    self.assertEquals([TEST_SERVICE_NAME], cfm.GetServiceList())

  def testGetStatusNonExistent(self):
    """Test the GetStatus RPC response when the service does not exist."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    self.assertFalse(TEST_SERVICE_NAME in cfm.service_states)

    status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [])
    self.assertEquals(status, cfm.GetStatus(TEST_SERVICE_NAME))

  def testGetStatusSingleService(self):
    """Test the GetStatus RPC response for a single service."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    s1name = TEST_SERVICE_NAME
    s2name = '%s_2' % s1name
    status1 = manager.SERVICE_STATUS(s1name, True, [])
    status2 = manager.SERVICE_STATUS(s2name, True, [])
    cfm.service_states[s1name] = status1
    cfm.service_states[s2name] = status2

    self.assertEquals(status1, cfm.GetStatus(s1name))
    self.assertEquals(status2, cfm.GetStatus(s2name))

  def testGetStatusAllServices(self):
    """Test the GetStatus RPC response when no service is specified."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    s1name = TEST_SERVICE_NAME
    s2name = '%s_2' % s1name
    status1 = manager.SERVICE_STATUS(s1name, True, [])
    status2 = manager.SERVICE_STATUS(s2name, True, [])
    cfm.service_states[s1name] = status1
    cfm.service_states[s2name] = status2

    result = cfm.GetStatus('')
    self.assertEquals(2, len(result))
    self.assertTrue(all([x in result for x in [status1, status2]]))

  def testRepairServiceHealthy(self):
    """Test the RepairService RPC when the service is healthy."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    healthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, [])
    cfm.service_states[TEST_SERVICE_NAME] = healthy_status

    self.assertEquals(healthy_status, cfm.RepairService(TEST_SERVICE_NAME,
                                                        'HealthcheckName',
                                                        'RepairFuncName',
                                                        [], {}))

  def testRepairServiceNonExistent(self):
    """Test the RepairService RPC when the service does not exist."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    self.assertFalse(TEST_SERVICE_NAME in cfm.service_states)

    expected = manager.SERVICE_STATUS(TEST_SERVICE_NAME, False, [])
    result = cfm.RepairService(TEST_SERVICE_NAME, 'DummyHealthcheck',
                               'DummyAction', [], {})
    self.assertEquals(expected, result)

  def testRepairServiceInvalidAction(self):
    """Test the RepairService RPC when the action is not recognized."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    hcobj = TestHealthCheckUnhealthy()
    cfm.service_checks[TEST_SERVICE_NAME] = {
        hcobj.__class__.__name__: (TEST_MTIME, hcobj)}

    unhealthy_status = manager.SERVICE_STATUS(
        TEST_SERVICE_NAME, False,
        [manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__,
                                    False, 'Always fails', [hcobj.Repair])])
    cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status

    status = cfm.GetStatus(TEST_SERVICE_NAME)
    self.assertFalse(status.health)
    self.assertEquals(1, len(status.healthchecks))

    status = cfm.RepairService(TEST_SERVICE_NAME, hcobj.__class__.__name__,
                               'Blah', [], {})
    self.assertFalse(status.health)
    self.assertEquals(1, len(status.healthchecks))

  def testRepairServiceInvalidActionArguments(self):
    """Test the RepairService RPC when the action arguments are invalid."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    hcobj = TestHealthCheckUnhealthy()
    cfm.service_checks[TEST_SERVICE_NAME] = {
        hcobj.__class__.__name__: (TEST_MTIME, hcobj)}

    unhealthy_status = manager.SERVICE_STATUS(
        TEST_SERVICE_NAME, False,
        [manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__,
                                    False, 'Always fails', [hcobj.Repair])])
    cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status

    status = cfm.GetStatus(TEST_SERVICE_NAME)
    self.assertFalse(status.health)
    self.assertEquals(1, len(status.healthchecks))

    status = cfm.RepairService(TEST_SERVICE_NAME, hcobj.__class__.__name__,
                               'Repair', [1, 2, 3], {})
    self.assertFalse(status.health)
    self.assertEquals(1, len(status.healthchecks))

  def testRepairService(self):
    """Test the RepairService RPC to repair an unhealthy service."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    hcobj = TestHealthCheckUnhealthy()
    cfm.service_checks[TEST_SERVICE_NAME] = {
        hcobj.__class__.__name__: (TEST_MTIME, hcobj)}

    unhealthy_status = manager.SERVICE_STATUS(
        TEST_SERVICE_NAME, False,
        [manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__,
                                    False, 'Always fails', [hcobj.Repair])])
    cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status

    status = cfm.GetStatus(TEST_SERVICE_NAME)
    self.assertFalse(status.health)
    self.assertEquals(1, len(status.healthchecks))

    status = cfm.RepairService(TEST_SERVICE_NAME,
                               hcobj.__class__.__name__,
                               hcobj.Repair.__name__,
                               [], {})
    self.assertTrue(status.health)
    self.assertEquals(0, len(status.healthchecks))

  def testActionInfoServiceNonExistent(self):
    """Test the ActionInfo RPC when the service does not exist."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    self.assertFalse(TEST_SERVICE_NAME in cfm.service_states)

    expect = manager.ACTION_INFO('test', 'Service not recognized.',
                                 [], {})
    result = cfm.ActionInfo(TEST_SERVICE_NAME, 'test', 'test')
    self.assertEquals(expect, result)

  def testActionInfoServiceHealthy(self):
    """Test the ActionInfo RPC when the service is healthy."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    healthy_status = manager.SERVICE_STATUS(TEST_SERVICE_NAME, True, [])
    cfm.service_states[TEST_SERVICE_NAME] = healthy_status

    expect = manager.ACTION_INFO('test', 'Service is healthy.',
                                 [], {})
    result = cfm.ActionInfo(TEST_SERVICE_NAME, 'test', 'test')
    self.assertEquals(expect, result)

  def testActionInfoActionNonExistent(self):
    """Test the ActionInfo RPC when the action does not exist."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    hcobj = TestHealthCheckUnhealthy()
    cfm.service_checks[TEST_SERVICE_NAME] = {
        hcobj.__class__.__name__: (TEST_MTIME, hcobj)}

    unhealthy_status = manager.SERVICE_STATUS(
        TEST_SERVICE_NAME, False,
        [manager.HEALTHCHECK_STATUS(hcobj.__class__.__name__,
                                    False, 'Always fails', [hcobj.Repair])])
    cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status

    expect = manager.ACTION_INFO('test', 'Action not recognized.', [], {})
    result = cfm.ActionInfo(TEST_SERVICE_NAME, hcobj.__class__.__name__,
                            'test')
    self.assertEquals(expect, result)

  def testActionInfo(self):
    """Test the ActionInfo RPC to collect information on a repair action."""
    cfm = manager.CheckFileManager(checkdir=CHECKDIR)

    hcobj = TestHealthCheckMultipleActions()
    hcname = hcobj.__class__.__name__
    actions = [hcobj.NoParams, hcobj.PositionalParams, hcobj.DefaultParams,
               hcobj.MixedParams]

    cfm.service_checks[TEST_SERVICE_NAME] = {hcname: (TEST_MTIME, hcobj)}

    unhealthy_status = manager.SERVICE_STATUS(
        TEST_SERVICE_NAME, False,
        [manager.HEALTHCHECK_STATUS(hcname, False, 'Always fails', actions)])
    cfm.service_states[TEST_SERVICE_NAME] = unhealthy_status

    # Test ActionInfo when the action has no parameters.
    expect = manager.ACTION_INFO('NoParams', 'NoParams Action.', [], {})
    self.assertEquals(expect,
                      cfm.ActionInfo(TEST_SERVICE_NAME, hcname, 'NoParams'))

    # Test ActionInfo when the action has only positional parameters.
    expect = manager.ACTION_INFO('PositionalParams', 'PositionalParams Action.',
                                 ['x', 'y', 'z'], {})
    self.assertEquals(expect,
                      cfm.ActionInfo(TEST_SERVICE_NAME,
                                     hcname, 'PositionalParams'))

    # Test ActionInfo when the action has only default parameters.
    expect = manager.ACTION_INFO('DefaultParams', 'DefaultParams Action.',
                                 [], {'x': 1, 'y': 2, 'z': 3})
    self.assertEquals(expect,
                      cfm.ActionInfo(TEST_SERVICE_NAME,
                                     hcname, 'DefaultParams'))

    # Test ActionInfo when the action has positional and default parameters.
    expect = manager.ACTION_INFO('MixedParams', 'MixedParams Action.',
                                 ['x', 'y'], {'z': 1})
    self.assertEquals(expect, cfm.ActionInfo(TEST_SERVICE_NAME,
                                             hcname, 'MixedParams'))