server/site_tests/suites/control.regression

# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

AUTHOR = "Chrome OS Team"
NAME = "Regression"
TIME = "LONG"
TEST_CATEGORY = "Functional"
TEST_CLASS = "suite"
TEST_TYPE = "server"

DOC = """
This test suite runs automated tests that should all pass and that, together
with the control.bvt suite, verify that all major features in the build are
fully functional.  These are meant to run in less than 8 hours.
"""

from autotest_lib.server import site_host_attributes

# List of client tests with parameters
TESTS = [
  ('compilebench', {}),
  ('dbench', {}),
  ('desktopui_FontCache', {}),
  ('desktopui_GTK2Config', {}),
  ('desktopui_IBusTest', {}),
# TODO(dalecurtis): desktopui_ImeTest is broken; see http://crosbug.com/12646
#  ('desktopui_ImeTest', {}),
  ('desktopui_SunSpiderBench', {}),
  ('desktopui_V8Bench', {}),
  ('desktopui_WindowManagerHotkeys', {}),
  ('disktest', {}),
  ('fsx', {}),
  ('graphics_GLAPICheck', {}),
  ('graphics_GLBench', {}),
  ('graphics_SanAngeles', {'creds': '$backdoor'}),
  ('graphics_WindowManagerGraphicsCapture', {}),
  ('hackbench', {}),
  ('hardware_Backlight', {}),
  ('hardware_DiskSize', {}),
  ('hardware_MemoryThroughput', {'num_iteration': 1,
                                 'test_list': '18'}),
  ('hardware_MemoryTotalSize', {}),
  ('hardware_SAT', {'seconds': 20}),
  ('hardware_StorageFio', {}),
  ('login_BadAuthentication', {'tag' : 'not_verified',
                               'error_string' : 'NotVerified'}),
  ('login_BadAuthentication', {'tag' : 'no_terms',
                               'error_string' : 'TermsNotAgreed'}),
  ('login_BadAuthentication', {'tag' : 'account_deleted',
                               'error_string' : 'AccountDeleted'}),
  ('login_BadAuthentication', {'tag' : 'account_disabled',
                               'error_string' : 'AccountDisabled'}),
  ('login_BadAuthentication', {'tag' : 'service_disabled',
                               'error_string' : 'ServiceDisabled'}),
  ('login_BadAuthentication', {'tag' : 'service_unavailable',
                               'error_string' : 'ServiceUnavailable'}),
  ('login_BadAuthentication', {'tag' : 'unknown',
                               'error_string' : 'Unknown'}),
  ('login_OwnershipApi', {}),
  ('login_DBusCalls', {}),
# TODO(kmixter): Fix log volume usage, or change test constraints. See http://crosbug.com/11350
#  ('logging_LogVolume', {'constraints': ['bytes_unexpected <= 100000',
#                                         'files_unexpected <= 50',
#                                         'percent_stateful_used <= 50',
#                                         'percent_unused_patterns <= 70']}),
#
# TODO: Until ltp can pass on chromiumos
#  ('ltp', {}),
# TODO(stanlyew): Re-open network_ConnmanCromoCrash when crosbug.com/13272
#                 is fixed.
#  ('network_ConnmanCromoCrash', {}),
  ('network_UdevRename', {}),
  ('platform_AccurateTime', {'constraints': ['seconds_offset < 60']}),
  ('platform_CleanShutdown', {}),
  ('platform_DaemonsRespawn', {}),
  ('platform_DiskIterate', {}),
  ('platform_FileNum', {}),
  ('platform_FileSize', {}),
  ('platform_KernelVersion', {}),
  ('platform_MemCheck', {}),
  ('platform_MiniJailCmdLine', {}),
  ('platform_MiniJailPidNamespace', {}),
  ('platform_MiniJailRootCapabilities', {}),
  ('platform_MiniJailVfsNamespace', {}),
  ('platform_ProcessPrivileges', {}),
  ('platform_Rootdev', {}),
  ('platform_SessionManagerTerm', {}),
# TODO(jimhebert): Restore StackProtector after http://crosbug.com/13444
#  ('platform_StackProtector', {}),
  ('platform_TempFS', {}),
# TODO(dalecurtis): AudioBench does not work; see http://crosbug.com/12647
#  ('realtimecomm_GTalkAudioBench', {}),
  ('realtimecomm_GTalkunittest', {}),
  ('unixbench', {}),
]

# A dict which holds {machine: [(test, argv), (test, argv), ...]}
test_dict = {}

# Assign tests with its argv to machines in a round-robin way.
index = 0
for test_with_argv in TESTS:
  m_index = index % len(machines)
  test_list = test_dict.setdefault(machines[m_index], [])
  test_list.append(test_with_argv)
  index += 1


def run_assigned_tests(machine):
  client = hosts.create_host(machine)
  client_at = autotest.Autotest(client)
  test_list = test_dict[machine]
  for test, argv in test_list:
    client_at.run_test(test, **argv)

  client_attributes = site_host_attributes.HostAttributes(machine)
  if client_attributes.has_ssd:
    client_at.run_test('hardware_SsdDetection')
  if client_attributes.has_bluetooth:
    client_at.run_test('network_DisableInterface',
                       iface_name='hci0', tag='hci0')
  if client_attributes.has_80211n:
    client_at.run_test('network_WiFiCaps')
  if not client_attributes.has_resume_bug:
    client_at.run_test('power_Resume')

# TODO(msb): re-enable KernelErrorPaths, see http://crosbug.com/7679
# job.run_test("platform_KernelErrorPaths", host=client)
  if client_attributes.has_chromeos_firmware:
    job.run_test("logging_KernelCrashServer", host=client)

  if client_attributes.has_chromeos_firmware:
    client_at.run_test('hardware_TPMCheck')

job.parallel_on_machines(run_assigned_tests, machines)


# Run server-side suites
job.run_control('server/site_tests/suites/control.security')