crosperf/experiment_file_unittest.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Copyright 2011 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""The unittest of experiment_file."""

import io
import unittest

from experiment_file import ExperimentFile


EXPERIMENT_FILE_1 = """
  board: x86-alex
  remote: chromeos-alex3
  perf_args: record -a -e cycles
  benchmark: PageCycler {
    iterations: 3
  }

  image1 {
    chromeos_image: /usr/local/google/cros_image1.bin
  }

  image2 {
    remote: chromeos-lumpy1
    chromeos_image: /usr/local/google/cros_image2.bin
  }
  """

EXPERIMENT_FILE_2 = """
  board: x86-alex
  remote: chromeos-alex3
  iterations: 3

  benchmark: PageCycler {
  }

  benchmark: AndroidBench {
    iterations: 2
  }

  image1 {
    chromeos_image:/usr/local/google/cros_image1.bin
  }

  image2 {
    chromeos_image: /usr/local/google/cros_image2.bin
  }
  """

EXPERIMENT_FILE_3 = """
  board: x86-alex
  remote: chromeos-alex3
  iterations: 3

  benchmark: PageCycler {
  }

  image1 {
    chromeos_image:/usr/local/google/cros_image1.bin
  }

  image1 {
    chromeos_image: /usr/local/google/cros_image2.bin
  }
  """

EXPERIMENT_FILE_4 = """
  board: x86-alex
  remote: chromeos-alex3
  iterations: 3

  benchmark: webrtc {
    test_args: --story-filter=datachannel
  }

  benchmark: webrtc {
    test_args: --story-tag-filter=smoothness
  }

  image1 {
    chromeos_image:/usr/local/google/cros_image1.bin
  }
  """

DUT_CONFIG_EXPERIMENT_FILE_GOOD = """
  board: kevin64
  remote: chromeos-kevin.cros
  turbostat: False
  intel_pstate: no_hwp
  cooldown_temp: 38
  cooldown_time: 5
  governor: powersave
  cpu_usage: exclusive_cores
  cpu_freq_pct: 50
  top_interval: 5

  benchmark: speedometer {
    iterations: 3
    suite: telemetry_Crosperf
  }

  image1 {
    chromeos_image:/usr/local/google/cros_image1.bin
  }
  """

DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV = """
  board: kevin64
  remote: chromeos-kevin.cros
  intel_pstate: active
  governor: misspelled_governor

  benchmark: speedometer2 {
    iterations: 3
    suite: telemetry_Crosperf
  }
  """

DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE = """
  board: kevin64
  remote: chromeos-kevin.cros
  turbostat: False
  governor: ondemand
  cpu_usage: unknown

  benchmark: speedometer2 {
    iterations: 3
    suite: telemetry_Crosperf
  }

  image1 {
    chromeos_image:/usr/local/google/cros_image1.bin
  }
  """

OUTPUT_FILE = """board: x86-alex
remote: chromeos-alex3
perf_args: record -a -e cycles

benchmark: PageCycler {
\titerations: 3
}

label: image1 {
\tchromeos_image: /usr/local/google/cros_image1.bin
\tremote: chromeos-alex3
}

label: image2 {
\tchromeos_image: /usr/local/google/cros_image2.bin
\tremote: chromeos-lumpy1
}\n\n"""


class ExperimentFileTest(unittest.TestCase):
    """The main class for Experiment File test."""

    def testLoadExperimentFile1(self):
        input_file = io.StringIO(EXPERIMENT_FILE_1)
        experiment_file = ExperimentFile(input_file)
        global_settings = experiment_file.GetGlobalSettings()
        self.assertEqual(global_settings.GetField("remote"), ["chromeos-alex3"])
        self.assertEqual(
            global_settings.GetField("perf_args"), "record -a -e cycles"
        )
        benchmark_settings = experiment_file.GetSettings("benchmark")
        self.assertEqual(len(benchmark_settings), 1)
        self.assertEqual(benchmark_settings[0].name, "PageCycler")
        self.assertEqual(benchmark_settings[0].GetField("iterations"), 3)

        label_settings = experiment_file.GetSettings("label")
        self.assertEqual(len(label_settings), 2)
        self.assertEqual(label_settings[0].name, "image1")
        self.assertEqual(
            label_settings[0].GetField("chromeos_image"),
            "/usr/local/google/cros_image1.bin",
        )
        self.assertEqual(
            label_settings[1].GetField("remote"), ["chromeos-lumpy1"]
        )
        self.assertEqual(
            label_settings[0].GetField("remote"), ["chromeos-alex3"]
        )

    def testOverrideSetting(self):
        input_file = io.StringIO(EXPERIMENT_FILE_2)
        experiment_file = ExperimentFile(input_file)
        global_settings = experiment_file.GetGlobalSettings()
        self.assertEqual(global_settings.GetField("remote"), ["chromeos-alex3"])

        benchmark_settings = experiment_file.GetSettings("benchmark")
        self.assertEqual(len(benchmark_settings), 2)
        self.assertEqual(benchmark_settings[0].name, "PageCycler")
        self.assertEqual(benchmark_settings[0].GetField("iterations"), 3)
        self.assertEqual(benchmark_settings[1].name, "AndroidBench")
        self.assertEqual(benchmark_settings[1].GetField("iterations"), 2)

    def testDuplicateLabel(self):
        input_file = io.StringIO(EXPERIMENT_FILE_3)
        self.assertRaises(Exception, ExperimentFile, input_file)

    def testDuplicateBenchmark(self):
        input_file = io.StringIO(EXPERIMENT_FILE_4)
        experiment_file = ExperimentFile(input_file)
        benchmark_settings = experiment_file.GetSettings("benchmark")
        self.assertEqual(benchmark_settings[0].name, "webrtc")
        self.assertEqual(
            benchmark_settings[0].GetField("test_args"),
            "--story-filter=datachannel",
        )
        self.assertEqual(benchmark_settings[1].name, "webrtc")
        self.assertEqual(
            benchmark_settings[1].GetField("test_args"),
            "--story-tag-filter=smoothness",
        )

    def testCanonicalize(self):
        input_file = io.StringIO(EXPERIMENT_FILE_1)
        experiment_file = ExperimentFile(input_file)
        res = experiment_file.Canonicalize()
        self.assertEqual(res, OUTPUT_FILE)

    def testLoadDutConfigExperimentFile_Good(self):
        input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_GOOD)
        experiment_file = ExperimentFile(input_file)
        global_settings = experiment_file.GetGlobalSettings()
        self.assertEqual(global_settings.GetField("turbostat"), False)
        self.assertEqual(global_settings.GetField("intel_pstate"), "no_hwp")
        self.assertEqual(global_settings.GetField("governor"), "powersave")
        self.assertEqual(
            global_settings.GetField("cpu_usage"), "exclusive_cores"
        )
        self.assertEqual(global_settings.GetField("cpu_freq_pct"), 50)
        self.assertEqual(global_settings.GetField("cooldown_time"), 5)
        self.assertEqual(global_settings.GetField("cooldown_temp"), 38)
        self.assertEqual(global_settings.GetField("top_interval"), 5)

    def testLoadDutConfigExperimentFile_WrongGovernor(self):
        input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_GOV)
        with self.assertRaises(RuntimeError) as msg:
            ExperimentFile(input_file)
        self.assertRegex(str(msg.exception), "governor: misspelled_governor")
        self.assertRegex(
            str(msg.exception),
            "Invalid enum value for field 'governor'."
            r" Must be one of \(performance, powersave, userspace, ondemand,"
            r" conservative, schedutils, sched, interactive\)",
        )

    def testLoadDutConfigExperimentFile_WrongCpuUsage(self):
        input_file = io.StringIO(DUT_CONFIG_EXPERIMENT_FILE_BAD_CPUUSE)
        with self.assertRaises(RuntimeError) as msg:
            ExperimentFile(input_file)
        self.assertRegex(str(msg.exception), "cpu_usage: unknown")
        self.assertRegex(
            str(msg.exception),
            "Invalid enum value for field 'cpu_usage'."
            r" Must be one of \(all, big_only, little_only, exclusive_cores\)",
        )


if __name__ == "__main__":
    unittest.main()