contrib/power_qual_compare.py

#!/usr/bin/env python3

"""Read two test logs and report differences in power consumption."""
# Copyright 2022 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Parse logs of 2 runs of PowerQual.full and compare them printing problems.
# Based on a script by jacobraz@google.com

from io import StringIO
import json

# Additions:
# - Collect energy ratings and compare
import os
from pathlib import Path
import sys
import time

# pylint: disable=import-error
from colorama import Fore
from colorama import Style


SUBSYSTEMS = ["PL1", "uncore", "dram", "core", "package-0", "psys"]
ENERGY_RATINGS = ["w_energy_rate", "wh_energy_used"]


def get_subsystem(val):
    """Return valid subsystem from given string or None."""
    for s in SUBSYSTEMS:
        if s in val:
            return s

    return None


def get_energy_metric(name):
    """Return valid energy metric or None."""
    if name in ENERGY_RATINGS:
        return name
    return None


def color_number(value):
    """Return the number as a string with color formatting"""
    if value == "NA" or value < 0:
        color = Fore.RED
    else:
        color = Fore.GREEN
    return (color + "{}" + Style.RESET_ALL).format(value)


def get_test_name(fname):
    """Read the test name from the JSON file, if we can find it."""
    json_file = Path(os.path.join(os.path.dirname(fname), "results.json"))
    if not json_file.exists():
        return ""
    try:
        results = json.loads(open(json_file).read())
        return results["tests"][0]["testname"]
    # pylint: disable=broad-except
    except Exception:
        return ""


def read_report(fname):
    """Read a test log and return a dictionary of metrics.

    There are two kinds of allowed  metrics:
    - A set of subsystems
    - A set of energy usage metrics
    The returned dictionary will have keys for both, if they are present
    in the log.
    """

    def is_float(val):
        try:
            float(val)
            return True
        except ValueError:
            return False

    def is_valid_name(subsystem, metric):
        if subsystem is None:
            return False
        if get_energy_metric(metric):
            return True

        if "temp" in metric:
            return False
        if "avg" in metric:
            return True

        return False

    metrics = {}
    with open(fname, "r") as results:
        for line in results.readlines():
            if not line:
                break
            arr = line.split()
            if not (len(arr) == 3 and is_float(arr[2])):
                continue
            metric_name = arr[1]
            if get_subsystem(metric_name):
                subsys_name = get_subsystem(metric_name)
            else:
                subsys_name = get_energy_metric(metric_name)
            if not is_valid_name(subsys_name, metric_name):
                # ignore max min std cnt and temperature measurements
                continue
            metrics[metric_name] = float(arr[2])
    return metrics


def main():
    """Read the files and create the report."""
    usage = """./power_qual_compare.py /path/to/old-result/test_report.log
            /path/to/kernelnext-result/test_report.log"""
    if len(sys.argv) != 3:
        print(usage)
        sys.exit(0)

    old_file = sys.argv[1]
    new_file = sys.argv[2]

    metric_diffs = {}
    subsys_diffs = {s: [] for s in SUBSYSTEMS}

    test_name = get_test_name(new_file)
    metrics_old = read_report(old_file)
    metric_diffs = dict(metrics_old)
    metrics_new = read_report(new_file)

    for metric_name, new_val in metrics_new.items():
        if metric_name not in metric_diffs:
            continue
        metric_diffs[metric_name] -= new_val
        subsys_name = get_subsystem(metric_name)
        if subsys_name is None:
            continue
        if new_val != 0:
            subsys_diffs[subsys_name].append(
                metric_diffs[metric_name] / new_val
            )
        elif metric_diffs[metric_name] != 0:
            # there is a difference but cant divide by kn value since its 0
            # use 100% change if one value is 0
            subsys_diffs[subsys_name].append(100)

    # Compute differences in w_energy_rate and wh_energy_used
    try:
        wer_diff = metrics_old["w_energy_rate"] - metrics_new["w_energy_rate"]
    except KeyError:
        # In case w_energy_rate is missing in either report
        wer_diff = "NA"

    try:
        whe_diff = metrics_old["wh_energy_used"] - metrics_new["wh_energy_used"]
    except KeyError:
        # In case w_energy_rate is missing in either report
        whe_diff = "NA"

    def summary_print(buf, format_fn):
        # if the diff is negative, the kernel-next used more power
        buf.write(f"******************* Test: {test_name} ")
        buf.write("ENERGY RATINGS ********************\n")
        buf.write("w_energy_rate difference: " + format_fn(wer_diff) + "\n")
        buf.write("wh_energy_used difference: " + format_fn(whe_diff) + "\n")
        buf.write(f"******************* Test: {test_name} ")
        buf.write("SUB SYSTEMS************************\n")
        buf.write("Average percent change in each subsystem ")
        buf.write("(Positive values indicate better performance)\n")
        for subsys in SUBSYSTEMS:
            if len(subsys_diffs[subsys]) == 0:
                continue
            buf.write(subsys)
            buf.write(": ")
            avg_percent_diff = (
                sum(subsys_diffs[subsys]) / len(subsys_diffs[subsys]) * 100
            )
            buf.write(format_fn(avg_percent_diff))
            buf.write("\n")

    summarybuf = StringIO()
    # Print a summary of statistic to standard out
    summary_print(summarybuf, color_number)

    print(summarybuf.getvalue())

    # Dump summary to file as well
    suffix = test_name + "_" + str(int(time.time())) + ".txt"
    with open("summary_" + suffix, "w") as summary:
        summary_print(summary, str)

    with open("diffs_" + suffix, "w") as diffs:
        diffs.write("Diff of each individual metric:\n")
        for metric in list(metric_diffs.keys()):
            diffs.write(metric)
            diffs.write(": ")
            diffs.write(str(metric_diffs[metric]))
            diffs.write("\n")


if __name__ == "__main__":
    main()