contrib/cvetriager/bin/triage_test

#!/usr/bin/env python3
# Copyright 2020 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tests bin/triage."""

import csv
import datetime
import json
import logging
import os
import subprocess
import sys

from cvelib import common
from cvelib import logutils


LOGGER = logutils.setuplogging(loglvl=logging.INFO, name="triage test")
KERNELS = ["v5.4", "v4.19", "v4.14", "v4.4", "v3.18", "v3.14", "v3.10", "v3.8"]
BUG_ID = "123"

FAIL_STATUS = True
PASS_STATUS = False


def get_test_cmd(cve_num):
    """Returns triage command."""
    return ["triage", cve_num, "--bug", BUG_ID, "--debug", "--test", "--json"]


def delete_branch(kernel, kernel_path, bug_id):
    """Deletes branch."""
    branch = common.get_cherry_pick_branch(bug_id, kernel)
    cros_branch = common.get_cros_branch(kernel)
    common.do_checkout(kernel, f"cros/{cros_branch}", kernel_path)
    try:
        subprocess.check_call(
            ["git", "branch", "-D", branch],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            cwd=kernel_path,
        )
    except subprocess.CalledProcessError:
        pass


def kernel_status_check(cve_num, commit, results, expected):
    """Compare returned results to expected results."""
    test_failed = PASS_STATUS

    for kern in KERNELS:
        delete_branch(
            kern, os.path.join(os.getenv("CHROMIUMOS_KERNEL"), kern), BUG_ID
        )

        if results["status"][commit][kern] != expected[kern]:
            LOGGER.error(
                f"Results did not match for {cve_num} with {commit} for {kern}"
            )
            test_failed = FAIL_STATUS

    return test_failed


def do_one_test(cve_num, commit, expected):
    """Perform one test and return PASS_STATUS or FAIL_STATUS."""
    output = None

    try:
        output = subprocess.check_output(get_test_cmd(cve_num))
    except subprocess.CalledProcessError:
        # If "triage" fails, but there were commits expected in the csv
        # file, we have a test failure. So, log the error.
        if commit == "No commits found":
            return PASS_STATUS
        LOGGER.error(f"triage for {cve_num} failed.")
        return FAIL_STATUS

    got = json.loads(output)
    return kernel_status_check(cve_num, commit, got, expected)


def main(args):
    """Main."""
    if len(args) != 1:
        print("Usage: triage_test <path_to_csv_file>")
        sys.exit(1)

    filename = args[0]
    failed_tests = []

    start = datetime.datetime.now()
    with open(filename, "r") as csvfile:
        csvreader = csv.DictReader(csvfile)
        for expected in csvreader:
            cve_num = expected["CVE #"]
            commit = expected["Commit"]

            LOGGER.info(f"Test for {cve_num}")
            status = do_one_test(cve_num, commit, expected)
            if status == FAIL_STATUS:
                failed_tests.append(cve_num)

            LOGGER.info("----------------------------")
    end = datetime.datetime.now()
    LOGGER.info(
        "Running all tests took %d minutes to test",
        (end - start).total_seconds() / 60,
    )

    if failed_tests:
        LOGGER.error("Failed tests are ")
        [LOGGER.error(i) for i in failed_tests]
        return 1

    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))