buildbot_test_llvm.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2017 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Script for running llvm validation tests on ChromeOS.

This script launches a buildbot to build ChromeOS with the llvm on
a particular board; then it finds and downloads the trybot image and the
corresponding official image, and runs test for correctness.
It then generates a report, emails it to the c-compiler-chrome, as
well as copying the result into a directory.
"""

# Script to test different toolchains against ChromeOS benchmarks.


import argparse
import datetime
import os
import sys
import time

from cros_utils import buildbot_utils
from cros_utils import command_executer
from cros_utils import logger


CROSTC_ROOT = "/usr/local/google/crostc"
ROLE_ACCOUNT = "mobiletc-prebuild"
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
MAIL_PROGRAM = "~/var/bin/mail-detective"
VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, "validation_result")
START_DATE = datetime.date(2016, 1, 1)
TEST_PER_DAY = 4
DATA_DIR = "/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/"

# Information about Rotating Boards
#  Board         Arch     Reference    Platform      Kernel
#                         Board                      Version
#  ------------  -------  ------------ ------------- -------
#  atlas         x86_64   poppy        kabylake-y    4.4.*
#  cave          x86_64   glados       skylake-y     3.18
#  coral         x86_64   reef         apollo lake   4.4.*
#  cyan          x86_64   strago       braswell      3.18
#  elm           aarch64  oak          mediatek-8173 3.18 arm32 userspace
#  eve           x86_64   poppy        kabylake-u/r  4.4.*
#  gale          armv7                               3.18
#  grunt         x86_64   grunt        stoney ridge  4.14.*
#  fizz-moblab   x86_64                              4.4
#  kevin         aarch64  gru          rockchip-3399 4.4.* arm32 userspace
#  kevin64       aarch64  gru          rockchip-3399 4.4.* arm64 userspace
#  lakitu        x86_64                              4.4.*
#  nyan_kitty    armv7    nyan         tegra         3.10.18
#  octopus       x86_64   octopus      GLK           4.14.*
#  sentry        x86_64   kunimitsu    skylake-u     3.18
#  tidus         x86_64   auron        broadwell     3.14
#  veyron_mighty armv7    veyron-pinky rockchip-3288 3.14
#  whirlwind     armv7                               3.14
#  winky         x86_64   rambi        baytrail      4.4.*

TEST_BOARD = [
    "atlas",
    "cave",
    "coral",
    "cyan",
    "elm",
    # 'eve', tested by amd64-llvm-next-toolchain builder.
    "gale",
    "grunt",
    "fizz-moblab",
    # 'kevin', tested by arm64-llvm-next-toolchain builder.
    "kevin64",
    "lakitu",
    "nyan_kitty",
    "octopus",
    "sentry",
    "tidus",
    # 'veyron_mighty', tested by arm-llvm-next-toolchain builder.
    "whirlwind",
    "winky",
]


class ToolchainVerifier(object):
    """Class for the toolchain verifier."""

    def __init__(self, board, chromeos_root, weekday, patches, compiler):
        self._board = board
        self._chromeos_root = chromeos_root
        self._base_dir = os.getcwd()
        self._ce = command_executer.GetCommandExecuter()
        self._l = logger.GetLogger()
        self._compiler = compiler
        self._build = "%s-%s-toolchain-tryjob" % (board, compiler)
        self._patches = patches.split(",") if patches else []
        self._patches_string = "_".join(str(p) for p in self._patches)

        if not weekday:
            self._weekday = time.strftime("%a")
        else:
            self._weekday = weekday
        self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)

    def DoAll(self):
        """Main function inside ToolchainComparator class.

        Launch trybot, get image names, create crosperf experiment file, run
        crosperf, and copy images into seven-day report directories.
        """
        buildbucket_id, _ = buildbot_utils.GetTrybotImage(
            self._chromeos_root,
            self._build,
            self._patches,
            tryjob_flags=["--hwtest"],
            asynchronous=True,
        )

        return buildbucket_id


def WriteRotatingReportsData(results_dict, date):
    """Write data for waterfall report."""
    fname = "%d-%02d-%02d.builds" % (date.year, date.month, date.day)
    filename = os.path.join(DATA_DIR, "rotating-builders", fname)
    with open(filename, "w", encoding="utf-8") as out_file:
        for board in results_dict.keys():
            buildbucket_id = results_dict[board]
            out_file.write("%s,%s\n" % (buildbucket_id, board))


def Main(argv):
    """The main function."""

    # Common initializations
    command_executer.InitCommandExecuter()
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--chromeos_root",
        dest="chromeos_root",
        help="The chromeos root from which to run tests.",
    )
    parser.add_argument(
        "--weekday",
        default="",
        dest="weekday",
        help="The day of the week for which to run tests.",
    )
    parser.add_argument(
        "--board", default="", dest="board", help="The board to test."
    )
    parser.add_argument(
        "--patch",
        dest="patches",
        default="",
        help="The patches to use for the testing, "
        "seprate the patch numbers with ',' "
        "for more than one patches.",
    )
    parser.add_argument(
        "--compiler",
        dest="compiler",
        help="Which compiler (llvm, llvm-next or gcc) to use for " "testing.",
    )

    options = parser.parse_args(argv[1:])
    if not options.chromeos_root:
        print("Please specify the ChromeOS root directory.")
        return 1
    if not options.compiler:
        print(
            "Please specify which compiler to test (gcc, llvm, or llvm-next)."
        )
        return 1

    if options.board:
        fv = ToolchainVerifier(
            options.board,
            options.chromeos_root,
            options.weekday,
            options.patches,
            options.compiler,
        )
        return fv.DoAll()

    today = datetime.date.today()
    delta = today - START_DATE
    days = delta.days

    start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
    results_dict = dict()
    for i in range(TEST_PER_DAY):
        try:
            board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
            fv = ToolchainVerifier(
                board,
                options.chromeos_root,
                options.weekday,
                options.patches,
                options.compiler,
            )
            buildbucket_id = fv.DoAll()
            if buildbucket_id:
                results_dict[board] = buildbucket_id
        except SystemExit:
            logfile = os.path.join(
                VALIDATION_RESULT_DIR, options.compiler, board
            )
            with open(logfile, "w", encoding="utf-8") as f:
                f.write("Verifier got an exception, please check the log.\n")
    WriteRotatingReportsData(results_dict, today)


if __name__ == "__main__":
    retval = Main(sys.argv)
    sys.exit(retval)