blob: 2b01d574b1c7f01fe48628e894c2b2ad18d95ef0 [file] [log] [blame]
// Copyright 2019 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
"firmware/internal/pkg/monorail"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
)
// daysSinceRecentCommit specifies how far to look back when determining
// who recently committed changes to a test.
const daysSinceRecentCommit = 60
// gitLogAuthorEmailRegexp captures an author's email address from git log.
var gitLogAuthorEmailRegexp = regexp.MustCompile(`Author:[^<]+<([^>]+)>`)
// controlFileAuthorRegexp captures the AUTHOR line from a test control file.
var controlFileAuthorRegexp = regexp.MustCompile(`AUTHOR = "+([^"]+)"`)
// controlFilePurposeRegexp captures the PURPOSE line from a test control file.
var controlFilePurposeRegexp = regexp.MustCompile(`PURPOSE = "+([^"]+)"`)
// e2eTestClassFirmware and e2eTestClassCr50 contain the names of Autotest classes
// which indicate that a site_test is a firmware test.
const (
e2eTestClassFirmware = "FirmwareTest"
e2eTestClassCr50 = "Cr50Test"
)
// autotestDir is a filepath to the root directory of autotest.
var autotestDir string
// siteTestsDir is a filepath to autotestDir/server/site_tests/.
var siteTestsDir string
// contains determines whether an array of strings contains a given string.
func contains(arr []string, elem string) bool {
for _, x := range arr {
if x == elem {
return true
}
}
return false
}
// e2eTest contains metadata for a single end-to-end tests.
// End-to-end tests are typically found as directories within
// autotest/server/site_tests
type e2eTest struct {
Name, TestClass, Purpose string
Authors, RecentEditors []string
}
// e2eTest.println displays the test struct's field names and values.
func (t *e2eTest) println() {
log.Printf("%+v\n", *t)
}
// e2eTest.dir returns the test's folder within autotest/server/site_tests/
func (t *e2eTest) dir() string {
return filepath.Join(siteTestsDir, t.Name)
}
// e2eTest.containsStr determines whether server/site_tests/$(t.Name)/*.py contains searchString.
func (t *e2eTest) containsStr(searchString string) (bool, error) {
testDir := filepath.Join(siteTestsDir, t.Name)
testDirContents, err := ioutil.ReadDir(testDir)
if err != nil {
return false, fmt.Errorf("%v when reading test directory %s", err, testDir)
}
for _, fo := range testDirContents {
testFile := filepath.Join(testDir, fo.Name())
if filepath.Ext(testFile) != ".py" {
continue
}
testFileContents, err := ioutil.ReadFile(testFile)
if err != nil {
return false, fmt.Errorf("%v when reading test file %s", err, testFile)
}
if strings.Contains(string(testFileContents), searchString) {
return true, nil
}
}
return false, nil
}
// e2eTest.setClass searches the test's .py files for any strings that indicate
// that the test is a firmware class. If any matches are found, then t.TestClass
// is set to the matching string, and the function returns true (indicating that
// the test is a firmware test).
// Otherwise, the function returns false (indicating that the test is not a
// firmware test).
func (t *e2eTest) setClass() bool {
for _, testClass := range [2]string{e2eTestClassFirmware, e2eTestClassCr50} {
hasClass, err := t.containsStr(testClass)
if err != nil {
log.Fatalf("%v for test %s", err, t.Name)
}
if hasClass {
t.TestClass = testClass
return true
}
}
return false
}
// e2e.setRecentEditors finds all authors who have committed changes to the test
// within the past $(daysSinceRecentCommit) days.
func (t *e2eTest) setRecentEditors() error {
sinceDate := time.Now().AddDate(0, 0, -1*daysSinceRecentCommit)
gitLogCmd := exec.Command(
"git",
"--no-pager",
"log",
"--since",
sinceDate.Format(time.RFC3339), ".")
gitLogCmd.Dir = t.dir()
gitLogOutput, err := gitLogCmd.Output()
if err != nil {
return fmt.Errorf("%v when running %s", err, gitLogCmd)
}
authorMatches := gitLogAuthorEmailRegexp.FindAllSubmatch(gitLogOutput, -1)
authors := []string{}
for _, authorMatch := range authorMatches {
author := strings.TrimSpace(string(authorMatch[1]))
if !contains(authors, author) {
authors = append(authors, author)
}
}
t.RecentEditors = authors
return nil
}
// newE2ETest creates a new e2eTest based on its name, and fills in its fields.
// If the test is not a firmware test, instead return a null pointer with no error.
func newE2ETest(name string) (*e2eTest, error) {
t := &e2eTest{Name: name}
if !t.setClass() {
return nil, nil
}
for _, f := range []func() error{t.setRecentEditors, t.setAuthors, t.setPurpose} {
err := f()
if err != nil {
return nil, fmt.Errorf("test %s: %v", t.Name, err)
}
}
return t, nil
}
// captureGroupFromFileContents runs a Regexp on the contents of a file,
// and returns the first capture group (or "" if no match is found).
func captureGroupFromFileContents(fp string, re *regexp.Regexp) (string, error) {
fileBytes, err := ioutil.ReadFile(fp)
if err != nil {
return "", fmt.Errorf("%v when reading file %s", err, fp)
}
match := re.FindStringSubmatch(string(fileBytes))
if match != nil {
return strings.TrimSpace(match[1]), nil
}
return "", nil
}
// e2eTest.setAuthors populates e2eTest.Authors based on the AUTHORS line of the control file
func (t *e2eTest) setAuthors() error {
controlFile := t.findOneControlFile()
authors, err := captureGroupFromFileContents(controlFile, controlFileAuthorRegexp)
if err != nil {
return fmt.Errorf("%v when setting Authors", err)
}
for _, author := range strings.Split(authors, ",") {
t.Authors = append(t.Authors, strings.TrimSpace(author))
}
return nil
}
// e2eTest.setPurpose populates e2eTest.Purpose based on the PURPOSE line of the control file.
// If there is a file named "control", then its PURPOSE is assumed to be the source-of-truth.
// Otherwise, all unique PURPOSEs from control-files are concatenated and semicolon-deliminated.
func (t *e2eTest) setPurpose() error {
allPurposes := []string{}
for _, controlFile := range t.findAllControlFiles() {
purpose, err := captureGroupFromFileContents(controlFile, controlFilePurposeRegexp)
if err != nil {
return fmt.Errorf("%v when setting Purpose", err)
}
if !contains(allPurposes, purpose) {
allPurposes = append(allPurposes, purpose)
}
if filepath.Base(controlFile) == "control" {
break
}
}
t.Purpose = strings.Join(allPurposes, "; ")
return nil
}
// e2eTest.findOneControlFile returns the path to the alphabetically first control file for t.
func (t *e2eTest) findOneControlFile() string {
controlFiles := t.findAllControlFiles()
return controlFiles[0]
}
// e2eTest.findAllControlFiles returns a slice of filepaths to the control files for t.
func (t *e2eTest) findAllControlFiles() []string {
controlFiles := []string{}
fileObjs, _ := ioutil.ReadDir(t.dir())
for _, fo := range fileObjs {
if strings.HasPrefix(fo.Name(), "control") {
controlFiles = append(controlFiles, filepath.Join(t.dir(), fo.Name()))
}
}
sort.Strings(controlFiles)
return controlFiles
}
// getDefaultAutotestDir attempts to find Autotest if the -autotest_dir flag is
// not provided.
func getDefaultAutotestDir() string {
defaultAutotestDir := "/"
thisDir, err := os.Getwd()
if err != nil {
log.Fatal(err, " when getting current working directory")
}
thisDir, err = filepath.Abs(thisDir)
if err != nil {
log.Fatal(err, " when parsing cwd as an absolute path")
}
thisDir = filepath.ToSlash(thisDir)
for _, dir := range strings.Split(thisDir, "/") {
defaultAutotestDir = filepath.Join(defaultAutotestDir, dir)
if dir == "src" {
break
}
}
defaultAutotestDir = filepath.Join(defaultAutotestDir, "third_party", "autotest", "files")
return defaultAutotestDir
}
// collectExistingE2ETests finds all e2e tests that are already written in Autotest,
// and collects information about them.
func collectExistingE2ETests() ([]*e2eTest, error) {
siteTestFileObjs, err := ioutil.ReadDir(siteTestsDir)
if err != nil {
return nil, fmt.Errorf("reading site_tests dir (%s): %v", siteTestsDir, err)
}
var e2eTests []*e2eTest
for _, fo := range siteTestFileObjs {
if !fo.IsDir() {
continue
}
if t, err := newE2ETest(fo.Name()); err != nil {
return nil, err
} else if t != nil {
e2eTests = append(e2eTests, t)
}
}
return e2eTests, nil
}
// Init parses command-line args to find autotest and site_tests.
func init() {
flag.StringVar(&autotestDir, "autotest_dir", getDefaultAutotestDir(), "The path to your autotest files")
flag.Parse()
if _, err := os.Stat(autotestDir); os.IsNotExist(err) {
log.Fatalf("Could not find autotest directory: <%s>. Try supplying -autotest_dir", autotestDir)
}
siteTestsDir = filepath.Join(autotestDir, "server", "site_tests")
if _, err := os.Stat(siteTestsDir); os.IsNotExist(err) {
log.Fatalf("Could not find site_tests directory: <%s>", siteTestsDir)
}
}
// Main collects and reports info on all firmware end-to-end tests.
func main() {
// Collect written tests and unwritten/flaky test issues
existingTests, err := collectExistingE2ETests()
if err != nil {
log.Fatalf("collecting existing tests: %v", err)
}
issues, err := monorail.IssuesList(monorail.UnwrittenFwTestReqParams)
if err != nil {
log.Fatalf("collecting unwritten tests: %v", err)
}
// Report on tests and issues
if len(existingTests) > 0 {
log.Print("### Existing end-to-end FW tests ###")
for _, t := range existingTests {
t.println()
}
log.Printf("Total: %d", len(existingTests))
} else {
log.Print("No end-to-end FW tests found.")
}
log.Print()
if len(issues) > 0 {
log.Print("### Unwritten/flaky FW tests ###")
for _, issue := range issues {
log.Print(issue)
}
log.Printf("Total: %d", len(issues))
} else {
log.Print("No unwritten FW tests found.")
}
log.Print()
log.Print("### TOTALS ###")
log.Printf("Existing e2e tests: %d", len(existingTests))
log.Printf("Unwritten/flaky test issues: %d", len(issues))
if len(issues) > 0 {
p := float64(len(existingTests)) / float64((len(issues) + len(existingTests))) * 100
log.Printf("Proportion of tests written: %.2f%%", p)
}
}