TF result given test cases cros-test reponse from XML
Adding given test cases to cros test response from XML parsed data. This is not replacing existing test case result slice.
BUG=None
TEST=go test
TEST=LED http://shortn/_LgQHR5kInT
Change-Id: I8afc664b261696bc620d2d04856609d62d916bae
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/5912801
Auto-Submit: Varun Srivastav <varunsrivastav@google.com>
Commit-Queue: Varun Srivastav <varunsrivastav@google.com>
Commit-Queue: Alex Bergman <abergman@google.com>
Reviewed-by: Alex Bergman <abergman@google.com>
Tested-by: Varun Srivastav <varunsrivastav@google.com>
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_driver.go b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_driver.go
index a3b0eeb..007b610 100644
--- a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_driver.go
+++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_driver.go
@@ -160,9 +160,10 @@
// One difference between Tast/Autotest and this is that the test case given will likely result in many
// testcases; where T/AT were normally 1:1
- results, artifacts := buildTradefedResult(td.logger, testType)
+ results, artifacts := buildTradefedResult(td.logger, testType, req)
if results.GetTestCaseResults() != nil {
allRspn.TestCaseResults = append(allRspn.TestCaseResults, results.TestCaseResults...)
+ allRspn.GivenTestResults = append(allRspn.GivenTestResults, results.GivenTestResults...)
} else {
td.logger.Println("No results to report: ", results)
}
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results.go b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results.go
index 88226b8..241a2a9 100644
--- a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results.go
+++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results.go
@@ -244,30 +244,77 @@
status = testInfo.Status
}
// TODO; if no testInfo matched, calculate status.
- errorText := ""
- switch status {
- case "ASSUMPTION_FAILURE":
- errorText = test.AsssumptionFailure
- case "FAILED":
- errorText = test.Failure
- case "INCOMPLETE":
- errorText = test.Incomplete
- case "SKIPPED":
- errorText = test.Skipped
- case "IGNORED":
- errorText = test.Ignored
- default:
- errorText = "Unknown test status"
- }
-
+ errorText := getErrorText(status, test)
allTestCases = append(allTestCases,
buildTcResult(fmt.Sprintf("tradefed.%s.%s#%s", testType, testInfo.ModuleName, fullTestName),
status, startTime, test.Time/1000, errorText))
}
+
return allTestCases
}
-func parseCompatibilityXmlResults(logger *log.Logger, testType string) ([]*api.TestCaseResult, []string, error) {
+func getErrorText(status string, test TestCase) string {
+ errorText := ""
+ switch status {
+ case "ASSUMPTION_FAILURE":
+ errorText = test.AsssumptionFailure
+ case "FAILED":
+ errorText = test.Failure
+ case "INCOMPLETE":
+ errorText = test.Incomplete
+ case "SKIPPED":
+ errorText = test.Skipped
+ case "IGNORED":
+ errorText = test.Ignored
+ default:
+ errorText = "Unknown test status"
+ }
+ return errorText
+}
+
+func processGivenTestResults(logger *log.Logger, TS TestSuite, testInfoMap map[string]EventLogTestInfo) []*api.CrosTestResponse_GivenTestResult {
+ givenTestCases := []*api.CrosTestResponse_GivenTestResult{}
+ // map format for holding parent child like relationship. This will be used later to populate givenTestCases
+ testMap := make(map[string][]*api.TestCaseResult)
+
+ startTime, err := time.Parse(time.RFC3339, normalizeTime(TS.Timestamp))
+ if err != nil {
+ logger.Println("Error parsing Tradefed test start time: ", TS.Timestamp)
+ startTime = time.Now()
+ }
+
+ logger.Println("Creating test map with parent/module as key and test case result as child list")
+ // for all test cases parsed from metrics-*.xml
+ for _, test := range TS.TestCases {
+ fullTestName := fmt.Sprintf("%s#%s", test.ClassName, test.Name)
+ testInfo, ok := testInfoMap[fullTestName]
+ // check if exists in testInfoMap parsed from aggregated-events*.txt, testInfoMap has module name, status info as value for each test in map
+ if ok {
+ // form child test case result
+ testCaseResult := buildTcResult(fullTestName, testInfo.Status, startTime, test.Time/1000, getErrorText(testInfo.Status, test))
+ // add to child case list by module
+ if val, ok := testMap[testInfo.ModuleName]; ok {
+ testMap[testInfo.ModuleName] = append(val, testCaseResult)
+ } else {
+ testMap[testInfo.ModuleName] = []*api.TestCaseResult{testCaseResult}
+ }
+ }
+ }
+
+ // create given test cases object from the testMap
+ logger.Println("Creating given test case results object from test map")
+ for parent, childCaseList := range testMap {
+ givenTestCases = append(givenTestCases, &api.CrosTestResponse_GivenTestResult{
+ ParentTest: parent,
+ ChildTestCaseResults: childCaseList,
+ })
+ logger.Printf("Given test case with parent: %s added with %d child test cases\n", parent, len(childCaseList))
+ }
+
+ return givenTestCases
+}
+
+func parseCompatibilityXmlResults(logger *log.Logger, testType string, req *api.CrosTestRequest) ([]*api.TestCaseResult, []*api.CrosTestResponse_GivenTestResult, []string, error) {
var logsDir []string
allTestCases := []*api.TestCaseResult{}
@@ -276,7 +323,7 @@
// Fallback to stub location for XML result file.
xmlResultFile, err = selectFileByPattern(filepath.Join(tfStageLogsPath, tfStageResultPattern))
if err != nil {
- return allTestCases, logsDir, fmt.Errorf("Failed to locate result file: %w", err)
+ return allTestCases, nil, logsDir, fmt.Errorf("Failed to locate result file: %w", err)
}
}
@@ -285,18 +332,49 @@
logger.Println("Parsing results from: ", xmlResultFile)
R := Result{}
if err = parseResultFile(logger, xmlResultFile, &R); err != nil {
- return allTestCases, logsDir, err
+ return allTestCases, nil, logsDir, err
}
+ allTestCases = generateTestCaseResult(logger, testType, R, req)
+ givenTestCases := generateGivenTestCases(logger, testType, R, req)
+ return allTestCases, givenTestCases, logsDir, nil
+}
+
+func testResponseNoXMLFound(req *api.CrosTestRequest) *api.CrosTestResponse {
+ r := &api.CrosTestResponse{}
+ if req == nil || req.GetTestSuites() == nil {
+ return r
+ }
+ for _, suites := range req.GetTestSuites() {
+ for _, testCaseIds := range suites.GetTestCaseIds().GetTestCaseIds() {
+ testCaseResult := buildTcResult(testCaseIds.GetValue(), "INCOMPLETE", time.Time{}, 1, "")
+ r.TestCaseResults = append(r.TestCaseResults, testCaseResult)
+ r.GivenTestResults = append(r.GivenTestResults, &api.CrosTestResponse_GivenTestResult{
+ ParentTest: testCaseIds.GetValue(),
+ ChildTestCaseResults: []*api.TestCaseResult{testCaseResult},
+ })
+ }
+ }
+ return r
+}
+
+func generateTestCaseResult(logger *log.Logger, testType string, R Result, req *api.CrosTestRequest) []*api.TestCaseResult {
+ allTestCases := []*api.TestCaseResult{}
startTimeMs, _ := strconv.ParseInt(R.Start, 10, 64)
startTime := time.UnixMilli(startTimeMs)
logger.Println("Found n module results:", len(R.Modules))
+ modulesFromReq := getAllModulesFromReq(req)
for _, module := range R.Modules {
+ delete(modulesFromReq, fmt.Sprintf("tradefed.%s.%s", testType, module.Name))
nTests, _ := strconv.Atoi(module.Total_Tests)
runTime, _ := strconv.Atoi(module.Runtime)
duration := runTime
if nTests != 0 {
duration = runTime / nTests
+ } else {
+ // mark module as Crash if no tests were executed for the module
+ allTestCases = append(allTestCases,
+ buildTcResult(fmt.Sprintf("tradefed.%s.%s", testType, module.Name), "INCOMPLETE", time.Time{}, 1, ""))
}
logger.Println("Parsing Module: ", module.Name)
@@ -314,11 +392,76 @@
}
}
}
-
- return allTestCases, logsDir, nil
+ for moduleName := range modulesFromReq {
+ allTestCases = append(allTestCases,
+ buildTcResult(moduleName, "INCOMPLETE", time.Time{}, 1, ""))
+ }
+ return allTestCases
}
-func buildTradefedResult(logger *log.Logger, testType string) (*api.CrosTestResponse, []string) {
+func getAllModulesFromReq(req *api.CrosTestRequest) map[string]struct{} {
+ modules := make(map[string]struct{})
+ if req == nil || req.GetTestSuites() == nil {
+ return modules
+ }
+ for _, suites := range req.GetTestSuites() {
+ for _, testCaseIds := range suites.GetTestCaseIds().GetTestCaseIds() {
+ modules[testCaseIds.GetValue()] = struct{}{}
+ }
+ }
+ return modules
+}
+
+func generateGivenTestCases(logger *log.Logger, testType string, R Result, req *api.CrosTestRequest) []*api.CrosTestResponse_GivenTestResult {
+ givenTestCases := []*api.CrosTestResponse_GivenTestResult{}
+ startTimeMs, _ := strconv.ParseInt(R.Start, 10, 64)
+ startTime := time.UnixMilli(startTimeMs)
+ logger.Println("Found n module results:", len(R.Modules))
+ modulesFromReq := getAllModulesFromReq(req)
+ for _, module := range R.Modules {
+ delete(modulesFromReq, fmt.Sprintf("tradefed.%s.%s", testType, module.Name))
+ nTests, _ := strconv.Atoi(module.Total_Tests)
+ runTime, _ := strconv.Atoi(module.Runtime)
+ duration := runTime
+ if nTests != 0 {
+ duration = runTime / nTests
+ } else {
+ // mark module as Crash if no tests were executed for the module
+ givenTestCases = append(givenTestCases, &api.CrosTestResponse_GivenTestResult{
+ ParentTest: fmt.Sprintf("tradefed.%s.%s", testType, module.Name),
+ ChildTestCaseResults: []*api.TestCaseResult{buildTcResult(fmt.Sprintf("tradefed.%s.%s", testType, module.Name), "INCOMPLETE", time.Time{}, 1, "")},
+ })
+
+ }
+
+ logger.Println("Parsing Module: ", module.Name)
+ // Make a duration of atleast 1 second so the proto isnt empty.
+ testDur := int64(duration / 1000)
+ if testDur == 0 {
+ testDur = 1
+ }
+ givenTestCase := &api.CrosTestResponse_GivenTestResult{}
+ for _, testcase := range module.TestCases {
+
+ childTestCases := []*api.TestCaseResult{}
+ for _, test := range testcase.Tests {
+ testName := fmt.Sprintf("%s#%s", testcase.Name, test.Name)
+ childTestCases = append(childTestCases,
+ buildTcResult(testName, test.Result, startTime, testDur, test.Failure.Message))
+ }
+ givenTestCase.ParentTest = module.Name
+ givenTestCase.ChildTestCaseResults = childTestCases
+ }
+ givenTestCases = append(givenTestCases, givenTestCase)
+ }
+ for moduleName := range modulesFromReq {
+ givenTestCases = append(givenTestCases,
+ &api.CrosTestResponse_GivenTestResult{ParentTest: moduleName, ChildTestCaseResults: []*api.TestCaseResult{buildTcResult(moduleName, "INCOMPLETE", time.Time{}, 1, "")}})
+ }
+ return givenTestCases
+}
+
+func buildTradefedResult(logger *log.Logger, testType string, req *api.CrosTestRequest) (*api.CrosTestResponse, []string) {
// List of glob patterns of artifacts that should be saved in test results.
var logsToSave = []string{}
@@ -329,9 +472,10 @@
if err != nil {
logger.Println("Error locating Tradefed metric result file: ", err)
logger.Println("Trying to locate and parse compatibility result file...")
- f.TestCaseResults, logsToSave, err = parseCompatibilityXmlResults(logger, testType)
+ f.TestCaseResults, f.GivenTestResults, logsToSave, err = parseCompatibilityXmlResults(logger, testType, req)
if err != nil {
logger.Println("Error locating or parsing Tradefed compatibility result file: ", err)
+ return testResponseNoXMLFound(req), logsToSave
}
return f, logsToSave
} else {
@@ -339,7 +483,7 @@
TS := TestSuite{}
if err = parseResultFile(logger, resultPath, &TS); err != nil {
- return f, logsToSave
+ return testResponseNoXMLFound(req), logsToSave
}
testInfoMap, eventLogs := loadEventLogReport(logger)
@@ -351,12 +495,42 @@
TS.Name, TS.Tests, TS.Failures)
f.TestCaseResults = processResults(logger, TS, testInfoMap, testType)
+ f.GivenTestResults = processGivenTestResults(logger, TS, testInfoMap)
+
+ // report modules as Crash for which no tests were reported.
+ modulesWithNoResult := noResultModules(logger, TS, testInfoMap, req, testType)
+ for module := range modulesWithNoResult {
+ crashStatusTestResult := buildTcResult(fmt.Sprintf("tradefed.%s.%s", testType, module),
+ "INCOMPLETE", time.Time{}, 1, "")
+ f.TestCaseResults = append(f.TestCaseResults, crashStatusTestResult)
+ f.GivenTestResults = append(f.GivenTestResults, &api.CrosTestResponse_GivenTestResult{
+ ParentTest: fmt.Sprintf("tradefed.%s.%s", testType, module),
+ ChildTestCaseResults: []*api.TestCaseResult{crashStatusTestResult},
+ })
+ }
}
logger.Println("Found n case results:", len(f.TestCaseResults))
return f, logsToSave
}
+func noResultModules(logger *log.Logger, TS TestSuite, testInfoMap map[string]EventLogTestInfo, req *api.CrosTestRequest, testType string) map[string]struct{} {
+ modulesFromRequest := getAllModulesFromReq(req)
+ for _, test := range TS.TestCases {
+ fullTestName := fmt.Sprintf("%s#%s", test.ClassName, test.Name)
+ testInfo, ok := testInfoMap[fullTestName]
+ if ok {
+ // remove module if a test has been reported for it.
+ delete(modulesFromRequest, fmt.Sprintf("tradefed.%s.%s", testType, testInfo.ModuleName))
+ }
+ }
+ // logging modules which were not found in results from xml
+ for module := range modulesFromRequest {
+ logger.Printf(module)
+ }
+ return modulesFromRequest
+}
+
func buildTcResult(testName string, testStatus string, startTime time.Time, duration int64,
errorMessage string) *api.TestCaseResult {
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results_test.go b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results_test.go
index 3961f25..3d82f44 100644
--- a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results_test.go
+++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_results_test.go
@@ -19,8 +19,10 @@
"google.golang.org/protobuf/types/known/timestamppb"
)
-const TradefedTestResultFile = "tradefed_test_data/test_result.xml"
-
+const (
+ TradefedTestResultFile = "tradefed_test_data/test_result.xml"
+ TradefedTestResultFileModulesNoTest = "tradefed_test_data/test_result_modules_no_test.xml"
+)
const (
FileSelectFileByPattern1 = "/tmp/test-SelectFileByPattern-file1.txt"
FileSelectFileByPattern2 = "/tmp/test-SelectFileByPattern-file2.txt"
@@ -282,6 +284,34 @@
// })
// }
+func Test_BuildTFResult(t *testing.T) {
+ R := Result{}
+ testType := "CTS"
+ moduleName := "tradefed" + "." + testType + "." + "CtsJvmtiRunTest1976HostTestCases"
+ logger := log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile)
+ err := parseResultFile(logger, TradefedTestResultFileModulesNoTest, &R)
+ if err != nil {
+ t.Logf("No error expected")
+ }
+ allTestCases := generateTestCaseResult(logger, testType, R, nil)
+ givenTestCases := generateGivenTestCases(logger, testType, R, nil)
+ if allTestCases[0].TestCaseId.Value != moduleName {
+ t.Errorf("Expected %s, found %s", moduleName, allTestCases[0].TestCaseId.Value)
+ }
+ if givenTestCases[0].GetParentTest() != moduleName {
+ t.Errorf("Expected %s, found %s", moduleName, givenTestCases[0].GetParentTest())
+ }
+ if _, ok := allTestCases[0].Verdict.(*api.TestCaseResult_Crash_); !ok {
+ t.Errorf("Expected verdict CRASH")
+ }
+ if givenTestCases[0].GetParentTest() != moduleName {
+ t.Errorf("Expected %s, found %s", moduleName, givenTestCases[0].GetParentTest())
+ }
+ if _, ok := givenTestCases[0].ChildTestCaseResults[0].Verdict.(*api.TestCaseResult_Crash_); !ok {
+ t.Errorf("Expected verdict CRASH")
+ }
+
+}
func TestBuildTcResult(t *testing.T) {
t.Parallel()
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_test_data/test_result_modules_no_test.xml b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_test_data/test_result_modules_no_test.xml
new file mode 100644
index 0000000..2ccac82
--- /dev/null
+++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/internal/driver/tradefed_test_data/test_result_modules_no_test.xml
@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8' standalone='no' ?><?xml-stylesheet type="text/xsl" href="compatibility_result.xsl"?>
+<Result start="1724450705582" end="1724450783748" start_display="Fri Aug 23 22:05:05 UTC 2024" end_display="Fri Aug 23 22:06:23 UTC 2024" command_line_args="cts --include-filter CtsJvmtiRunTest1976HostTestCases --include-filter CtsJvmtiRunTest1977HostTestCases --include-filter CtsJvmtiRunTest1978HostTestCases --include-filter CtsJvmtiRunTest1979HostTestCases --include-filter CtsJvmtiRunTest1981HostTestCases --logcat-on-failure -s satlab-0wgatfqi22088039-host3:5555" suite_name="CTS" suite_variant="CTS" suite_version="14_r4" suite_plan="cts" suite_build_number="11801623" report_version="5.0" devices="satlab-0wgatfqi22088039-host3:5555" host_name="0ed35a7e333f" os_name="Linux" os_version="5.15.103-17409-g07029265d738" os_arch="amd64" java_vendor="N/A" java_version="17.0.4.1">
+ <Build invocation-id="1" command_line_args="cts --include-filter CtsJvmtiRunTest1976HostTestCases --include-filter CtsJdwpTestCases.jar" />
+ <Summary pass="3" failed="2" modules_done="1" modules_total="1" />
+ <Module name="CtsJvmtiRunTest1976HostTestCases" abi="x86_64" runtime="6254" done="true" pass="3" total_tests="0">
+ </Module>
+</Result>
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/testexecserver.go b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/testexecserver.go
index 70376da..643a4e2 100644
--- a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/testexecserver.go
+++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/testexecserver.go
@@ -107,6 +107,7 @@
return nil, err
}
allRspn.TestCaseResults = append(allRspn.TestCaseResults, rspn.TestCaseResults...)
+ allRspn.GivenTestResults = append(allRspn.GivenTestResults, rspn.GivenTestResults...)
}
return &allRspn, nil
}
