test: custom_results: graphics: Performance testing protobufs.
Protobufs to allow individual developers to record results of traces
that are not executing actual tests on non-standard builds, including
non Chrome OS environments like Crouton or from Linux Desktops.
Protobufs can be written out to bigquery for further analysis.
Messages exist to model
- Machine is gathering the result.
- Package identifies a particular version of software running.
- Result is the actual result allowing multiple metrics of interest.
- SoftwareConfig is the set of software running on a device for both
Chrome OS and Debian based systems.
- SoftwareOverridesConfig is the set of software that was overriden
from the standard system software for a particular run.
- Trace is general information about an apitrace trace.
BUG=b:152987133
TEST=bqschemauploader, generate data, post results, query via plx
Change-Id: I8101dcdb782d3d2c185a50bb78942dd132b488ac
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/infra/proto/+/2124777
Commit-Queue: David Riley <davidriley@chromium.org>
Tested-by: David Riley <davidriley@chromium.org>
Reviewed-by: C Shapiro <shapiroc@chromium.org>
diff --git a/src/test/custom_results/graphics/machine.proto b/src/test/custom_results/graphics/machine.proto
new file mode 100644
index 0000000..f7fd7d1
--- /dev/null
+++ b/src/test/custom_results/graphics/machine.proto
@@ -0,0 +1,26 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+import "google/protobuf/timestamp.proto";
+import "machine_id.proto";
+
+// Next Tag: 5
+message Machine {
+ // Global unique machine identifier.
+ MachineId name = 1;
+
+ google.protobuf.Timestamp create_time = 2;
+
+ // Identity of owner.
+ string owner = 3;
+
+ // Hardware identification.
+ string hwid = 4;
+}
diff --git a/src/test/custom_results/graphics/machine_id.proto b/src/test/custom_results/graphics/machine_id.proto
new file mode 100644
index 0000000..1959b03
--- /dev/null
+++ b/src/test/custom_results/graphics/machine_id.proto
@@ -0,0 +1,14 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+// Uniquely identifies a machine.
+message MachineId {
+ string value = 1;
+}
diff --git a/src/test/custom_results/graphics/package.proto b/src/test/custom_results/graphics/package.proto
new file mode 100644
index 0000000..13e0dc4
--- /dev/null
+++ b/src/test/custom_results/graphics/package.proto
@@ -0,0 +1,30 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+import "google/protobuf/timestamp.proto";
+
+// Next Tag: 7
+message Package {
+ string name = 1;
+ string version = 2;
+
+ // Hash of last commit in git repo.
+ string git_hash = 3;
+
+ // Git branch package was build from.
+ string branch = 4;
+
+ // Commit date of last commit in git repo.
+ google.protobuf.Timestamp commit_date = 5;
+
+ // For packages coming from git, indicate if the build was generated
+ // with a dirty git repo.
+ bool repo_dirty = 6;
+}
diff --git a/src/test/custom_results/graphics/result.proto b/src/test/custom_results/graphics/result.proto
new file mode 100644
index 0000000..37280cd
--- /dev/null
+++ b/src/test/custom_results/graphics/result.proto
@@ -0,0 +1,106 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+import "google/protobuf/timestamp.proto";
+import "machine_id.proto";
+import "result_id.proto";
+import "software_config_id.proto";
+import "software_overrides_config.proto";
+import "trace_id.proto";
+
+// Result of a single test, trace, or benchmark run.
+// Next Tag: 19
+message Result {
+ // Global unique result identifier.
+ ResultId id = 1;
+
+ // When the result started and finished being collected.
+ google.protobuf.Timestamp start_time = 2;
+ google.protobuf.Timestamp end_time = 3;
+
+ // Machine which generated the result.
+ MachineId machine = 4;
+
+ // Installed software at the time the result was collected.
+ SoftwareConfigId software_config = 5;
+
+ enum ExecutionEnvironment {
+ UNKNOWN = 0;
+ HOST = 1;
+ TERMINA = 2;
+ CROSTINI = 3;
+ STEAM = 4;
+ ARC = 5;
+ ARCVM = 6;
+ CROUTON = 7;
+ CROSVM = 8;
+ }
+ ExecutionEnvironment execution_environment = 6;
+
+ // Lab or user originating this result.
+ // Format:
+ // - lab/LABNAME
+ // - user/USERNAME
+ string invocation_source = 7;
+
+ // Details on the invocation if it was generated as part of a test.
+ // TODO(davidriley): Flesh these out better.
+ // TODO(davidriley): Record test status and error messages.
+ string test_id = 17;
+ string test_name = 8;
+ string test_job_id = 9;
+
+ // Command line used invoke the test.
+ string command_line = 10;
+
+ // Thing being tested.
+ string benchmark = 11;
+ TraceId trace = 12;
+
+ // TODO(davidriley): This unit could be made into a separate Message
+ // a) This stanza is directly part of TraceResult with TraceResult repeated.
+ // b) This stanze becomes a separate repeated Message within TraceResult.
+ // c) This stanza becomes a separate Message with TraceResultId link to them.
+ // Next Tag: 7
+ message Metric {
+ string name = 1;
+ uint64 index = 2;
+ double value = 3;
+ string units = 4;
+ bool larger_is_better = 5;
+ bool externally_gathered = 6;
+ }
+
+ // fps, frames, time
+ // usec for frame N
+ // cpu/gpu utilization/frequency
+ repeated Metric metrics = 13;
+
+ // Identify the most interesting metric to show by default for a given result.
+ string primary_metric_name = 14;
+
+ // General non-numeric details collected about the environment or system
+ // state. eg Interesting environment variables, glxinfo info.
+ // Next Tag: 4
+ message Label {
+ string name = 1;
+ string value = 2;
+ string grouping = 3;
+ }
+ repeated Label labels = 15;
+
+ // Software that was explicitly overriden from the normal system software
+ // for a given result.
+ SoftwareOverridesConfig overrides = 18;
+}
+
+message ResultList {
+ repeated Result value = 1;
+}
diff --git a/src/test/custom_results/graphics/result_id.proto b/src/test/custom_results/graphics/result_id.proto
new file mode 100644
index 0000000..5d9f779
--- /dev/null
+++ b/src/test/custom_results/graphics/result_id.proto
@@ -0,0 +1,14 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+// Uniquely identifies a test result.
+message ResultId {
+ string value = 1;
+}
diff --git a/src/test/custom_results/graphics/software_config.proto b/src/test/custom_results/graphics/software_config.proto
new file mode 100644
index 0000000..ecf3810
--- /dev/null
+++ b/src/test/custom_results/graphics/software_config.proto
@@ -0,0 +1,65 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+import "google/protobuf/timestamp.proto";
+import "package.proto";
+import "software_config_id.proto";
+
+// Software configuration of installed software on a device.
+// Next Tag: 11
+message SoftwareConfig {
+ // Global unique software configuration identifier.
+ SoftwareConfigId id = 1;
+
+ google.protobuf.Timestamp create_time = 2;
+
+ // TODO(davidriley): Decide if the hierarchy of the software config should
+ // be expressed with parent pointers or as part of the run.
+ // TODO(davidriley): How to handle same system invoked as chroot or VM?
+ // TODO(davidriley): How to handle crosvm vs qemu.
+
+ // For nested systems, the software configuration of the host system.
+ SoftwareConfigId parent = 3;
+
+ repeated Package packages = 4;
+
+ string kernel_release = 5;
+ string kernel_version = 6;
+
+ message ChromeOS {
+ string board = 1;
+ uint32 branch_number = 2;
+ string builder_path = 3;
+ uint32 build_number = 4;
+ string build_type = 5;
+ uint32 chrome_milestone = 6;
+ string description = 7;
+ string keyset = 8;
+ string name = 9;
+ string patch_number = 10;
+ string track = 11;
+ string version = 12;
+ }
+ ChromeOS chromeos = 7;
+
+ message OS {
+ string build_id = 1;
+ string codename = 2;
+ string id = 3;
+ string name = 4;
+ string pretty_name = 5;
+ string version_id = 6;
+ string version = 7;
+ }
+ OS os = 8;
+
+ string bios_version = 9;
+ string ec_version = 10;
+}
diff --git a/src/test/custom_results/graphics/software_config_id.proto b/src/test/custom_results/graphics/software_config_id.proto
new file mode 100644
index 0000000..f04e8e5
--- /dev/null
+++ b/src/test/custom_results/graphics/software_config_id.proto
@@ -0,0 +1,14 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+// Uniquely identifies a software configuration.
+message SoftwareConfigId {
+ string value = 1;
+}
diff --git a/src/test/custom_results/graphics/software_overrides_config.proto b/src/test/custom_results/graphics/software_overrides_config.proto
new file mode 100644
index 0000000..16937e3
--- /dev/null
+++ b/src/test/custom_results/graphics/software_overrides_config.proto
@@ -0,0 +1,17 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+import "package.proto";
+
+// Next Tag: 2
+message SoftwareOverridesConfig {
+ // Packages that were explicitly overriden from the normal system softare.
+ repeated Package packages = 1;
+}
diff --git a/src/test/custom_results/graphics/trace.proto b/src/test/custom_results/graphics/trace.proto
new file mode 100644
index 0000000..e586c32
--- /dev/null
+++ b/src/test/custom_results/graphics/trace.proto
@@ -0,0 +1,51 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+import "trace_id.proto";
+
+// Details about a graphics trace file.
+// Next Tag: 10
+message Trace {
+ // Global unique trace identifier.
+ TraceId id = 1;
+ string filename = 2;
+ uint64 size = 3;
+
+ // Source of the trace.
+ string source = 4;
+
+ // Source specific identifier for the program being traced. (eg Steam ID)
+ string application_id = 5;
+
+ // Number of frames in the trace.
+ uint32 frame_count = 6;
+
+ message Range {
+ uint32 start = 1;
+ uint32 end = 2;
+ }
+
+ // The range of frames after the game began to interact.
+ Range frame_range = 7;
+
+ // Frames that are worth checking for pixel correctness.
+ repeated uint32 key_frames = 8;
+
+ // Frames that are worth looping on to gauge performance.
+ repeated uint32 loop_frames = 9;
+
+ // TODO(davidriley): Add GL/Vulkan and profile.
+ // TODO(davidriley): Add extensions used.
+ // TODO(davidriley): Add compressed size?
+}
+
+message TraceList {
+ repeated Trace value = 1;
+}
diff --git a/src/test/custom_results/graphics/trace_id.proto b/src/test/custom_results/graphics/trace_id.proto
new file mode 100644
index 0000000..071e04c
--- /dev/null
+++ b/src/test/custom_results/graphics/trace_id.proto
@@ -0,0 +1,14 @@
+// Copyright 2020 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto3";
+
+package test.custom_results.graphics;
+
+option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/custom_results/graphics";
+
+// Uniquely identifies a trace used for replay tests.
+message TraceId {
+ string value = 1;
+}
