src/test/metadata/v1/metadata.proto

// Copyright 2020 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

syntax = "proto3";

// This package contains the definitions for the test metadata generated as
// build artifacts for all supported tests.
//
// This metadata is used for test execution requests, scheduling decisions and
// results analytics in various Test Lab Environments.
//
// Metadata MUST be generated for all tests in supported Remote Test Drivers and
// MUST be respected in all Test Lab Environments.
package test.metadata.v1;

option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/metadata/v1;metadata";
option java_outer_classname = "MetadataProto";

import "google/protobuf/struct.proto";
import "lab/peripherals.proto";

// From chromiumos/config repo.
import "api/hardware_topology.proto";
import "api/topology.proto";

// The test metadata specification.
//
// Test metadata MUST be generated for each Remote Test Driver as one of the
// test artifacts generated from Chrome OS build system.
message Specification {
  // A set of Remote Test Driver packages.
  //
  // In practice, a complete specification of all known Remote Test Drivers may
  // consist of multiple Specification instances. In that case the test names
  // MUST be unique across different Specification instances.
  //
  // Unlike tests, a RemoteTestDriver (identified by its name) may be specified
  // multiple times, in a single Specification or across different Specification
  // instances. In that case, all fields except the list of tests MUST be
  // identical across these RemoteTestDriver instances. The list of tests across
  // these instances will be concatenated together.
  repeated RemoteTestDriver remote_test_drivers = 1;
}

// go/cros-rtd-spec describes the Remote Test Driver bootstrapping flow inside
// the Remote Test Server. Remote Test Driver configures the bootstrapping flow
// for a test invocation.
message RemoteTestDriver {
  // Globally unique name for a Remote Test Driver package.
  //
  //
  // MUST be a valid resource per https://aip.dev/122.
  //
  // Pattern: remoteTestDrivers/{remoteTestDriver}
  //
  // The two most commonly used Remote Test Drivers are named
  // - remoteTestDrivers/tauto
  // - remoteTestDrivers/tast
  // and contain the common public tests written in the Tauto and Tast framework
  // respectively.
  string name = 1;

  // A container image containing all the required dependencies and other build
  // artifacts required for test invocations.
  //
  // Test Lab Environments SHOULD fail with an error indicating that the request
  // was invalid if the image fails to be found for a test invocation.
  BuildArtifact image = 2;

  // Command to invoke the Remote Test Driver inside `image`.
  //
  // Remote Test Servers MUST run `command` as
  //   $ ${command} -input_json ${input_json}
  // where ${command} and ${input_json} are absolute paths inside the container.
  //
  // Remote Test Servers MUST populate `input_json` with a JSON encoded
  // test.invocation.Request message.
  string command = 3;

  // Metadata for the smallest schedulable test units.
  repeated Test tests = 4;
}

// A Chrome OS build artifact required for test execution.
message BuildArtifact {
  // All build artifacts are assumed to be archived in some common directory
  // root. `relative_path` is the relative path inside the root archive
  // directory.
  //
  // Infrastructure Test Lab Environment: Chrome OS Continuous Integration
  // builders archive build artifacts to a path on Google Storage. The Test
  // Platform requests include a reference to the root archival path.
  // e.g.,
  // * The builder archives all build artifacts to
  //   gs://chromeos-image-archive/link-release/R64-0.0.0.0/
  // * Test Platform request includes this archival path.
  // * `relative_path` is set to tast/mighty.bin
  // * Then, the full path to the artifact is
  //   gs://chromeos-image-archive/link-release/R64-0.0.0.0/tast/mighty.bin
  //
  // Developer Test Lab Environment: For developer flow outside of the
  // infrastructure, it is assumed that the build artifacts are available at a
  // local path with the same directory layout as the Google Storage archival.
  // e.g.,
  // * Local build copies all artifacts to /tmp/cbuildbot/archive/
  // * Local tooling includes a reference to this path.
  // * `relative_path` is set to tauto/ssp.bin
  // * Then, the full path to the artifact is
  //   /tmp/cbuildbot/archive/tauto/ssp.bin
  string relative_path = 1;
}

// The smallest schedulable test unit.
//
// A Test is an atomic schdulable unit. In particular, it is not possible to
// modify the behaviour of a Remote Test Driver execution for a given Test by
// supplying test arguments. See the documentation of
// test.metadata.v1.Informational on how paramaterized tests may be in light of
// this restriction.
//
// A single test platform request or Remote Test Driver invocation may contain
// multiple instances of multiple Tests, but each instance of a Test MUST
// correspond to exactly one reported result.
//
// See Also:
//    Test Platform request: TODO(pprabhu)
//    Remote Test Driver invocation request: test/rtd/invocation.proto
//    Remote Test Driver progress API: test/rtd/progress.proto
message Test {
  // Globally unique name for this test.
  //
  // MUST be a valid resource per https://aip.dev/122.
  //
  // Pattern: remoteTestDrivers/{remoteTestDriver}/tests/{test}
  //   where {remoteTestDriver} is the Remote Test Driver package that contains
  //   this test.
  string name = 1;

  // Attributes are used to include tests in test plans.
  //
  // See Also:
  //   Test plans: test/plan/plan.proto
  repeated Attribute attributes = 2;

  // Required condition to be met for each Device Under Test targeted by this
  // test.
  //
  // Condition enforcement is an optional feature for test scheduling, i.e.,
  // some Test Lab Environments may ignore conditions entirely.
  //
  // May be specified multiple times for tests that target more than one Device
  // Under Test, once per required device.
  repeated DUTCondition conditions = 3;

  // Metadata about the test that doesn't affect scheduling or execution.
  Informational informational = 4;
}

// Attributes used to include tests in test plans.
message Attribute {
  // Opaque name for this attribute.
  //
  // Value MUST be valid resource names per https://aip.dev/122.
  //
  // MUST NOT be interpreted by Test Lab Environments.
  string name = 1;
}

// Condition to be met for each Device Under Test targeted by a test.
message DUTCondition {
  // A Common Expression Language (CEL) expression to specify DUT conditions.
  //
  // Test Lab Environments may optionally support targeting test requests to
  // Device Under Test based on DUTConditions. If this feature is supported, the
  // Test Lab Environment MUST interpret `expression` in the scope of the
  // protobuf message DUTCondition.Scope as defined below.
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  // This API only currently only supports a small sub-set of the CEL features,
  // as described here. Test Lab Environments SHOULD validate the expression and
  // reject use of unsupported features.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - A specific characteristic for a hardware feature, e.g.:
  //     scope.hardware_topology.screen.milliinch.value == 14000
  // - A specific topology for a hardware feature, e.g.:
  //     scope.hardware_topology.form_factor.id == "fancy_clamshell"
  // - Existence of a hardware feature, e.g.:
  //     scope.hardware_features.lte == scope.hardware_features.PRESENT
  // - Exclude certain hardware topologies, e.g.:
  //     scope.hardware_topology.stylus.id != "pencil"
  //
  // ## CEL support
  //
  // Current support for `expression` evaluation is very restricted due to
  // limitations in the scheduling platform used by Test Platform. Specifying
  // the conditions in CEL will allow gradual lifting of support restrictions.
  //
  // As this API matures, features will be added to the scheduling
  // infrastructure of Test Platform and restrictions here will be lifted based
  // on requirements collected from test authors. See milestones in
  // go/cros-f20-plan for expected feature iterations.
  //
  // ### Evaluation context
  //
  // In general, a CEL expression must be evaluated in some context that
  // provides the basic bindings for name resolution.
  //
  // In this case, `expression` MUST be evaluated in a context that contains
  //
  // - A variable 'scope' of type DUTCondition.Scope. This variable contains the
  //   information about a particular Device Under Test being tested for
  //   acceptance via `expression`.
  // - Protobuf definitions in this git project (i.e., rooted at infra/proto/).
  //
  // ### Syntax
  //
  // See full syntax definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#syntax
  //
  // CEL standard syntax allows expressions that evaluate to errors (e.g.,
  // syntax allows negation of lists, which has no semantics in CEL).
  // Thus, this spec does not attempt to restrict the syntax, but specifies what
  // operations are unsupported to aid metadata producers. Ultimately, the
  // reference metadata validator is the authority on what expressions are
  // allowed.
  //
  // Unsupported standard CEL semantics:
  //   - Binary arithmetic operations
  //     e.g.: +, *, /, % ...
  //   - Relational Operators beyond (in)equality are not supported.
  //     e.g.: (>, <, >=, <= ...)
  //   - Logical OR in expressions is not supported.
  //     e.g.: (a || b), !(a && b) ...
  //
  // ### Macros
  //
  // See full macro definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#macros
  //
  // Supported macros: has(), e.all()
  // Unsupported macros: e.exists(), e.exists_one(), e.map(), e.filter()
  //
  // ### Standard functions
  //
  // See full list of standard definitions at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions
  //
  // Most standard functions are not supported.
  //
  // - Supported operators: !_, -_, _!=_, _&&_, _=_, _[_]
  //   - All other operators are not supported.
  // - All other standard functions are not supported. In particular take note:
  //   - size() is not supported.
  //   - string functions like endsWith() and contains() are not supported.
  //   - type conversions like int() and string() are not supported.
  //   - reflection with type(), null_type() and dyn() is not supported.
  string expression = 1;

  // Protocol buffer scope for interpretation of `expression`.
  //
  // Scope includes the Device Under Test features that can be targeted for test
  // targeting.
  message Scope {
    // Peripherals information about the lab deployment of the device.
    lab.Peripherals peripherals = 1;
    // The relationship between topology and features is described at
    // https://chromium.googlesource.com/chromiumos/config/+/master/proto/api/hardware_topology.md
    chromiumos.config.api.HardwareTopology hardware_topology = 2;
    chromiumos.config.api.HardwareFeatures hardware_features = 3;
  }
}

// Contains metadata about the test that doesn't affect scheduling or execution.
message Informational {
  // Contacts for ownership / flakiness notification etc.
  repeated Contact authors = 1;

  // Machine readable test-specific information.
  //
  // Remote Test Drivers SHOULD include detailed information to aid analytics.
  // For example, test authors may minimize code duplication by writing
  // paramterized tests. Thus, multiple test metadata may refer to the
  // same test implementation with different arguments. It is useful to include
  // this information as details. An example for Tauto:
  //   {
  //      "test_project": "chromiumos/third_party/autotest",
  //      "control_file": "site_tests/dummy_Pass/control.stress",
  //      "args": {
  //        "run_count": 35
  //      }
  //   }
  //
  // This field MUST NOT be interpreted by the Test Lab Environments, but Remote
  // Test Drivers can enrich analytics by using uniform stable schema for
  // details across all their tests.
  google.protobuf.Struct details = 2;
}

// Contact information of individuals or teams.
message Contact {
  oneof type {
    // e.g.: user@google.com
    string email = 1;
    // e.g.: team-name (not mdb/team-name)
    string mdb_group = 2;
  }
}