src/test/plan/v1/plan.proto

// Copyright 2020 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

syntax = "proto3";

package test.plan.v1;

option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/plan/v1;plan";

import "test/fleet/v1/dut.proto";

// From chromiumos/config repo.
import "chromiumos/config/api/topology.proto";

// A set of known test plans.
//
// In practice, a complete specification of all known plans may consist of
// multiple Specification instances. In that case the plan names MUST be unique
// across different Specification instances.
message Specification {
  repeated Plan plans = 1;
}

// A Plan fully specifies a Test Platform end-user's coverage needs.
//
// Plans SHOULD associate platform software and Device Under Test condition
// coverage rules with tests that exercise those components. Plans MUST be used
// in the Test Platform request API. Other Test Lab Environments may use plans
// to ease interoperation with the Test Platform.
message Plan {
  // A globally unique test plan name.
  //
  // MUST be valid resource name per https://aip.dev/122.
  //
  // Pattern: plans/{plan}
  string name = 1;

  // Each test plan unit specifies a particular set of tests to be run to meet
  // specific conditions.
  repeated Unit units = 2;
}

// Specifies a particular set of tests to be run to meet specific conditions.
message Unit {
  // A globally unique test plan unit name.
  //
  // MUST be valid resource name per https://aip.dev/122.
  //
  // Pattern: plans/{plan}/units/{unit}
  //   where {plan} is the parent Plan of this Unit.
  string name = 1;

  // Selects tests to include in this test plan unit.
  TestConstraint test_constraint = 2;

  // Selects the set of Devices Under Test that satisfy the coverage
  // requirements of this test plan unit.
  DUTCoverageConstraint dut_coverage_constraint = 3;

  // Chrome OS platform software covered by this test plan.
  //
  // Test Platform requests may optionally include a reference to platform
  // software to be tested via the test plan. For such requests, Test Platform
  // MUST only execute test plan units that have non-trivial coverage of the
  // referenced platform software. Unrelated test plan units MUST be skipped.
  //
  // A typical example is presubmit testing: When testing a change to the
  // network manager, it may be desirable to only run test plan units that
  // are known to exercise the network manager.
  //
  // For a test plan unit with no `code_coverage` specified, Test Platform MUST
  // assume coverage of all platform software (i.e., this test plan unit MUST
  // never be skipped for code coverage considerations).
  CodeCoverage code_coverage = 4;
}

// Selects the test.metadata.Test to include in a test plan unit.
message TestConstraint {
  // A Common Expression Language (CEL) expression to specify set of tests that
  // are included in a test plan unit.
  //
  // Test Platform MUST effectively evaluate `expression` with the following
  // declarations in scope for every test and include the test for which
  // `expression` evaluates to true in the test plan unit.
  //
  // - Constant: `test` of type TestConstraint.Test defined below,
  //   populated with metadata for a particular test.
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - Tauto dummy tests
  //     "suite:dummy" in test.attributes
  // - Tast's crosbolt tests
  //     !("disabled" in test.attributes)
  //     && "group:crosbolt" in test.attributes
  //     && "crosbolt_perbuild" in test.attributes
  // - network Tauto tests, selected by name
  //     test.name.startsWith("network_")
  //
  // ## CEL support
  //
  // All standard CEL syntax, macros and functions MUST be supported.
  string expression = 1;

  message Test {
    // Name of the test as specified in test.metadata.Test.name
    string name = 1;

    // The test attribute name as specified in the test.metadata.Attribute.name
    //
    // attributes are populated from test.metadata.Test.attributes.
    repeated string attributes = 2;
  }
}

// Selects sets of Devices Under Test to run the tests on in a test plan unit.
message DUTCoverageConstraint {
  // A Common Expression Language (CEL) expression to specify set of DUTs that
  // provide the necessary coverage. `expression` MUST evaluate to a boolean
  // value in the evaluation context described below.
  //
  // Test Platform MUST support scheduling test requests on a set of Devices
  // Under Test that satisfy some constraints on their Chrome OS configuration.
  //
  // Test Platform MUST effectively evaluate `expression` with the following
  // declarations in scope for each possible subset of Devices Under Test and
  // ensure that the test is scheduled on a set of Devices Under Test for which
  // `expression` evaluates to true.
  //
  // - Constant: `duts` of type repeated DUTConfigConstraint.DUT defined below,
  //   each entry in `duts` set to the Chrome OS configuration payload of a
  //   particular Device Under Test.
  // - Types: Protobuf messages from `chromiumos.config.api.*`
  //   - Additionally available with the short-hand `api.*`
  // - Types: Protobuf messages from `test.fleet.v1.*`
  //   - Additionally available with the short-hand `fleet.*`
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  // This API only supports a sub-set of the CEL features as described here.
  // Test Platform MUST validate the expression and reject use of unsupported
  // features.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - Constraints: All pre-F20 plans only specify constraints on what devices
  //   may be used, e.g.:
  //    - Run each test on one DUT with model 'nautilus'.
  //        TODO(pprabhu, shapiroc): Need a way to specify model for migration.
  //        duts.all(dut, dut.id.model_id.value = 'nautilus') && size(duts) == 1
  //    - Run each test on one DUT with a stylus.
  //        (duts.all(dut, dut.hardware_features.stylus
  //                       == api.HardwareFeatures.Present.PRESENT))
  //         && size(duts) == 1)
  //  - Fanout (not yet supported): In addition to constraints, the
  //    plan may want to ensure coverage across some device features.
  //    TODO(pprabhu) This is an instructive example. We can't say what arch
  //    yet.
  //    - Run each test on one x86 and ARM DUT with a specific camera.
  //        (duts.all(dut, dut.hardware_toplogy.camera.id != "gomoe")
  //         && duts.exist(dut,
  //                       dut.cpu = api.Component.Soc.Architecture.X86)
  //         && duts.exist(dut,
  //                       dut.cpu = api.Component.Soc.Architecture.ARM))
  //      Note that this expression does not have a clause for size(duts). Thus
  //      Test Platform may satisfy this expression by running the test on more
  //      than two DUTs. This behaviour can be unexpected, especially if the
  //      plan fails due to an error on test execution on one of the selected
  //      devices, even though it passed on other devices in the set such that
  //      the expression was satsifed by the passing subset. Selection of a
  //      minimal set of devices to run the plan is best effort.
  //
  // Currently, only single-DUT constraints are supported by the Test Platform.
  // i.e., `expression` MUST be of the form:
  //   duts.all(DUT_SELECTOR) && size(duts) == 1
  // where DUT_SELECTOR is an expression that does not refer to `duts`.
  // This restriction will be lifted as this API matures.
  //
  // ## CEL support
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // Current support for `expression` evaluation is very restricted due to
  // limitations in the scheduling infrastructure used by Test Platform.
  //
  // As this API matures, features will be added to the scheduling
  // infrastructure of Test Platform and restrictions here will be lifted based
  // on requirements collected from test authors. See milestones in
  // go/cros-f20-plan for expected feature iterations. Test Lab Environments
  // SHOULD validate the expression and reject use of unsupported features.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ### Syntax
  //
  // See full syntax definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#syntax
  //
  // CEL standard syntax allows expressions that evaluate to errors (e.g.,
  // syntax allows negation of lists, which has no semantics in CEL).
  // Thus, this spec does not attempt to restrict the syntax, but specifies what
  // operations are unsupported to aid metadata producers. Ultimately, the
  // reference metadata validator is the authority on what expressions are
  // allowed.
  //
  // Unsupported standard CEL semantics:
  //   - Binary arithmetic operations
  //     e.g.: +, *, /, % ...
  //   - Relational Operators beyond (in)equality are not supported.
  //     e.g.: (>, <, >=, <= ...)
  //   - Logical OR in expressions is not supported.
  //     e.g.: (a || b), !(a && b) ...
  //
  // ### Macros
  //
  // See full macro definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#macros
  //
  // Supported macros: has(), e.all()
  // Unsupported macros: e.exists(), e.exists_one(), e.map(), e.filter()
  //
  // ### Standard functions
  //
  // See full list of standard definitions at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions
  //
  // Most standard functions are not supported.
  //
  // - Supported operators: !_, -_, _!=_, _&&_, _=_, _[_]
  //   - All other operators are not supported.
  // - All other standard functions are not supported. In particular:
  //   - size() is not supported.
  //   - string functions like endsWith() and contains() are not supported.
  //   - type conversions like int() and string() are not supported.
  //   - reflection with type(), null_type() and dyn() is not supported.
  string expression = 1;

  // The evaluation context for `expression` MUST include the Chrome OS
  // configuration payload for a set of Devices Under Test as a typed
  // constant of the following type.
  //
  //   repeated DUT duts = 1;
  //
  message DUT {
    // Configuration information about the lab deployment of the device.
    test.fleet.v1.DeviceUnderTest fleet_dut = 1;
    chromiumos.config.api.HardwareFeatures hardware_features = 2;
  }
}

// Specifies platforms software covered by a test plan unit.
message CodeCoverage {
  // A Common Expression Language (CEL) expression to specify Chrome OS platform
  // sources covered by a test plan unit.
  //
  // Test Platform may optionally support skipping test plan units that are not
  // applicable for incoming changes to Chrome OS. If this feature is supported,
  // and if a non-empty blamelist is available, Test Platform MUST effectively
  // evaluate `expression` with the following declarations in scope for the
  // blamelist only include the test plan unit if the `expression` evaluates to
  // true.
  //
  // - Constant: `blamelist` of type CodeCoverage.Blamelist defined below,
  //   populated with blamelist attached to the request, if blamelist is
  //   non-empty.
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - Only include test plan unit if there are changes to chromite
  //     blamelist.repo_paths.exists(p, p.startsWith("chromite"))
  // - Only include test plan unit if there are changes to chromite or autotest
  //     blamelist.repo_paths.exists(p, [
  //         "src/third_party/autotest/files",
  //         "chromite"
  //       ].exists(s, p.startsWith(s)))
  // - Do not include test plan unit if there are changes to private overlays
  //     blamelist.repo_paths.all(p, !p.contains("private-overlays"))
  //
  // ## CEL support
  //
  // All standard CEL syntax, macros and functions MUST be supported.
  string expression = 1;

  message Blamelist {
    // Changed paths (usually directories) in a typical checkout of the Chrome
    // OS source tree.
    //
    // Paths are relative to `repo` root. e.g., `src/platform2/shill`.
    repeated string repo_paths = 1;
  }
}