src/test/plan/v1/plan.proto

// Copyright 2020 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

syntax = "proto3";

package test.plan.v1;

option go_package = "go.chromium.org/chromiumos/infra/proto/go/test/plan/v1;plan";

import "test/fleet/v1/dut.proto";

// From chromiumos/config repo.
import "chromiumos/config/api/topology.proto";

// A set of known test plans.
//
// In practice, a complete specification of all known plans may consist of
// multiple Specification instances. In that case the plan names MUST be unique
// across different Specification instances.
message Specification {
  repeated Plan plans = 1;
}

// A Plan fully specifies a Test Platform end-user's coverage needs.
//
// Plans SHOULD associate platform software and Device Under Test condition
// coverage rules with tests that exercise those components. Plans MUST be used
// in the Test Platform request API. Other Test Lab Environments may use plans
// to ease interoperation with the Test Platform.
message Plan {
  // A globally unique test plan name.
  //
  // MUST be valid resource name per https://aip.dev/122.
  //
  // Pattern: plans/{plan}
  string name = 1;

  // Each test plan unit specifies a particular set of tests to be run to meet
  // specific conditions.
  repeated Unit units = 2;
}

// Specifies a particular set of tests to be run to meet specific conditions.
message Unit {
  // A globally unique test plan unit name.
  //
  // MUST be valid resource name per https://aip.dev/122.
  //
  // Pattern: plans/{plan}/units/{unit}
  //   where {plan} is the parent Plan of this Unit.
  string name = 1;

  // Selects tests to include in this test plan unit.
  //
  // See also: exclusions.
  TestConstraint test_constraint = 2;

  // Selects the set of Devices Under Test that satisfy the coverage
  // requirements of this test plan unit.
  //
  // See also: exclusions.
  DUTCoverageConstraint dut_coverage_constraint = 3;

  // Chrome OS platform software covered by this test plan.
  //
  // Test Platform requests may optionally include a reference to platform
  // software to be tested via the test plan. For such requests, Test Platform
  // MUST only execute test plan units that have non-trivial coverage of the
  // referenced platform software. Unrelated test plan units MUST be skipped.
  //
  // A typical example is presubmit testing: When testing a change to the
  // network manager, it may be desirable to only run test plan units that
  // are known to exercise the network manager.
  //
  // For a test plan unit with no `code_coverage` specified, Test Platform MUST
  // assume coverage of all platform software (i.e., this test plan unit MUST
  // never be skipped for code coverage considerations).
  CodeCoverage code_coverage = 4;

  // A list of criteria for (test, Device Under Test) pairs that MUST be
  // excluded from this test plan.
  //
  // Exclusions are used to record exceptions to the test plan specification for
  // devices that are known to cause test failures for some temporary or
  // permanent reasons. The test_constraint and dut_coverage_constraint fields
  // together specify the _intent_ of the test plan unit. The exclusions further
  // restrict what Devices Under Test can be used to satisfy the test plan unit
  // due to practical considerations.
  //
  // Each exclusion SHOULD correspond to a different business reason.
  // Conceptually, fixing a known issue should result in an exclusion being
  // removed.
  repeated Exclusion exclusions = 5;
}

// Selects the test.metadata.Test to include in a test plan unit.
message TestConstraint {
  // A Common Expression Language (CEL) expression to specify set of tests that
  // are included in a test plan unit.
  //
  // Test Platform MUST effectively evaluate `expression` with the following
  // declarations in scope for every test and include the test for which
  // `expression` evaluates to true in the test plan unit.
  //
  // - Constant: `test` of type TestConstraint.Test defined below,
  //   populated with metadata for a particular test.
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - Tauto dummy tests
  //     "suite:dummy" in test.attributes
  // - Tast's crosbolt tests
  //     !("disabled" in test.attributes)
  //     && "group:crosbolt" in test.attributes
  //     && "crosbolt_perbuild" in test.attributes
  // - network Tauto tests, selected by name
  //     test.name.startsWith("network_")
  //
  // ## CEL support
  //
  // All standard CEL syntax, macros and functions MUST be supported.
  string expression = 1;

  message Test {
    // Name of the test as specified in test.metadata.Test.name
    string name = 1;

    // The test attribute name as specified in the test.metadata.Attribute.name
    //
    // attributes are populated from test.metadata.Test.attributes.
    repeated string attributes = 2;
  }
}

// Selects sets of Devices Under Test to run the tests on in a test plan unit.
message DUTCoverageConstraint {
  // A Common Expression Language (CEL) expression to specify set of DUTs that
  // provide the necessary coverage. `expression` MUST evaluate to a boolean
  // value in the evaluation context described below.
  //
  // Test Platform MUST support scheduling test requests on a set of Devices
  // Under Test that satisfy some constraints on their Chrome OS configuration.
  //
  // Test Platform MUST effectively evaluate `expression` with the following
  // declarations in scope for each possible subset of Devices Under Test and
  // ensure that the test is scheduled on a set of Devices Under Test for which
  // `expression` evaluates to true.
  //
  // - Constant: `duts` of type repeated DUTConfigConstraint.DUT defined below,
  //   each entry in `duts` set to the Chrome OS configuration payload of a
  //   particular Device Under Test.
  // - Types: Protobuf messages from `chromiumos.config.api.*`
  //   - Additionally available with the short-hand `api.*`
  // - Types: Protobuf messages from `test.fleet.v1.*`
  //   - Additionally available with the short-hand `fleet.*`
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  // This API only supports a sub-set of the CEL features as described here.
  // Test Platform MUST validate the expression and reject use of unsupported
  // features.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - Constraints: All pre-F20 plans only specify constraints on what devices
  //   may be used, e.g.:
  //    - Run each test on one DUT with model 'nautilus'.
  //        TODO(pprabhu, shapiroc): Need a way to specify model for migration.
  //        duts.all(dut, dut.id.model_id.value = 'nautilus') && size(duts) == 1
  //    - Run each test on one DUT with a stylus.
  //        (duts.all(dut, dut.hardware_features.stylus
  //                       == api.HardwareFeatures.Present.PRESENT))
  //         && size(duts) == 1)
  //  - Fanout (not yet supported): In addition to constraints, the
  //    plan may want to ensure coverage across some device features.
  //    TODO(pprabhu) This is an instructive example. We can't say what arch
  //    yet.
  //    - Run each test on one x86 and ARM DUT with a specific camera.
  //        (duts.all(dut, dut.hardware_toplogy.camera.id != "gomoe")
  //         && duts.exist(dut,
  //                       dut.cpu = api.Component.Soc.Architecture.X86)
  //         && duts.exist(dut,
  //                       dut.cpu = api.Component.Soc.Architecture.ARM))
  //      Note that this expression does not have a clause for size(duts). Thus
  //      Test Platform may satisfy this expression by running the test on more
  //      than two DUTs. This behaviour can be unexpected, especially if the
  //      plan fails due to an error on test execution on one of the selected
  //      devices, even though it passed on other devices in the set such that
  //      the expression was satsifed by the passing subset. Selection of a
  //      minimal set of devices to run the plan is best effort.
  //
  // Currently, only single-DUT constraints are supported by the Test Platform.
  // i.e., `expression` MUST be of the form:
  //   duts.all(DUT_SELECTOR) && size(duts) == 1
  // where DUT_SELECTOR is an expression that does not refer to `duts`.
  // This restriction will be lifted as this API matures.
  //
  // ## CEL support
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // Current support for `expression` evaluation is very restricted due to
  // limitations in the scheduling infrastructure used by Test Platform.
  //
  // As this API matures, features will be added to the scheduling
  // infrastructure of Test Platform and restrictions here will be lifted based
  // on requirements collected from test authors. See milestones in
  // go/cros-f20-plan for expected feature iterations. Test Lab Environments
  // SHOULD validate the expression and reject use of unsupported features.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ### Syntax
  //
  // See full syntax definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#syntax
  //
  // CEL standard syntax allows expressions that evaluate to errors (e.g.,
  // syntax allows negation of lists, which has no semantics in CEL).
  // Thus, this spec does not attempt to restrict the syntax, but specifies what
  // operations are unsupported to aid metadata producers. Ultimately, the
  // reference metadata validator is the authority on what expressions are
  // allowed.
  //
  // Unsupported standard CEL semantics:
  //   - Binary arithmetic operations
  //     e.g.: +, *, /, % ...
  //   - Relational Operators beyond (in)equality are not supported.
  //     e.g.: (>, <, >=, <= ...)
  //   - Logical OR in expressions is not supported.
  //     e.g.: (a || b), !(a && b) ...
  //
  // ### Macros
  //
  // See full macro definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#macros
  //
  // Supported macros: has(), e.all()
  // Unsupported macros: e.exists(), e.exists_one(), e.map(), e.filter()
  //
  // ### Standard functions
  //
  // See full list of standard definitions at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions
  //
  // Most standard functions are not supported.
  //
  // - Supported operators: !_, -_, _!=_, _&&_, _=_, _[_]
  //   - All other operators are not supported.
  // - All other standard functions are not supported. In particular:
  //   - size() is not supported.
  //   - string functions like endsWith() and contains() are not supported.
  //   - type conversions like int() and string() are not supported.
  //   - reflection with type(), null_type() and dyn() is not supported.
  string expression = 1;

  // The evaluation context for `expression` MUST include the Chrome OS
  // configuration payload for a set of Devices Under Test as a typed
  // constant of the following type.
  //
  //   repeated DUT duts = 1;
  //
  message DUT {
    // Configuration information about the lab deployment of the device.
    test.fleet.v1.DeviceUnderTest fleet_dut = 1;
    chromiumos.config.api.HardwareFeatures hardware_features = 2;
  }
}

// Exlclusion is used to record exceptions to the test plan
// specification for devices that are known to cause test failures for some
// temporary or permanent reasons.
message Exclusion {
  enum Type {
    // Do not use.
    TYPE_UNSPECIFIED = 0;
    // There are no active plans to remove these exclusions becuase it is
    // prohibitive to fix the issue or business needs do not justify the effort
    // to fix the issue.
    //
    // Each PERMANENT exclusion MUST include references that point to a business
    // justification for its addition.
    PERMANENT = 1;

    // Use for excluding new tests from running on devices where the test has
    // not yet been stabilized. The intention is to support incremental rollout
    // of new tests.
    //
    // These exclsusion are temporary. These exclusions SHOULD be routinely
    // audited and resolved or promoted to PERMANENT exclusions.
    TEMPORARY_NEW_TEST = 2;

    // Use for excluding broken / flakey tests while a fix is being worked on.
    //
    // These exclsusion are temporary. These exclusions SHOULD be routinely
    // audited and resolved or promoted to PERMANENT exclusions.
    TEMPORARY_PENDING_FIX = 3;
  }
  // Required.
  Type type = 1;

  enum Action {
    // Specify no action.
    //
    // The Test Lab Environment may choose any of the available actions based
    // on internal heuristics.
    //
    // The default Action is a good default if none of the considersations for
    // the specific actions below apply.
    ACTION_UNSPECIFIED = 0;

    // Do not schedule the selected Tests on specified Devices Under Test.
    //
    // It is especially useful to set this Action for permanent exceptions as it
    // is definitely not useful to run the Tests at all in those cases.
    DO_NOT_SCHEDULE = 1;

    // Schedule the Test on Devices Under Test as required by the test plan
    // unit, but mark the results for selected tests on the specified Devices
    // Under Test as non-critical. Results marked non-critical are intended to
    // be ignored by result consumers (e.g, presubmit system should consider the
    // result irrelevant for validatting a Chrome OS build).
    //
    // This action does not guarantee that results for the selected (Test,
    // Device Under Test) pairs are always available because that depends no how
    // the test plan Unit is interpreted overall. When this action is set, the
    // Test Lab Environment SHOULD NOT make any special effort to include /
    // exclude the selected (Test, Device Under Test) pairs.
    //
    // It is useful to set this Action for temporary exclusions where the
    // results generated from test execution can be uesd to root cause and fix
    // the underlying issues.
    MARK_NON_CRITICAL = 2;
  }
  Action action = 5;

  // The tests this exclusion applies to, within a test plan unit.
  //
  // Tests selected by this condition that are not in the test plan unit are
  // ignored. e.g., simply selecting all tests via the test_constraint will
  // apply this exclusion to all tests in the test plan unit.
  TestConstraint test_constraint = 2;

  // Constraints to exclude particular Device Under Test from being considered
  // to satisfy the test plan.
  //
  // Effectively, the negation of the constraint is added to the dut_constraints
  // for each selected test.
  DUTExclusionConstraint dut_constraint = 3;

  // External references useful for archeology for this exclusion.
  //
  // PERMANENT exclusions MUST add references for the decision to make the
  // exclusion PERMANENT.
  //
  // References should be links with more context behind the decision.
  // Suggested forms:
  // * Monorail bug: https://bugs.chromium.org/p/chromium/issues/detail?id=XXX
  // * Buganizer bug: https://b.corp.google.com/issues/XXX
  repeated string references = 4;
}

// Conditions to be met for the Chrome OS configuration of a Device Under
// Test for it to be excluded from consideration to satisfy a test plan unit.
message DUTExclusionConstraint {
  // A Common Expression Language (CEL) expression to specify constraints on a
  // Device Under Test's Chrome OS configuration payload. `expression` MUST
  // evaluate to a boolean value in the evaluation context described below.
  //
  // Test Lab Environments may optionally support excluding specific Devices
  // Under Test from being considered for a test plan unit.
  // When supported, the Test Lab Environment MUST effectively evaluate
  // `expression` with the following declarations in scope for each available
  // Device Under Test and ensure that the test is *not* scheduled on a Device
  // Under Test for which `expression` evaluates to true.
  //
  // - Constant: `dut` of type DUTConfigConstraint.DUT defined below, set to the
  //   Chrome OS configuration payload of a particular Device Under Test.
  // - Types: Protobuf messages from `chromiumos.config.api.*`
  //   - Additionally available with the short-hand `api.*`
  //
  // ## Examples
  //
  // Typical examples of expressions are:
  //
  // - Must not run on a device with a given screen size:
  //     dut.hardware_features.screen.milliinch.value == 14000
  // - TODO: Add model / build target example.
  //
  // ## CEL support
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // Current support for `expression` evaluation is very restricted due to
  // limitations in the scheduling infrastructure used by Test Platform.
  //
  // As this API matures, features will be added to the scheduling
  // infrastructure of Test Platform and restrictions here will be lifted based
  // on requirements collected from test authors. See milestones in
  // go/cros-f20-plan for expected feature iterations. Test Lab Environments
  // SHOULD validate the expression and reject use of unsupported features.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ### Syntax
  //
  // See full syntax definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#syntax
  //
  // CEL standard syntax allows expressions that evaluate to errors (e.g.,
  // syntax allows negation of lists, which has no semantics in CEL).
  // Thus, this spec does not attempt to restrict the syntax, but specifies what
  // operations are unsupported to aid metadata producers. Ultimately, the
  // reference metadata validator is the authority on what expressions are
  // allowed.
  //
  // Unsupported standard CEL semantics:
  //   - Binary arithmetic operations
  //     e.g.: +, *, /, % ...
  //   - Relational Operators beyond (in)equality are not supported.
  //     e.g.: (>, <, >=, <= ...)
  //   - Logical OR in expressions is not supported.
  //     e.g.: (a || b), !(a && b) ...
  //
  // ### Macros
  //
  // See full macro definition at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#macros
  //
  // Supported macros: has(), e.all()
  // Unsupported macros: e.exists(), e.exists_one(), e.map(), e.filter()
  //
  // ### Standard functions
  //
  // See full list of standard definitions at
  // https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions
  //
  // Most standard functions are not supported.
  //
  // - Supported operators: !_, -_, _!=_, _&&_, _=_, _[_]
  //   - All other operators are not supported.
  // - All other standard functions are not supported. In particular:
  //   - size() is not supported.
  //   - string functions like endsWith() and contains() are not supported.
  //   - type conversions like int() and string() are not supported.
  //   - reflection with type(), null_type() and dyn() is not supported.
  string expression = 1;

  // The evaluation context for `expression` MUST include the Chrome OS
  // configuration payload for a particular Device Under Test as a typed
  // constant of the following type.
  message DUT {
    chromiumos.config.api.HardwareFeatures hardware_features = 1;
    // TODO(pprabhu, shapiroc) Add a way to target model and build_target here.
  }
}

// Specifies platforms software covered by a test plan unit.
message CodeCoverage {
  // A Common Expression Language (CEL) expression to specify Chrome OS platform
  // sources covered by a test plan unit.
  //
  // Test Platform may optionally support skipping test plan units that are not
  // applicable for incoming changes to Chrome OS. If this feature is supported,
  // and if a non-empty blamelist is available, Test Platform MUST effectively
  // evaluate `expression` with the following declarations in scope for the
  // blamelist only include the test plan unit if the `expression` evaluates to
  // true.
  //
  // - Constant: `blamelist` of type CodeCoverage.Blamelist defined below,
  //   populated with blamelist attached to the request, if blamelist is
  //   non-empty.
  //
  // The full CEL spec can be found at https://github.com/google/cel-spec.
  //
  // TODO(crbug.com/1051689) Add reference to the metadata validator package.
  //
  // ## Examples
  //
  // Typical instructive examples of expressions are:
  //
  // - Only include test plan unit if there are changes to chromite
  //     blamelist.repo_paths.exists(p, p.startsWith("chromite"))
  // - Only include test plan unit if there are changes to chromite or autotest
  //     blamelist.repo_paths.exists(p, [
  //         "src/third_party/autotest/files",
  //         "chromite"
  //       ].exists(s, p.startsWith(s)))
  // - Do not include test plan unit if there are changes to private overlays
  //     blamelist.repo_paths.all(p, !p.contains("private-overlays"))
  //
  // ## CEL support
  //
  // All standard CEL syntax, macros and functions MUST be supported.
  string expression = 1;

  message Blamelist {
    // Changed paths (usually directories) in a typical checkout of the Chrome
    // OS source tree.
    //
    // Paths are relative to `repo` root. e.g., `src/platform2/shill`.
    repeated string repo_paths = 1;
  }
}