ml_benchmark/proto/benchmark_config.proto

syntax = "proto3";

package chrome.ml_benchmark;

enum AccelerationMode {
  NO_ACCELERATION = 0;
  NNAPI = 1;
}

// For details on what each of the fields do
// Refer to the CrOS ML Benchmarking Suite Design Document
// at go/cros-ml-benchmark-suite
message CrOSBenchmarkConfig {
  reserved 2, 3;

  AccelerationMode acceleration_mode = 1;

  // String representation of the driver proto configuration, to be decoded by
  // the benchmark driver implementation.
  string driver_config = 4;
}

// Benchmark-specific configurations
message SodaBenchmarkConfig {
  // Path to the SoDA configuration file, for speech recognition
  // this is usually dictation.config
  string soda_config_file_path = 1;
  // Whether to test with
  // - Realtime audio, which tests the latency of the audio
  // - Non-realtime audio, which tests the Real time factor (RTF)
  bool realtime_audio = 2;
  // number of times to test each input file
  int32 num_runs = 3;
  // Input files to test
  repeated string input_filename = 4;
  // The path to load the benchmarking library
  string soda_driver_path = 5;
}

enum BenchmarkReturnStatus {
  OK = 0;
  INCORRECT_CONFIGURATION = 1;
  INITIALIZATION_FAILED = 2;
  BENCHMARK_TIMED_OUT = 3;
  RUNTIME_ERROR = 4;
}

message BenchmarkResults {
  BenchmarkReturnStatus status = 1;
  string results_message = 2;
  // accuracy of inference
  float total_accuracy = 3;
  // originally, test drivers only outputs the average latency
  // This has been modified to output latencies at different percentiles
  int64 average_latency_in_us = 4 [deprecated = true];
  // For new code, use this instead, which allows us to output 50th and 90th
  // Percentile latencies
  map<int32, int64> percentile_latencies_in_us = 5;
}