media_perception/mojom/media_types.mojom

// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

module media.mojom;

import "mojom/mailbox_holder.mojom";
import "mojom/vulkan_ycbcr_info.mojom";
import "mojom/time.mojom";
import "mojom/values.mojom";
import "mojom/unguessable_token.mojom";
import "mojom/geometry.mojom";
import "mojom/buffer_types.mojom";
import "mojom/color_space.mojom";
import "mojom/hdr_metadata.mojom";

// See media/base/audio_codecs.h for descriptions.
[Native]
enum AudioCodec;
[Native]
enum AudioCodecProfile;

// See media/base/buffering_state.h for descriptions.
[Native]
enum BufferingState;
[Native]
enum BufferingStateChangeReason;

// See media/base/channel_layout.h for descriptions.
[Native]
enum ChannelLayout;

// See media/base/decode_status.h for descriptions.
[Native]
enum DecodeStatus;

// See media/base/status_codes.h for descriptions.
[Native]
enum StatusCode;

// See media/base/media_content_type.h for descriptions.
[Native]
enum MediaContentType;

// See media/base/media_log_record.h for description.
[Native]
struct MediaLogRecord;

// See media/base/output_device_info.h for descriptions.
[Native]
enum OutputDeviceStatus;

// See media/base/pipeline_status.h for descriptions.
[Native]
enum PipelineStatus;

// See media/base/sample_format.h for descriptions.
[Native]
enum SampleFormat;

// See media/base/video_codecs.h for descriptions.
[Native]
enum VideoCodec;

// See media/base/video_codecs.h for descriptions.
[Native]
enum VideoCodecProfile;

// See media/base/video_types.h for descriptions.
[Native]
enum VideoPixelFormat;

// See media/base/decoder.h for descriptions.
[Native]
enum VideoDecoderType;

// See media/base/decoder.h for descriptions.
[Native]
enum AudioDecoderType;

// See media/base/video_transformation.h for descriptions.
enum VideoRotation {
  kVideoRotation0,
  kVideoRotation90,
  kVideoRotation180,
  kVideoRotation270,
};

// see media/base/video_frame_metadata.h for descriptions.
enum CopyMode {
  kCopyToNewTexture,
  kCopyMailboxesOnly,
};

// see third_party/blink/public/platform/web_fullscreen_video_status.h for
// descriptions.
[Native]
enum FullscreenVideoStatus;

// See media/base/video_transformation.h for descriptions.
struct VideoTransformation {
  VideoRotation rotation;
  bool mirrored;
};

// See media/base/waiting.h for descriptions.
[Native]
enum WaitingReason;

// See media/base/watch_time_keys.h for descriptions.
[Native]
enum WatchTimeKey;

// See media/base/container_names.h for descriptions.
[Native]
enum MediaContainerName;

// See media/base/media_status.h for description.
[Native]
enum MediaStatusState;

// This defines a mojo transport format for media::EncryptionPattern
// See media/base/encryption_pattern.h for description.
struct EncryptionPattern {
  uint32 crypt_byte_block;
  uint32 skip_byte_block;
};

// See media/base/encryption_scheme.h for description.
[Native]
enum EncryptionScheme;

// This defines a mojo transport format for media::VideoColorSpace.
// See media/base/video_color_space.h for description.
struct VideoColorSpace {
  [Native]
  enum PrimaryID;

  [Native]
  enum TransferID;

  [Native]
  enum MatrixID;

  [Native]
  enum RangeID;

  PrimaryID primaries;
  TransferID transfer;
  MatrixID matrix;
  RangeID range;
};


// This defines a mojo transport format for media::AudioDecoderConfig.
// See media/base/audio_decoder_config.h for descriptions.
struct AudioDecoderConfig {
  AudioCodec codec;
  AudioCodecProfile profile;
  SampleFormat sample_format;
  ChannelLayout channel_layout;
  int32 samples_per_second;
  array<uint8> extra_data;
  mojo_base.mojom.TimeDelta seek_preroll;
  int32 codec_delay;
  EncryptionScheme encryption_scheme;
};

// This defines a mojo transport format for media::VideoDecoderConfig.
// See media/base/video_decoder_config.h for descriptions.
struct VideoDecoderConfig {
  VideoCodec codec;
  VideoCodecProfile profile;
  uint32 level;
  bool has_alpha;
  VideoTransformation transformation;
  gfx.mojom.Size coded_size;
  gfx.mojom.Rect visible_rect;
  gfx.mojom.Size natural_size;
  array<uint8> extra_data;
  EncryptionScheme encryption_scheme;
  VideoColorSpace color_space_info;
  gfx.mojom.HDRMetadata? hdr_metadata;
};

// Native struct media::SubsampleEntry;
[Native]
struct SubsampleEntry;

// This defines a mojo transport format for media::DecryptConfig.
// See media/base/decrypt_config.h for descriptions.
struct DecryptConfig {
  EncryptionScheme encryption_scheme;
  string key_id;
  string iv;
  array<SubsampleEntry> subsamples;
  EncryptionPattern? encryption_pattern;
};

// This defines a mojo transport format for media::DecoderBuffer.
struct DecoderBuffer {
  mojo_base.mojom.TimeDelta timestamp;
  mojo_base.mojom.TimeDelta duration;

  // Whether the buffer is an end-of-stream (EOS) buffer.
  bool is_end_of_stream;

  // The number of bytes present in this buffer. The data is not serialized
  // along with this structure and must be read from a separate DataPipe.
  // Note that |data_size| could be zero even for a non-EOS buffer (e.g.
  // with non-empty |size_data|). See http://crbug.com/663438
  uint32 data_size;

  // Indicates whether or not this buffer is a random access point.
  bool is_key_frame;

  // Empty when |side_data| doesn't exist.
  array<uint8> side_data;

  // DecryptConfig for a encrypted buffer. NULL if the buffer is not encrypted.
  DecryptConfig? decrypt_config;

  // These fields indicate the amount of data to discard after decoding.
  mojo_base.mojom.TimeDelta front_discard;
  mojo_base.mojom.TimeDelta back_discard;
};

// This defines a mojo transport format for media::AudioBuffer.
struct AudioBuffer {
  // Format of the audio.
  SampleFormat sample_format;

  // How the channels are laid out.
  ChannelLayout channel_layout;

  // Number of channels.
  int32 channel_count;

  // Sample rate of the buffer.
  int32 sample_rate;

  // Number of frames in the buffer.
  int32 frame_count;

  // True if end of stream.
  bool end_of_stream;

  // Timestamp in microseconds of the first frame.
  mojo_base.mojom.TimeDelta timestamp;

  // Channel data. Will be empty for EOS buffers.
  array<uint8> data;
};

// This defines a mojo transport format for an interleaved, signed
// 16-bit audio buffer.
struct AudioDataS16 {
  // Number of channels.
  int32 channel_count;

  // Sample rate of the buffer.
  int32 sample_rate;

  // Number of frames in the buffer.
  int32 frame_count;

  // Channel data.
  array<int16> data;
};

// See media/base/video_frame_metadata.h for a description of fields.
// TODO(crbug.com/657632): Remove |has_*| values and use nullable types.
struct VideoFrameMetadata {
  bool allow_overlay;

  mojo_base.mojom.TimeTicks? capture_begin_time;
  mojo_base.mojom.TimeTicks? capture_end_time;

  bool has_capture_counter;
  int32 capture_counter;

  gfx.mojom.Rect? capture_update_rect;

  bool has_copy_mode;
  CopyMode copy_mode;

  bool end_of_stream;

  mojo_base.mojom.TimeDelta? frame_duration;

  bool has_frame_rate;
  double frame_rate;

  bool interactive_content;

  mojo_base.mojom.TimeTicks? reference_time;

  bool read_lock_fences_enabled;

  VideoTransformation? transformation;

  bool texture_owner;

  bool wants_promotion_hint;

  bool protected_video;

  bool hw_protected;

  mojo_base.mojom.UnguessableToken? overlay_plane_id;

  bool power_efficient;

  bool has_device_scale_factor;
  double device_scale_factor;

  bool has_page_scale_factor;
  double page_scale_factor;

  bool has_root_scroll_offset_x;
  double root_scroll_offset_x;

  bool has_root_scroll_offset_y;
  double root_scroll_offset_y;

  bool has_top_controls_visible_height;
  double top_controls_visible_height;

  mojo_base.mojom.TimeTicks? decode_begin_time;
  mojo_base.mojom.TimeTicks? decode_end_time;

  mojo_base.mojom.TimeDelta? processing_time;

  bool has_rtp_timestamp;
  double rtp_timestamp;

  mojo_base.mojom.TimeTicks? receive_time;

  mojo_base.mojom.TimeDelta? wallclock_frame_duration;

  uint32 hw_protected_validation_id;
};

// This defines a mojo transport format for media::VideoFrame.
struct VideoFrame {
  // Format of the frame.
  VideoPixelFormat format;

  // Width and height of the video frame, in pixels.
  gfx.mojom.Size coded_size;

  // Visible size of the frame.
  gfx.mojom.Rect visible_rect;

  // Natural size of the frame.
  gfx.mojom.Size natural_size;

  // Timestamp in microseconds of the associated frame.
  mojo_base.mojom.TimeDelta timestamp;

  // Contents of the video frame (or EOS marker).
  VideoFrameData data;

  // Extra properties associated with the VideoFrame.
  VideoFrameMetadata metadata;

  gfx.mojom.ColorSpace color_space;
  gfx.mojom.HDRMetadata? hdr_metadata;
};

// Possible choices for storing VideoFrame data.
union VideoFrameData {
  EosVideoFrameData eos_data;
  SharedBufferVideoFrameData shared_buffer_data;
  DmabufVideoFrameData dmabuf_data;
  GpuMemoryBufferVideoFrameData gpu_memory_buffer_data;
  MailboxVideoFrameData mailbox_data;
};

// A marker for EOS frames.
struct EosVideoFrameData {
};

// This defines video frame data stored in a Mojo shared buffer.
struct SharedBufferVideoFrameData {
  // Reference to the shared memory containing the frame's data.
  handle<shared_buffer> frame_data;
  uint64 frame_data_size;

  // Stride and offsets for each plane. Offsets are relative to the start
  // of |frame_data|.
  array<int32> strides;
  array<uint32> offsets;
};

// This defines video frame data stored in dmabuf.
struct DmabufVideoFrameData {
  // Size depends on media::VideoFrame::NumPlanes with frame format.
  array<handle<platform>> dmabuf_fds;
};

struct GpuMemoryBufferVideoFrameData {
  gfx.mojom.GpuMemoryBufferHandle gpu_memory_buffer_handle;
  array<gpu.mojom.MailboxHolder, 4> mailbox_holder;
};

// This defines video frame data stored in texture mailboxes.
struct MailboxVideoFrameData {
  // Size must be kept in sync with media::VideoFrame::kMaxPlanes.
  array<gpu.mojom.MailboxHolder, 4> mailbox_holder;
  gpu.mojom.VulkanYCbCrInfo? ycbcr_data;
};

struct PipelineStatistics {
  uint64 audio_bytes_decoded;
  uint64 video_bytes_decoded;
  uint32 video_frames_decoded;
  uint32 video_frames_dropped;
  int64 audio_memory_usage;
  int64 video_memory_usage;
};

// Set of features for MediaCapabilities prediction.
// TODO(liberato): consider generalizing this.
struct PredictionFeatures {
    // It would be nice to initialize this to VIDEO_CODEC_PROFILE_UNKNOWN (-1),
    // but we can't do that with native enums.
    VideoCodecProfile profile;
    gfx.mojom.Size video_size;
    // Frames per second may ultimately be a bucketed as an integer, but we want
    // to do that as late as possible. Using a double avoids early truncation.
    double frames_per_sec = 0;
    // Name of the key system used for EME playbacks.
    string key_system;
    // Indicates when CDM will use HW secure decoding for EME playbacks.
    bool use_hw_secure_codecs = false;
};

// Target values for MediaCapabilities predictions.
// TODO(liberato): consider generalizing this.
struct PredictionTargets {
  uint32 frames_decoded = 0;
  uint32 frames_dropped = 0;
  uint32 frames_power_efficient = 0;
};

// See media/base/pipeline_status.h for descriptions.
struct AudioDecoderInfo {
  AudioDecoderType decoder_type;
  bool is_platform_decoder = false;
  bool has_decrypting_demuxer_stream = false;
};

// See media/base/pipeline_status.h for descriptions.
struct VideoDecoderInfo {
  VideoDecoderType decoder_type;
  bool is_platform_decoder = false;
  bool has_decrypting_demuxer_stream = false;
};

// See media/base/status.h for descriptions.
struct Status {
  StatusCode code;
  string? message;
  array<mojo_base.mojom.Value> frames;
  array<media.mojom.Status> causes;
  mojo_base.mojom.Value? data;
};

// Types of media stream, categorised by the media stream's source.
// The enum values are emitted to metrics. Do not reorder.
enum MediaStreamType {
  kLocalElementCapture = 0, // The source is a local capture from element.
  kLocalDeviceCapture = 1, // The source is a local device capture, e.g. webcam.
  kLocalTabCapture = 2, // The source is a local tab capture.
  kLocalDesktopCapture = 3, // The source is a local desktop capture.
  kLocalDisplayCapture = 4, // The source is a local display capture.
  kRemote = 5, // The source is a remote peer connection.
  kNone = 6, // Not a media stream.
};