Sync to upstream webrtc checkout
This change updates webrtc source to commit 62c0c2d1e
and modifies Makefile so everything can build.
BUG=1041306
TEST=emerge-eve webrtc-apm
emerge-veyron_jerry webrtc-apm
Change-Id: Idae39b1ad32ea7826999985461271a2a71e21f83
Reviewed-on: https://chromium-review.googlesource.com/1046725
Commit-Ready: Hsinyu Chao <hychao@chromium.org>
Tested-by: Hsinyu Chao <hychao@chromium.org>
Reviewed-by: Cheng-Yi Chiang <cychiang@chromium.org>
diff --git a/Makefile b/Makefile
index 70040e9..0a23258 100644
--- a/Makefile
+++ b/Makefile
@@ -36,10 +36,10 @@
CXX_LIBRARY(libwebrtc_apm.so): LDLIBS += \
libaudio_processing.pic.a \
- rtc_base/librtc_base.pic.a \
common_audio/libcommon_audio.pic.a \
system_wrappers/source/libsystem_wrappers.pic.a \
modules/audio_coding/libaudio_coding.pic.a \
+ rtc_base/librtc_base.pic.a \
libaudioproc_debug_proto.pic.a \
$(call get_pc_libs,$(webrtc_apm_PC_DEPS))
diff --git a/api/audio/audio_frame.cc b/api/audio/audio_frame.cc
index 108a523..b477a17 100644
--- a/api/audio/audio_frame.cc
+++ b/api/audio/audio_frame.cc
@@ -8,12 +8,11 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include <string.h>
-
#include "api/audio/audio_frame.h"
+#include <string.h>
+
#include "rtc_base/checks.h"
-#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/timeutils.h"
namespace webrtc {
@@ -43,12 +42,12 @@
}
void AudioFrame::UpdateFrame(uint32_t timestamp,
- const int16_t* data,
- size_t samples_per_channel,
- int sample_rate_hz,
- SpeechType speech_type,
- VADActivity vad_activity,
- size_t num_channels) {
+ const int16_t* data,
+ size_t samples_per_channel,
+ int sample_rate_hz,
+ SpeechType speech_type,
+ VADActivity vad_activity,
+ size_t num_channels) {
timestamp_ = timestamp;
samples_per_channel_ = samples_per_channel;
sample_rate_hz_ = sample_rate_hz;
@@ -119,67 +118,10 @@
bool AudioFrame::muted() const { return muted_; }
-AudioFrame& AudioFrame::operator>>=(const int rhs) {
- RTC_CHECK_GT(num_channels_, 0);
- RTC_CHECK_LT(num_channels_, 3);
- if ((num_channels_ > 2) || (num_channels_ < 1)) return *this;
- if (muted_) return *this;
-
- for (size_t i = 0; i < samples_per_channel_ * num_channels_; i++) {
- data_[i] = static_cast<int16_t>(data_[i] >> rhs);
- }
- return *this;
-}
-
-AudioFrame& AudioFrame::operator+=(const AudioFrame& rhs) {
- // Sanity check
- RTC_CHECK_GT(num_channels_, 0);
- RTC_CHECK_LT(num_channels_, 3);
- if ((num_channels_ > 2) || (num_channels_ < 1)) return *this;
- if (num_channels_ != rhs.num_channels_) return *this;
-
- bool noPrevData = muted_;
- if (samples_per_channel_ != rhs.samples_per_channel_) {
- if (samples_per_channel_ == 0) {
- // special case we have no data to start with
- samples_per_channel_ = rhs.samples_per_channel_;
- noPrevData = true;
- } else {
- return *this;
- }
- }
-
- if ((vad_activity_ == kVadActive) || rhs.vad_activity_ == kVadActive) {
- vad_activity_ = kVadActive;
- } else if (vad_activity_ == kVadUnknown || rhs.vad_activity_ == kVadUnknown) {
- vad_activity_ = kVadUnknown;
- }
-
- if (speech_type_ != rhs.speech_type_) speech_type_ = kUndefined;
-
- if (!rhs.muted()) {
- muted_ = false;
- if (noPrevData) {
- memcpy(data_, rhs.data(),
- sizeof(int16_t) * rhs.samples_per_channel_ * num_channels_);
- } else {
- // IMPROVEMENT this can be done very fast in assembly
- for (size_t i = 0; i < samples_per_channel_ * num_channels_; i++) {
- int32_t wrap_guard =
- static_cast<int32_t>(data_[i]) + static_cast<int32_t>(rhs.data_[i]);
- data_[i] = rtc::saturated_cast<int16_t>(wrap_guard);
- }
- }
- }
-
- return *this;
-}
-
// static
const int16_t* AudioFrame::empty_data() {
- static const int16_t kEmptyData[kMaxDataSizeSamples] = {0};
- static_assert(sizeof(kEmptyData) == kMaxDataSizeBytes, "kMaxDataSizeBytes");
- return kEmptyData;
+ static int16_t* null_data = new int16_t[kMaxDataSizeSamples]();
+ return &null_data[0];
}
} // namespace webrtc
diff --git a/api/audio/audio_frame.h b/api/audio/audio_frame.h
index 5cb2019..39840e5 100644
--- a/api/audio/audio_frame.h
+++ b/api/audio/audio_frame.h
@@ -11,11 +11,9 @@
#ifndef API_AUDIO_AUDIO_FRAME_H_
#define API_AUDIO_AUDIO_FRAME_H_
-#include <stdint.h>
-#include <stdlib.h>
+#include <stddef.h>
#include "rtc_base/constructormagic.h"
-#include "rtc_base/deprecation.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
@@ -68,17 +66,6 @@
// ResetWithoutMuting() to skip this wasteful zeroing.
void ResetWithoutMuting();
- // TODO(solenberg): Remove once downstream users of AudioFrame have updated.
- RTC_DEPRECATED
- void UpdateFrame(int id, uint32_t timestamp, const int16_t* data,
- size_t samples_per_channel, int sample_rate_hz,
- SpeechType speech_type, VADActivity vad_activity,
- size_t num_channels = 1) {
- RTC_UNUSED(id);
- UpdateFrame(timestamp, data, samples_per_channel, sample_rate_hz,
- speech_type, vad_activity, num_channels);
- }
-
void UpdateFrame(uint32_t timestamp, const int16_t* data,
size_t samples_per_channel, int sample_rate_hz,
SpeechType speech_type, VADActivity vad_activity,
@@ -108,13 +95,6 @@
// Frame is muted by default.
bool muted() const;
- // These methods are deprecated. Use the functions in
- // webrtc/audio/utility instead. These methods will exists for a
- // short period of time until webrtc clients have updated. See
- // webrtc:6548 for details.
- RTC_DEPRECATED AudioFrame& operator>>=(const int rhs);
- RTC_DEPRECATED AudioFrame& operator+=(const AudioFrame& rhs);
-
// RTP timestamp of the first sample in the AudioFrame.
uint32_t timestamp_ = 0;
// Time since the first frame in milliseconds.
diff --git a/api/audio/echo_canceller3_config.cc b/api/audio/echo_canceller3_config.cc
index d74d7a8..c17b0d5 100644
--- a/api/audio/echo_canceller3_config.cc
+++ b/api/audio/echo_canceller3_config.cc
@@ -12,5 +12,9 @@
namespace webrtc {
EchoCanceller3Config::EchoCanceller3Config() = default;
+EchoCanceller3Config::EchoCanceller3Config(const EchoCanceller3Config& e) =
+ default;
+EchoCanceller3Config::Mask::Mask() = default;
+EchoCanceller3Config::Mask::Mask(const EchoCanceller3Config::Mask& m) = default;
} // namespace webrtc
diff --git a/api/audio/echo_canceller3_config.h b/api/audio/echo_canceller3_config.h
index fd5bf09..174ef6c 100644
--- a/api/audio/echo_canceller3_config.h
+++ b/api/audio/echo_canceller3_config.h
@@ -18,7 +18,7 @@
// Configuration struct for EchoCanceller3
struct EchoCanceller3Config {
EchoCanceller3Config();
-
+ EchoCanceller3Config(const EchoCanceller3Config& e);
struct Delay {
size_t default_delay = 5;
size_t down_sampling_factor = 4;
@@ -57,7 +57,7 @@
struct Erle {
float min = 1.f;
- float max_l = 8.f;
+ float max_l = 4.f;
float max_h = 1.5f;
} erle;
@@ -71,20 +71,32 @@
} ep_strength;
struct Mask {
+ Mask();
+ Mask(const Mask& m);
float m1 = 0.01f;
float m2 = 0.0001f;
float m3 = 0.01f;
- float m4 = 0.1f;
- float m5 = 0.1f;
+ float m5 = 0.01f;
float m6 = 0.0001f;
float m7 = 0.01f;
float m8 = 0.0001f;
float m9 = 0.1f;
+
+ float gain_curve_offset = 1.45f;
+ float gain_curve_slope = 5.f;
+ float temporal_masking_lf = 0.9f;
+ float temporal_masking_hf = 0.6f;
+ size_t temporal_masking_lf_bands = 3;
} gain_mask;
struct EchoAudibility {
float low_render_limit = 4 * 64.f;
float normal_render_limit = 64.f;
+ float floor_power = 2 * 64.f;
+ float audibility_threshold_lf = 10;
+ float audibility_threshold_mf = 10;
+ float audibility_threshold_hf = 10;
+ bool use_stationary_properties = false;
} echo_audibility;
struct RenderLevels {
@@ -120,6 +132,22 @@
bool has_clock_drift = false;
} echo_removal_control;
+
+ struct EchoModel {
+ size_t noise_floor_hold = 50;
+ float min_noise_floor_power = 1638400.f;
+ float stationary_gate_slope = 10.f;
+ float noise_gate_power = 27509.42f;
+ float noise_gate_slope = 0.3f;
+ size_t render_pre_window_size = 1;
+ size_t render_post_window_size = 1;
+ float nonlinear_hold = 1;
+ float nonlinear_release = 0.001f;
+ } echo_model;
+
+ struct Suppressor {
+ size_t bands_with_reliable_coherence = 5;
+ } suppressor;
};
} // namespace webrtc
diff --git a/api/audio/echo_control.h b/api/audio/echo_control.h
index 021bbf8..f549f40 100644
--- a/api/audio/echo_control.h
+++ b/api/audio/echo_control.h
@@ -38,6 +38,9 @@
// Collect current metrics from the echo controller.
virtual Metrics GetMetrics() const = 0;
+ // Provides an optional external estimate of the audio buffer delay.
+ virtual void SetAudioBufferDelay(size_t delay_ms) = 0;
+
virtual ~EchoControl() {}
};
diff --git a/api/audio_options.cc b/api/audio_options.cc
new file mode 100644
index 0000000..c196d7d
--- /dev/null
+++ b/api/audio_options.cc
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/audio_options.h"
+
+namespace cricket {
+
+AudioOptions::AudioOptions() = default;
+AudioOptions::~AudioOptions() = default;
+
+} // namespace cricket
diff --git a/api/audio_options.h b/api/audio_options.h
index 8d2880b..5d69842 100644
--- a/api/audio_options.h
+++ b/api/audio_options.h
@@ -23,6 +23,8 @@
// We are moving all of the setting of options to structs like this,
// but some things currently still use flags.
struct AudioOptions {
+ AudioOptions();
+ ~AudioOptions();
void SetAll(const AudioOptions& change) {
SetFrom(&echo_cancellation, change.echo_cancellation);
#if defined(WEBRTC_IOS)
diff --git a/api/candidate.cc b/api/candidate.cc
index 62cd1bd..d51fb84 100644
--- a/api/candidate.cc
+++ b/api/candidate.cc
@@ -73,8 +73,9 @@
sensitive ? address_.ToSensitiveString() : address_.ToString();
ost << "Cand[" << transport_name_ << ":" << foundation_ << ":" << component_
<< ":" << protocol_ << ":" << priority_ << ":" << address << ":" << type_
- << ":" << related_address_ << ":" << username_ << ":" << password_ << ":"
- << network_id_ << ":" << network_cost_ << ":" << generation_ << "]";
+ << ":" << related_address_.ToString() << ":" << username_ << ":"
+ << password_ << ":" << network_id_ << ":" << network_cost_ << ":"
+ << generation_ << "]";
return ost.str();
}
diff --git a/api/optional.cc b/api/optional.cc
index 9412617..0f74bd2 100644
--- a/api/optional.cc
+++ b/api/optional.cc
@@ -15,7 +15,7 @@
#if RTC_HAS_ASAN
-void* FunctionThatDoesNothingImpl(void* x) {
+const void* FunctionThatDoesNothingImpl(const void* x) {
return x;
}
diff --git a/api/optional.h b/api/optional.h
index 7a62335..ba06831 100644
--- a/api/optional.h
+++ b/api/optional.h
@@ -33,18 +33,18 @@
// This is a non-inlined function. The optimizer can't see inside it. It
// prevents the compiler from generating optimized code that reads value_ even
// if it is unset. Although safe, this causes memory sanitizers to complain.
-void* FunctionThatDoesNothingImpl(void*);
+const void* FunctionThatDoesNothingImpl(const void*);
template <typename T>
-inline T* FunctionThatDoesNothing(T* x) {
- return reinterpret_cast<T*>(
- FunctionThatDoesNothingImpl(reinterpret_cast<void*>(x)));
+inline const T* FunctionThatDoesNothing(T* x) {
+ return reinterpret_cast<const T*>(
+ FunctionThatDoesNothingImpl(reinterpret_cast<const void*>(x)));
}
#else
template <typename T>
-inline T* FunctionThatDoesNothing(T* x) {
+inline const T* FunctionThatDoesNothing(T* x) {
return x;
}
@@ -281,12 +281,6 @@
: default_val;
}
- // Dereference and move value.
- T MoveValue() {
- RTC_DCHECK(has_value_);
- return std::move(value_);
- }
-
// Equality tests. Two Optionals are equal if they contain equivalent values,
// or if they're both empty.
friend bool operator==(const Optional& m1, const Optional& m2) {
diff --git a/api/optional_unittest.cc b/api/optional_unittest.cc
index 2149033..ad700dc 100644
--- a/api/optional_unittest.cc
+++ b/api/optional_unittest.cc
@@ -861,7 +861,7 @@
{
Optional<Logger> x(Logger(42));
log->push_back("---");
- Logger moved = x.MoveValue();
+ Logger moved = std::move(x.value());
log->push_back("---");
}
EXPECT_EQ(
diff --git a/api/peerconnectionfactoryproxy.h b/api/peerconnectionfactoryproxy.h
index 7601ed1..7777809 100644
--- a/api/peerconnectionfactoryproxy.h
+++ b/api/peerconnectionfactoryproxy.h
@@ -44,6 +44,10 @@
std::unique_ptr<cricket::PortAllocator>,
std::unique_ptr<rtc::RTCCertificateGeneratorInterface>,
PeerConnectionObserver*);
+ PROXY_METHOD2(rtc::scoped_refptr<PeerConnectionInterface>,
+ CreatePeerConnection,
+ const PeerConnectionInterface::RTCConfiguration&,
+ PeerConnectionDependencies);
PROXY_METHOD1(rtc::scoped_refptr<MediaStreamInterface>,
CreateLocalMediaStream, const std::string&)
PROXY_METHOD1(rtc::scoped_refptr<AudioSourceInterface>,
diff --git a/api/peerconnectioninterface.h b/api/peerconnectioninterface.h
index 4cb3d65..6e39b4f 100644
--- a/api/peerconnectioninterface.h
+++ b/api/peerconnectioninterface.h
@@ -67,10 +67,6 @@
#ifndef API_PEERCONNECTIONINTERFACE_H_
#define API_PEERCONNECTIONINTERFACE_H_
-// TODO(sakal): Remove this define after migration to virtual PeerConnection
-// observer is complete.
-#define VIRTUAL_PEERCONNECTION_OBSERVER_DESTRUCTOR
-
#include <memory>
#include <string>
#include <utility>
@@ -158,9 +154,7 @@
virtual ~StatsObserver() {}
};
-// For now, kDefault is interpreted as kPlanB.
-// TODO(bugs.webrtc.org/8530): Switch default to kUnifiedPlan.
-enum class SdpSemantics { kDefault, kPlanB, kUnifiedPlan };
+enum class SdpSemantics { kPlanB, kUnifiedPlan };
class PeerConnectionInterface : public rtc::RefCountInterface {
public:
@@ -563,15 +557,12 @@
// will also cause PeerConnection to ignore all but the first a=ssrc lines
// that form a Plan B stream.
//
- // For users who only send at most one audio and one video track, this
- // choice does not matter and should be left as kDefault.
- //
// For users who wish to send multiple audio/video streams and need to stay
- // interoperable with legacy WebRTC implementations, specify kPlanB.
+ // interoperable with legacy WebRTC implementations or use legacy APIs,
+ // specify kPlanB.
//
- // For users who wish to send multiple audio/video streams and/or wish to
- // use the new RtpTransceiver API, specify kUnifiedPlan.
- SdpSemantics sdp_semantics = SdpSemantics::kDefault;
+ // For all other users, specify kUnifiedPlan.
+ SdpSemantics sdp_semantics = SdpSemantics::kPlanB;
//
// Don't forget to update operator== if adding something.
@@ -809,15 +800,46 @@
return {};
}
+ // The legacy non-compliant GetStats() API. This correspond to the
+ // callback-based version of getStats() in JavaScript. The returned metrics
+ // are UNDOCUMENTED and many of them rely on implementation-specific details.
+ // The goal is to DELETE THIS VERSION but we can't today because it is heavily
+ // relied upon by third parties. See https://crbug.com/822696.
+ //
+ // This version is wired up into Chrome. Any stats implemented are
+ // automatically exposed to the Web Platform. This has BYPASSED the Chrome
+ // release processes for years and lead to cross-browser incompatibility
+ // issues and web application reliance on Chrome-only behavior.
+ //
+ // This API is in "maintenance mode", serious regressions should be fixed but
+ // adding new stats is highly discouraged.
+ //
+ // TODO(hbos): Deprecate and remove this when third parties have migrated to
+ // the spec-compliant GetStats() API. https://crbug.com/822696
virtual bool GetStats(StatsObserver* observer,
- MediaStreamTrackInterface* track,
+ MediaStreamTrackInterface* track, // Optional
StatsOutputLevel level) = 0;
- // Gets stats using the new stats collection API, see webrtc/api/stats/. These
- // will replace old stats collection API when the new API has matured enough.
- // TODO(hbos): Default implementation that does nothing only exists as to not
- // break third party projects. As soon as they have been updated this should
- // be changed to "= 0;".
+ // The spec-compliant GetStats() API. This correspond to the promise-based
+ // version of getStats() in JavaScript. Implementation status is described in
+ // api/stats/rtcstats_objects.h. For more details on stats, see spec:
+ // https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnection-getstats
+ // TODO(hbos): Takes shared ownership, use rtc::scoped_refptr<> instead. This
+ // requires stop overriding the current version in third party or making third
+ // party calls explicit to avoid ambiguity during switch. Make the future
+ // version abstract as soon as third party projects implement it.
virtual void GetStats(RTCStatsCollectorCallback* callback) {}
+ // Spec-compliant getStats() performing the stats selection algorithm with the
+ // sender. https://w3c.github.io/webrtc-pc/#dom-rtcrtpsender-getstats
+ // TODO(hbos): Make abstract as soon as third party projects implement it.
+ virtual void GetStats(
+ rtc::scoped_refptr<RtpSenderInterface> selector,
+ rtc::scoped_refptr<RTCStatsCollectorCallback> callback) {}
+ // Spec-compliant getStats() performing the stats selection algorithm with the
+ // receiver. https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getstats
+ // TODO(hbos): Make abstract as soon as third party projects implement it.
+ virtual void GetStats(
+ rtc::scoped_refptr<RtpReceiverInterface> selector,
+ rtc::scoped_refptr<RTCStatsCollectorCallback> callback) {}
// Clear cached stats in the RTCStatsCollector.
// Exposed for testing while waiting for automatic cache clear to work.
// https://bugs.webrtc.org/8693
@@ -897,13 +919,6 @@
virtual void SetRemoteDescription(
std::unique_ptr<SessionDescriptionInterface> desc,
rtc::scoped_refptr<SetRemoteDescriptionObserverInterface> observer) {}
- // Deprecated; Replaced by SetConfiguration.
- // TODO(deadbeef): Remove once Chrome is moved over to SetConfiguration.
- virtual bool UpdateIce(const IceServers& configuration,
- const MediaConstraintsInterface* constraints) {
- return false;
- }
- virtual bool UpdateIce(const IceServers& configuration) { return false; }
// TODO(deadbeef): Make this pure virtual once all Chrome subclasses of
// PeerConnectionInterface implement it.
@@ -1125,13 +1140,6 @@
virtual void OnTrack(
rtc::scoped_refptr<RtpTransceiverInterface> transceiver) {}
- // TODO(hbos,deadbeef): Add |OnAssociatedStreamsUpdated| with |receiver| and
- // |streams| as arguments. This should be called when an existing receiver its
- // associated streams updated. https://crbug.com/webrtc/8315
- // This may be blocked on supporting multiple streams per sender or else
- // this may count as the removal and addition of a track?
- // https://crbug.com/webrtc/7932
-
// Called when a receiver is completely removed. This is current (Plan B SDP)
// behavior that occurs when processing the removal of a remote track, and is
// called when the receiver is removed and the track is muted. When Unified
@@ -1145,6 +1153,30 @@
rtc::scoped_refptr<RtpReceiverInterface> receiver) {}
};
+// PeerConnectionDependencies holds all of PeerConnections dependencies.
+// A dependency is distinct from a configuration as it defines significant
+// executable code that can be provided by a user of the API.
+//
+// All new dependencies should be added as a unique_ptr to allow the
+// PeerConnection object to be the definitive owner of the dependencies
+// lifetime making injection safer.
+struct PeerConnectionDependencies final {
+ explicit PeerConnectionDependencies(PeerConnectionObserver* observer_in)
+ : observer(observer_in) {}
+ // This object is not copyable or assignable.
+ PeerConnectionDependencies(const PeerConnectionDependencies&) = delete;
+ PeerConnectionDependencies& operator=(const PeerConnectionDependencies&) =
+ delete;
+ // This object is only moveable.
+ PeerConnectionDependencies(PeerConnectionDependencies&&) = default;
+ PeerConnectionDependencies& operator=(PeerConnectionDependencies&&) = default;
+ // Mandatory dependencies
+ PeerConnectionObserver* observer = nullptr;
+ // Optional dependencies
+ std::unique_ptr<cricket::PortAllocator> allocator;
+ std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator;
+};
+
// PeerConnectionFactoryInterface is the factory interface used for creating
// PeerConnection, MediaStream and MediaStreamTrack objects.
//
@@ -1197,8 +1229,18 @@
// Set the options to be used for subsequently created PeerConnections.
virtual void SetOptions(const Options& options) = 0;
- // |allocator| and |cert_generator| may be null, in which case default
- // implementations will be used.
+ // The preferred way to create a new peer connection. Simply provide the
+ // configuration and a PeerConnectionDependencies structure.
+ // TODO(benwright): Make pure virtual once downstream mock PC factory classes
+ // are updated.
+ virtual rtc::scoped_refptr<PeerConnectionInterface> CreatePeerConnection(
+ const PeerConnectionInterface::RTCConfiguration& configuration,
+ PeerConnectionDependencies dependencies) {
+ return nullptr;
+ }
+
+ // Deprecated; |allocator| and |cert_generator| may be null, in which case
+ // default implementations will be used.
//
// |observer| must not be null.
//
diff --git a/api/peerconnectionproxy.h b/api/peerconnectionproxy.h
index 7235f5b..9325adc 100644
--- a/api/peerconnectionproxy.h
+++ b/api/peerconnectionproxy.h
@@ -70,6 +70,14 @@
MediaStreamTrackInterface*,
StatsOutputLevel)
PROXY_METHOD1(void, GetStats, RTCStatsCollectorCallback*)
+ PROXY_METHOD2(void,
+ GetStats,
+ rtc::scoped_refptr<RtpSenderInterface>,
+ rtc::scoped_refptr<RTCStatsCollectorCallback>);
+ PROXY_METHOD2(void,
+ GetStats,
+ rtc::scoped_refptr<RtpReceiverInterface>,
+ rtc::scoped_refptr<RTCStatsCollectorCallback>);
PROXY_METHOD2(rtc::scoped_refptr<DataChannelInterface>,
CreateDataChannel,
const std::string&,
diff --git a/api/rtcerror.cc b/api/rtcerror.cc
index f9a31d0..55ac15e 100644
--- a/api/rtcerror.cc
+++ b/api/rtcerror.cc
@@ -93,9 +93,10 @@
}
}
-std::ostream& operator<<(std::ostream& stream, RTCErrorType error) {
+// TODO(jonasolsson): Change to use absl::string_view when it's available.
+std::string ToString(RTCErrorType error) {
int index = static_cast<int>(error);
- return stream << kRTCErrorTypeNames[index];
+ return std::string(kRTCErrorTypeNames[index]);
}
} // namespace webrtc
diff --git a/api/rtcerror.h b/api/rtcerror.h
index 962f46d..c87ce91 100644
--- a/api/rtcerror.h
+++ b/api/rtcerror.h
@@ -11,7 +11,9 @@
#ifndef API_RTCERROR_H_
#define API_RTCERROR_H_
+#ifdef UNIT_TEST
#include <ostream>
+#endif // UNIT_TEST
#include <string>
#include <utility> // For std::move.
@@ -143,16 +145,24 @@
// error type.
//
// Only intended to be used for logging/disagnostics.
-std::ostream& operator<<(std::ostream& stream, RTCErrorType error);
+std::string ToString(RTCErrorType error);
+
+#ifdef UNIT_TEST
+inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982)
+ std::ostream& stream, // no-presubmit-check TODO(webrtc:8982)
+ RTCErrorType error) {
+ return stream << ToString(error);
+}
+#endif // UNIT_TEST
// Helper macro that can be used by implementations to create an error with a
// message and log it. |message| should be a string literal or movable
// std::string.
-#define LOG_AND_RETURN_ERROR_EX(type, message, severity) \
- { \
- RTC_DCHECK(type != RTCErrorType::NONE); \
- RTC_LOG(severity) << message << " (" << type << ")"; \
- return webrtc::RTCError(type, message); \
+#define LOG_AND_RETURN_ERROR_EX(type, message, severity) \
+ { \
+ RTC_DCHECK(type != RTCErrorType::NONE); \
+ RTC_LOG(severity) << message << " (" << ToString(type) << ")"; \
+ return webrtc::RTCError(type, message); \
}
#define LOG_AND_RETURN_ERROR(type, message) \
diff --git a/api/rtcerror_unittest.cc b/api/rtcerror_unittest.cc
index d8f7ca6..90593cf 100644
--- a/api/rtcerror_unittest.cc
+++ b/api/rtcerror_unittest.cc
@@ -58,15 +58,6 @@
namespace webrtc {
-// Simple test for ostream operator for RTCErrorType.
-TEST(RTCErrorTypeTest, OstreamOperator) {
- std::ostringstream oss;
- oss << webrtc::RTCErrorType::NONE << ' '
- << webrtc::RTCErrorType::INVALID_PARAMETER << ' '
- << webrtc::RTCErrorType::INTERNAL_ERROR;
- EXPECT_EQ("NONE INVALID_PARAMETER INTERNAL_ERROR", oss.str());
-}
-
// Test that the default constructor creates a "no error" error.
TEST(RTCErrorTest, DefaultConstructor) {
RTCError e;
diff --git a/api/rtpparameters.cc b/api/rtpparameters.cc
index 79fd3a9..cb9c1cf 100644
--- a/api/rtpparameters.cc
+++ b/api/rtpparameters.cc
@@ -114,6 +114,9 @@
"http://www.webrtc.org/experiments/rtp-hdrext/video-timing";
const int RtpExtension::kVideoTimingDefaultId = 8;
+const char RtpExtension::kMidUri[] = "urn:ietf:params:rtp-hdrext:sdes:mid";
+const int RtpExtension::kMidDefaultId = 9;
+
const char RtpExtension::kEncryptHeaderExtensionsUri[] =
"urn:ietf:params:rtp-hdrext:encrypt";
@@ -122,7 +125,8 @@
bool RtpExtension::IsSupportedForAudio(const std::string& uri) {
return uri == webrtc::RtpExtension::kAudioLevelUri ||
- uri == webrtc::RtpExtension::kTransportSequenceNumberUri;
+ uri == webrtc::RtpExtension::kTransportSequenceNumberUri ||
+ uri == webrtc::RtpExtension::kMidUri;
}
bool RtpExtension::IsSupportedForVideo(const std::string& uri) {
@@ -132,7 +136,8 @@
uri == webrtc::RtpExtension::kTransportSequenceNumberUri ||
uri == webrtc::RtpExtension::kPlayoutDelayUri ||
uri == webrtc::RtpExtension::kVideoContentTypeUri ||
- uri == webrtc::RtpExtension::kVideoTimingUri;
+ uri == webrtc::RtpExtension::kVideoTimingUri ||
+ uri == webrtc::RtpExtension::kMidUri;
}
bool RtpExtension::IsEncryptionSupported(const std::string& uri) {
@@ -149,7 +154,8 @@
uri == webrtc::RtpExtension::kVideoRotationUri ||
uri == webrtc::RtpExtension::kTransportSequenceNumberUri ||
uri == webrtc::RtpExtension::kPlayoutDelayUri ||
- uri == webrtc::RtpExtension::kVideoContentTypeUri;
+ uri == webrtc::RtpExtension::kVideoContentTypeUri ||
+ uri == webrtc::RtpExtension::kMidUri;
}
const RtpExtension* RtpExtension::FindHeaderExtensionByUri(
diff --git a/api/rtpparameters.h b/api/rtpparameters.h
index d9ac1b6..12e0419 100644
--- a/api/rtpparameters.h
+++ b/api/rtpparameters.h
@@ -276,6 +276,11 @@
static const char kPlayoutDelayUri[];
static const int kPlayoutDelayDefaultId;
+ // Header extension for identifying media section within a transport.
+ // https://tools.ietf.org/html/draft-ietf-mmusic-sdp-bundle-negotiation-49#section-15
+ static const char kMidUri[];
+ static const int kMidDefaultId;
+
// Encryption of Header Extensions, see RFC 6904 for details:
// https://tools.ietf.org/html/rfc6904
static const char kEncryptHeaderExtensionsUri[];
diff --git a/api/rtpsenderinterface.h b/api/rtpsenderinterface.h
index 2ca2edc..01279a5 100644
--- a/api/rtpsenderinterface.h
+++ b/api/rtpsenderinterface.h
@@ -48,8 +48,9 @@
// to uniquely identify a receiver until we implement Unified Plan SDP.
virtual std::string id() const = 0;
- // Returns a list of streams associated with this sender's track. Although we
- // only support one track per stream, in theory the API allows for multiple.
+ // Returns a list of media stream ids associated with this sender's track.
+ // These are signalled in the SDP so that the remote side can associate
+ // tracks.
virtual std::vector<std::string> stream_ids() const = 0;
virtual RtpParameters GetParameters() const = 0;
diff --git a/api/rtptransceiverinterface.h b/api/rtptransceiverinterface.h
index 3ea75fd..7d2a1df 100644
--- a/api/rtptransceiverinterface.h
+++ b/api/rtptransceiverinterface.h
@@ -14,6 +14,7 @@
#include <string>
#include <vector>
+#include "api/array_view.h"
#include "api/optional.h"
#include "api/rtpreceiverinterface.h"
#include "api/rtpsenderinterface.h"
@@ -29,9 +30,6 @@
kInactive
};
-// This is provided as a debugging aid. The format of the output is unspecified.
-std::ostream& operator<<(std::ostream& os, RtpTransceiverDirection direction);
-
// Structure for initializing an RtpTransceiver in a call to
// PeerConnectionInterface::AddTransceiver.
// https://w3c.github.io/webrtc-pc/#dom-rtcrtptransceiverinit
@@ -40,9 +38,6 @@
RtpTransceiverDirection direction = RtpTransceiverDirection::kSendRecv;
// The added RtpTransceiver will be added to these streams.
- // TODO(shampson): Change name to stream_id & update native wrapper's naming
- // as well.
- // TODO(bugs.webrtc.org/7600): Not implemented.
std::vector<std::string> stream_ids;
// TODO(bugs.webrtc.org/7600): Not implemented.
diff --git a/api/umametrics.h b/api/umametrics.h
index 4de1ce4..f885416 100644
--- a/api/umametrics.h
+++ b/api/umametrics.h
@@ -42,6 +42,9 @@
kEnumCounterSdpSemanticNegotiated,
kEnumCounterKeyProtocolMediaType,
kEnumCounterSdpFormatReceived,
+ // The next 2 counters log the value of srtp_err_status_t defined in libsrtp.
+ kEnumCounterSrtpUnprotectError,
+ kEnumCounterSrtcpUnprotectError,
kPeerConnectionEnumCounterMax
};
diff --git a/api/video/encoded_frame.cc b/api/video/encoded_frame.cc
new file mode 100644
index 0000000..37da35f
--- /dev/null
+++ b/api/video/encoded_frame.cc
@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/encoded_frame.h"
+
+namespace webrtc {
+namespace video_coding {
+
+bool EncodedFrame::delayed_by_retransmission() const { return 0; }
+
+} // namespace video_coding
+} // namespace webrtc
diff --git a/api/video/encoded_frame.h b/api/video/encoded_frame.h
index 1374ea0..1e919d0 100644
--- a/api/video/encoded_frame.h
+++ b/api/video/encoded_frame.h
@@ -16,7 +16,39 @@
namespace webrtc {
namespace video_coding {
+// NOTE: This class is still under development and may change without notice.
+struct VideoLayerFrameId {
+ // TODO(philipel): The default ctor is currently used internaly, but have a
+ // look if we can remove it.
+ VideoLayerFrameId() : picture_id(-1), spatial_layer(0) {}
+ VideoLayerFrameId(int64_t picture_id, uint8_t spatial_layer)
+ : picture_id(picture_id), spatial_layer(spatial_layer) {}
+
+ bool operator==(const VideoLayerFrameId& rhs) const {
+ return picture_id == rhs.picture_id && spatial_layer == rhs.spatial_layer;
+ }
+
+ bool operator!=(const VideoLayerFrameId& rhs) const {
+ return !(*this == rhs);
+ }
+
+ bool operator<(const VideoLayerFrameId& rhs) const {
+ if (picture_id == rhs.picture_id)
+ return spatial_layer < rhs.spatial_layer;
+ return picture_id < rhs.picture_id;
+ }
+
+ bool operator<=(const VideoLayerFrameId& rhs) const { return !(rhs < *this); }
+ bool operator>(const VideoLayerFrameId& rhs) const { return rhs < *this; }
+ bool operator>=(const VideoLayerFrameId& rhs) const { return rhs <= *this; }
+
+ int64_t picture_id;
+ uint8_t spatial_layer;
+};
+
// TODO(philipel): Remove webrtc::VCMEncodedFrame inheritance.
+// TODO(philipel): Move transport specific info out of EncodedFrame.
+// NOTE: This class is still under development and may change without notice.
class EncodedFrame : public webrtc::VCMEncodedFrame {
public:
static const uint8_t kMaxFrameReferences = 5;
@@ -38,17 +70,13 @@
// This information is currently needed by the timing calculation class.
// TODO(philipel): Remove this function when a new timing class has
// been implemented.
- virtual bool delayed_by_retransmission() const { return 0; }
+ virtual bool delayed_by_retransmission() const;
size_t size() const { return _length; }
bool is_keyframe() const { return num_references == 0; }
- // The tuple (|picture_id|, |spatial_layer|) uniquely identifies a frame
- // object. For codec types that don't necessarily have picture ids they
- // have to be constructed from the header data relevant to that codec.
- int64_t picture_id = 0;
- uint8_t spatial_layer = 0;
+ VideoLayerFrameId id;
uint32_t timestamp = 0;
// TODO(philipel): Add simple modify/access functions to prevent adding too
diff --git a/api/video/i420_buffer.h b/api/video/i420_buffer.h
index bdac80b..2bd37bd 100644
--- a/api/video/i420_buffer.h
+++ b/api/video/i420_buffer.h
@@ -13,9 +13,9 @@
#include <memory>
-#include "api/video/video_rotation.h"
#include "api/video/video_frame_buffer.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "api/video/video_rotation.h"
+#include "rtc_base/memory/aligned_malloc.h"
namespace webrtc {
diff --git a/api/video/video_bitrate_allocation.cc b/api/video/video_bitrate_allocation.cc
new file mode 100644
index 0000000..059eb8f
--- /dev/null
+++ b/api/video/video_bitrate_allocation.cc
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/video_bitrate_allocation.h"
+
+#include <limits>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/numerics/safe_conversions.h"
+#include "rtc_base/strings/string_builder.h"
+#include "rtc_base/stringutils.h"
+
+namespace webrtc {
+
+VideoBitrateAllocation::VideoBitrateAllocation() : sum_(0) {}
+
+bool VideoBitrateAllocation::SetBitrate(size_t spatial_index,
+ size_t temporal_index,
+ uint32_t bitrate_bps) {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ RTC_CHECK_LT(temporal_index, kMaxTemporalStreams);
+ int64_t new_bitrate_sum_bps = sum_;
+ rtc::Optional<uint32_t>& layer_bitrate =
+ bitrates_[spatial_index][temporal_index];
+ if (layer_bitrate) {
+ RTC_DCHECK_LE(*layer_bitrate, sum_);
+ new_bitrate_sum_bps -= *layer_bitrate;
+ }
+ new_bitrate_sum_bps += bitrate_bps;
+ if (new_bitrate_sum_bps > kMaxBitrateBps)
+ return false;
+
+ layer_bitrate = bitrate_bps;
+ sum_ = rtc::dchecked_cast<uint32_t>(new_bitrate_sum_bps);
+ return true;
+}
+
+bool VideoBitrateAllocation::HasBitrate(size_t spatial_index,
+ size_t temporal_index) const {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ RTC_CHECK_LT(temporal_index, kMaxTemporalStreams);
+ return bitrates_[spatial_index][temporal_index].has_value();
+}
+
+uint32_t VideoBitrateAllocation::GetBitrate(size_t spatial_index,
+ size_t temporal_index) const {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ RTC_CHECK_LT(temporal_index, kMaxTemporalStreams);
+ return bitrates_[spatial_index][temporal_index].value_or(0);
+}
+
+// Whether the specific spatial layers has the bitrate set in any of its
+// temporal layers.
+bool VideoBitrateAllocation::IsSpatialLayerUsed(size_t spatial_index) const {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ for (size_t i = 0; i < kMaxTemporalStreams; ++i) {
+ if (bitrates_[spatial_index][i].has_value())
+ return true;
+ }
+ return false;
+}
+
+// Get the sum of all the temporal layer for a specific spatial layer.
+uint32_t VideoBitrateAllocation::GetSpatialLayerSum(
+ size_t spatial_index) const {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ return GetTemporalLayerSum(spatial_index, kMaxTemporalStreams - 1);
+}
+
+uint32_t VideoBitrateAllocation::GetTemporalLayerSum(
+ size_t spatial_index,
+ size_t temporal_index) const {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ RTC_CHECK_LT(temporal_index, kMaxTemporalStreams);
+ uint32_t sum = 0;
+ for (size_t i = 0; i <= temporal_index; ++i) {
+ sum += bitrates_[spatial_index][i].value_or(0);
+ }
+ return sum;
+}
+
+std::vector<uint32_t> VideoBitrateAllocation::GetTemporalLayerAllocation(
+ size_t spatial_index) const {
+ RTC_CHECK_LT(spatial_index, kMaxSpatialLayers);
+ std::vector<uint32_t> temporal_rates;
+
+ // Find the highest temporal layer with a defined bitrate in order to
+ // determine the size of the temporal layer allocation.
+ for (size_t i = kMaxTemporalStreams; i > 0; --i) {
+ if (bitrates_[spatial_index][i - 1].has_value()) {
+ temporal_rates.resize(i);
+ break;
+ }
+ }
+
+ for (size_t i = 0; i < temporal_rates.size(); ++i) {
+ temporal_rates[i] = bitrates_[spatial_index][i].value_or(0);
+ }
+
+ return temporal_rates;
+}
+
+bool VideoBitrateAllocation::operator==(
+ const VideoBitrateAllocation& other) const {
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (bitrates_[si][ti] != other.bitrates_[si][ti])
+ return false;
+ }
+ }
+ return true;
+}
+
+std::string VideoBitrateAllocation::ToString() const {
+ if (sum_ == 0)
+ return "VideoBitrateAllocation [ [] ]";
+
+ // Max string length in practice is 260, but let's have some overhead and
+ // round up to nearest power of two.
+ char string_buf[512];
+ rtc::SimpleStringBuilder ssb(string_buf);
+
+ ssb << "VideoBitrateAllocation [";
+ uint32_t spatial_cumulator = 0;
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ RTC_DCHECK_LE(spatial_cumulator, sum_);
+ if (spatial_cumulator == sum_)
+ break;
+
+ const uint32_t layer_sum = GetSpatialLayerSum(si);
+ if (layer_sum == sum_) {
+ ssb << " [";
+ } else {
+ if (si > 0)
+ ssb << ",";
+ ssb << '\n' << " [";
+ }
+ spatial_cumulator += layer_sum;
+
+ uint32_t temporal_cumulator = 0;
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ RTC_DCHECK_LE(temporal_cumulator, layer_sum);
+ if (temporal_cumulator == layer_sum)
+ break;
+
+ if (ti > 0)
+ ssb << ", ";
+
+ uint32_t bitrate = bitrates_[si][ti].value_or(0);
+ ssb << bitrate;
+ temporal_cumulator += bitrate;
+ }
+ ssb << "]";
+ }
+
+ RTC_DCHECK_EQ(spatial_cumulator, sum_);
+ ssb << " ]";
+ return ssb.str();
+}
+
+} // namespace webrtc
diff --git a/api/video/video_bitrate_allocation.h b/api/video/video_bitrate_allocation.h
new file mode 100644
index 0000000..b748b67
--- /dev/null
+++ b/api/video/video_bitrate_allocation.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_BITRATE_ALLOCATION_H_
+#define API_VIDEO_VIDEO_BITRATE_ALLOCATION_H_
+
+#include <limits>
+#include <string>
+#include <vector>
+
+#include "api/optional.h"
+#include "typedefs.h" // NOLINT(build/include)
+
+namespace webrtc {
+
+// TODO(sprang): Move back to common_types when include of this is removed.
+enum : int { kMaxSimulcastStreams = 4 };
+enum : int { kMaxSpatialLayers = 5 };
+enum : int { kMaxTemporalStreams = 4 };
+
+// Class that describes how video bitrate, in bps, is allocated across temporal
+// and spatial layers. Not that bitrates are NOT cumulative. Depending on if
+// layers are dependent or not, it is up to the user to aggregate.
+// For each index, the bitrate can also both set and unset. This is used with a
+// set bps = 0 to signal an explicit "turn off" signal.
+class VideoBitrateAllocation {
+ public:
+ static constexpr uint32_t kMaxBitrateBps =
+ std::numeric_limits<uint32_t>::max();
+ VideoBitrateAllocation();
+
+ bool SetBitrate(size_t spatial_index,
+ size_t temporal_index,
+ uint32_t bitrate_bps);
+
+ bool HasBitrate(size_t spatial_index, size_t temporal_index) const;
+
+ uint32_t GetBitrate(size_t spatial_index, size_t temporal_index) const;
+
+ // Whether the specific spatial layers has the bitrate set in any of its
+ // temporal layers.
+ bool IsSpatialLayerUsed(size_t spatial_index) const;
+
+ // Get the sum of all the temporal layer for a specific spatial layer.
+ uint32_t GetSpatialLayerSum(size_t spatial_index) const;
+
+ // Sum of bitrates of temporal layers, from layer 0 to |temporal_index|
+ // inclusive, of specified spatial layer |spatial_index|. Bitrates of lower
+ // spatial layers are not included.
+ uint32_t GetTemporalLayerSum(size_t spatial_index,
+ size_t temporal_index) const;
+
+ // Returns a vector of the temporal layer bitrates for the specific spatial
+ // layer. Length of the returned vector is cropped to the highest temporal
+ // layer with a defined bitrate.
+ std::vector<uint32_t> GetTemporalLayerAllocation(size_t spatial_index) const;
+
+ uint32_t get_sum_bps() const { return sum_; } // Sum of all bitrates.
+ uint32_t get_sum_kbps() const {
+ // Round down to not exceed the allocated bitrate.
+ return sum_ / 1000;
+ }
+
+ bool operator==(const VideoBitrateAllocation& other) const;
+ inline bool operator!=(const VideoBitrateAllocation& other) const {
+ return !(*this == other);
+ }
+
+ std::string ToString() const;
+
+ private:
+ uint32_t sum_;
+ rtc::Optional<uint32_t> bitrates_[kMaxSpatialLayers][kMaxTemporalStreams];
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_VIDEO_BITRATE_ALLOCATION_H_
diff --git a/api/video/video_stream_decoder.h b/api/video/video_stream_decoder.h
new file mode 100644
index 0000000..1c4c5ff
--- /dev/null
+++ b/api/video/video_stream_decoder.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_STREAM_DECODER_H_
+#define API_VIDEO_VIDEO_STREAM_DECODER_H_
+
+#include <map>
+#include <memory>
+#include <utility>
+
+#include "api/video/encoded_frame.h"
+#include "api/video/video_frame.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_decoder_factory.h"
+
+namespace webrtc {
+// NOTE: This class is still under development and may change without notice.
+class VideoStreamDecoder {
+ public:
+ class Callbacks {
+ public:
+ virtual ~Callbacks() = default;
+
+ // Called when the VideoStreamDecoder enters a non-decodable state.
+ virtual void OnNonDecodableState() = 0;
+
+ // Called with the last continuous frame.
+ virtual void OnContinuousUntil(
+ const video_coding::VideoLayerFrameId& key) = 0;
+
+ // Called with the decoded frame.
+ virtual void OnDecodedFrame(VideoFrame decodedImage,
+ rtc::Optional<int> decode_time_ms,
+ rtc::Optional<int> qp) = 0;
+ };
+
+ virtual ~VideoStreamDecoder() = default;
+
+ virtual void OnFrame(std::unique_ptr<video_coding::EncodedFrame> frame) = 0;
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_VIDEO_STREAM_DECODER_H_
diff --git a/api/video/video_stream_decoder_create.cc b/api/video/video_stream_decoder_create.cc
new file mode 100644
index 0000000..e756096
--- /dev/null
+++ b/api/video/video_stream_decoder_create.cc
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/video_stream_decoder_create.h"
+
+#include "rtc_base/ptr_util.h"
+#include "video/video_stream_decoder_impl.h"
+
+namespace webrtc {
+std::unique_ptr<VideoStreamDecoder> CreateVideoStreamDecoder(
+ VideoStreamDecoder::Callbacks* callbacks,
+ VideoDecoderFactory* decoder_factory,
+ std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings) {
+ return rtc::MakeUnique<VideoStreamDecoderImpl>(callbacks, decoder_factory,
+ std::move(decoder_settings));
+}
+} // namespace webrtc
diff --git a/api/video/video_stream_decoder_create.h b/api/video/video_stream_decoder_create.h
new file mode 100644
index 0000000..0468290
--- /dev/null
+++ b/api/video/video_stream_decoder_create.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_STREAM_DECODER_CREATE_H_
+#define API_VIDEO_VIDEO_STREAM_DECODER_CREATE_H_
+
+#include <map>
+#include <memory>
+#include <utility>
+
+#include "api/video/video_stream_decoder.h"
+
+namespace webrtc {
+// The |decoder_settings| parameter is a map between:
+// <payload type> --> <<video format>, <number of cores>>.
+// The video format is used when instantiating a decoder, and
+// the number of cores is used when initializing the decoder.
+std::unique_ptr<VideoStreamDecoder> CreateVideoStreamDecoder(
+ VideoStreamDecoder::Callbacks* callbacks,
+ VideoDecoderFactory* decoder_factory,
+ std::map<int, std::pair<SdpVideoFormat, int>> decoder_settings);
+
+} // namespace webrtc
+
+#endif // API_VIDEO_VIDEO_STREAM_DECODER_CREATE_H_
diff --git a/api/videosourceinterface.cc b/api/videosourceinterface.cc
index 5eda369..8cf8202 100644
--- a/api/videosourceinterface.cc
+++ b/api/videosourceinterface.cc
@@ -13,6 +13,7 @@
namespace rtc {
VideoSinkWants::VideoSinkWants() = default;
+VideoSinkWants::VideoSinkWants(const VideoSinkWants&) = default;
VideoSinkWants::~VideoSinkWants() = default;
} // namespace rtc
diff --git a/api/videosourceinterface.h b/api/videosourceinterface.h
index ffb017a..065e2dc 100644
--- a/api/videosourceinterface.h
+++ b/api/videosourceinterface.h
@@ -22,6 +22,7 @@
// should have when it is delivered to a certain sink.
struct VideoSinkWants {
VideoSinkWants();
+ VideoSinkWants(const VideoSinkWants&);
~VideoSinkWants();
// Tells the source whether the sink wants frames with rotation applied.
// By default, any rotation must be applied by the sink.
diff --git a/audio/BUILD.gn b/audio/BUILD.gn
index b086971..8981dd3 100644
--- a/audio/BUILD.gn
+++ b/audio/BUILD.gn
@@ -52,6 +52,7 @@
"../api:optional",
"../api:transport_api",
"../api/audio:aec3_factory",
+ "../api/audio:audio_frame_api",
"../api/audio:audio_mixer_api",
"../api/audio_codecs:audio_codecs_api",
"../api/audio_codecs:builtin_audio_encoder_factory",
@@ -62,7 +63,6 @@
"../common_audio:common_audio_c",
"../logging:rtc_event_audio",
"../logging:rtc_event_log_api",
- "../modules:module_api",
"../modules/audio_coding",
"../modules/audio_coding:audio_format_conversion",
"../modules/audio_coding:audio_network_adaptor_config",
@@ -75,6 +75,7 @@
"../modules/rtp_rtcp",
"../modules/rtp_rtcp:rtp_rtcp_format",
"../modules/utility",
+ "../rtc_base:audio_format_to_string",
"../rtc_base:checks",
"../rtc_base:rate_limiter",
"../rtc_base:rtc_base",
@@ -126,13 +127,13 @@
":audio",
":audio_end_to_end_test",
"../api:mock_audio_mixer",
+ "../api/audio:audio_frame_api",
"../call:mock_call_interfaces",
"../call:mock_rtp_interfaces",
"../call:rtp_interfaces",
"../call:rtp_receiver",
"../common_audio",
"../logging:mocks",
- "../modules:module_api",
"../modules/audio_device:mock_audio_device",
"../modules/audio_mixer:audio_mixer_impl",
"../modules/audio_processing:audio_processing_statistics",
@@ -189,11 +190,34 @@
}
data = [
- "../resources/voice_engine/audio_dtx16.wav",
"../resources/voice_engine/audio_tiny16.wav",
"../resources/voice_engine/audio_tiny48.wav",
- "test/low_bandwidth_audio_test.py",
]
+
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163)
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+ }
+
+ group("low_bandwidth_audio_perf_test") {
+ testonly = true
+
+ deps = [
+ ":low_bandwidth_audio_test",
+ ]
+
+ data = [
+ "test/low_bandwidth_audio_test.py",
+ "../resources/voice_engine/audio_tiny16.wav",
+ "../resources/voice_engine/audio_tiny48.wav",
+ ]
+ if (is_win) {
+ data += [ "${root_out_dir}/low_bandwidth_audio_test.exe" ]
+ } else {
+ data += [ "${root_out_dir}/low_bandwidth_audio_test" ]
+ }
+
if (is_linux || is_android) {
data += [
"../tools_webrtc/audio_quality/linux/PolqaOem64",
@@ -212,10 +236,7 @@
data += [ "../tools_webrtc/audio_quality/mac/pesq" ]
}
- if (!build_with_chromium && is_clang) {
- # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163)
- suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
- }
+ write_runtime_deps = "${root_out_dir}/${target_name}.runtime_deps"
}
}
diff --git a/audio/audio_level.cc b/audio/audio_level.cc
index ca52522..f1c5b68 100644
--- a/audio/audio_level.cc
+++ b/audio/audio_level.cc
@@ -10,8 +10,8 @@
#include "audio/audio_level.h"
+#include "api/audio/audio_frame.h"
#include "common_audio/signal_processing/include/signal_processing_library.h"
-#include "modules/include/module_common_types.h"
namespace webrtc {
namespace voe {
diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc
index 8a1fbed..352261e 100644
--- a/audio/audio_receive_stream.cc
+++ b/audio/audio_receive_stream.cc
@@ -70,13 +70,12 @@
RTC_DCHECK(audio_state);
internal::AudioState* internal_audio_state =
static_cast<internal::AudioState*>(audio_state);
- return std::unique_ptr<voe::ChannelProxy>(new voe::ChannelProxy(
- std::unique_ptr<voe::Channel>(new voe::Channel(
- module_process_thread,
- internal_audio_state->audio_device_module(),
- config.jitter_buffer_max_packets,
- config.jitter_buffer_fast_accelerate,
- config.decoder_factory))));
+ return std::unique_ptr<voe::ChannelProxy>(
+ new voe::ChannelProxy(std::unique_ptr<voe::Channel>(new voe::Channel(
+ module_process_thread, internal_audio_state->audio_device_module(),
+ nullptr /* RtcpRttStats */, config.jitter_buffer_max_packets,
+ config.jitter_buffer_fast_accelerate, config.decoder_factory,
+ config.codec_pair_id))));
}
} // namespace
diff --git a/audio/audio_receive_stream.h b/audio/audio_receive_stream.h
index a47b59c..09007f0 100644
--- a/audio/audio_receive_stream.h
+++ b/audio/audio_receive_stream.h
@@ -15,6 +15,7 @@
#include <vector>
#include "api/audio/audio_mixer.h"
+#include "api/rtp_headers.h"
#include "audio/audio_state.h"
#include "call/audio_receive_stream.h"
#include "call/rtp_packet_sink_interface.h"
diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc
index 04dffcd..8798d43 100644
--- a/audio/audio_send_stream.cc
+++ b/audio/audio_send_stream.cc
@@ -23,6 +23,7 @@
#include "rtc_base/event.h"
#include "rtc_base/function_view.h"
#include "rtc_base/logging.h"
+#include "rtc_base/strings/audio_format_to_string.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/timeutils.h"
#include "system_wrappers/include/field_trial.h"
@@ -46,15 +47,15 @@
std::unique_ptr<voe::ChannelProxy> CreateChannelAndProxy(
webrtc::AudioState* audio_state,
rtc::TaskQueue* worker_queue,
- ProcessThread* module_process_thread) {
+ ProcessThread* module_process_thread,
+ RtcpRttStats* rtcp_rtt_stats) {
RTC_DCHECK(audio_state);
internal::AudioState* internal_audio_state =
static_cast<internal::AudioState*>(audio_state);
- return std::unique_ptr<voe::ChannelProxy>(new voe::ChannelProxy(
- std::unique_ptr<voe::Channel>(new voe::Channel(
- worker_queue,
- module_process_thread,
- internal_audio_state->audio_device_module()))));
+ return std::unique_ptr<voe::ChannelProxy>(
+ new voe::ChannelProxy(std::unique_ptr<voe::Channel>(new voe::Channel(
+ worker_queue, module_process_thread,
+ internal_audio_state->audio_device_module(), rtcp_rtt_stats))));
}
} // namespace
@@ -103,7 +104,8 @@
overall_call_lifetime,
CreateChannelAndProxy(audio_state.get(),
worker_queue,
- module_process_thread)) {}
+ module_process_thread,
+ rtcp_rtt_stats)) {}
AudioSendStream::AudioSendStream(
const webrtc::AudioSendStream::Config& config,
@@ -138,7 +140,6 @@
RTC_DCHECK(overall_call_lifetime_);
channel_proxy_->SetRtcEventLog(event_log_);
- channel_proxy_->SetRtcpRttStats(rtcp_rtt_stats);
channel_proxy_->SetRTCPStatus(true);
RtpReceiver* rtpReceiver = nullptr; // Unused, but required for call.
channel_proxy_->GetRtpRtcp(&rtp_rtcp_module_, &rtpReceiver);
@@ -159,7 +160,6 @@
channel_proxy_->RegisterTransport(nullptr);
channel_proxy_->ResetSenderCongestionControlObjects();
channel_proxy_->SetRtcEventLog(nullptr);
- channel_proxy_->SetRtcpRttStats(nullptr);
// Lifetime can only be updated after deregistering
// |timed_send_transport_adapter_| in the underlying channel object to avoid
// data races in |active_lifetime_|.
@@ -185,6 +185,8 @@
ids.audio_level = extension.id;
} else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
ids.transport_sequence_number = extension.id;
+ } else if (extension.uri == RtpExtension::kMidUri) {
+ ids.mid = extension.id;
}
}
return ids;
@@ -261,6 +263,13 @@
bandwidth_observer);
}
+ // MID RTP header extension.
+ if ((first_time || new_ids.mid != old_ids.mid ||
+ new_config.rtp.mid != old_config.rtp.mid) &&
+ new_ids.mid != 0 && !new_config.rtp.mid.empty()) {
+ channel_proxy->SetMid(new_config.rtp.mid, new_ids.mid);
+ }
+
if (!ReconfigureSendCodec(stream, new_config)) {
RTC_LOG(LS_ERROR) << "Failed to set up send codec state.";
}
@@ -277,14 +286,16 @@
return;
}
+ bool has_transport_sequence_number =
+ FindExtensionIds(config_.rtp.extensions).transport_sequence_number != 0;
if (config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1 &&
- (FindExtensionIds(config_.rtp.extensions).transport_sequence_number !=
- 0 ||
+ (has_transport_sequence_number ||
!webrtc::field_trial::IsEnabled("WebRTC-Audio-SendSideBwe"))) {
// Audio BWE is enabled.
transport_->packet_sender()->SetAccountForAudioPackets(true);
ConfigureBitrateObserver(config_.min_bitrate_bps, config_.max_bitrate_bps,
- config_.bitrate_priority);
+ config_.bitrate_priority,
+ has_transport_sequence_number);
}
channel_proxy_->StartSend();
sending_ = true;
@@ -495,11 +506,12 @@
RTC_DCHECK(new_config.encoder_factory);
std::unique_ptr<AudioEncoder> encoder =
- new_config.encoder_factory->MakeAudioEncoder(spec.payload_type,
- spec.format, rtc::nullopt);
+ new_config.encoder_factory->MakeAudioEncoder(
+ spec.payload_type, spec.format, new_config.codec_pair_id);
if (!encoder) {
- RTC_DLOG(LS_ERROR) << "Unable to create encoder for " << spec.format;
+ RTC_DLOG(LS_ERROR) << "Unable to create encoder for "
+ << rtc::ToString(spec.format);
return false;
}
// If a bitrate has been specified for the codec, use it over the
@@ -670,12 +682,13 @@
return;
}
+ bool has_transport_sequence_number = new_transport_seq_num_id != 0;
if (new_config.min_bitrate_bps != -1 && new_config.max_bitrate_bps != -1 &&
- (new_transport_seq_num_id != 0 ||
+ (has_transport_sequence_number ||
!webrtc::field_trial::IsEnabled("WebRTC-Audio-SendSideBwe"))) {
- stream->ConfigureBitrateObserver(new_config.min_bitrate_bps,
- new_config.max_bitrate_bps,
- new_config.bitrate_priority);
+ stream->ConfigureBitrateObserver(
+ new_config.min_bitrate_bps, new_config.max_bitrate_bps,
+ new_config.bitrate_priority, has_transport_sequence_number);
} else {
stream->RemoveBitrateObserver();
}
@@ -683,7 +696,8 @@
void AudioSendStream::ConfigureBitrateObserver(int min_bitrate_bps,
int max_bitrate_bps,
- double bitrate_priority) {
+ double bitrate_priority,
+ bool has_packet_feedback) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
RTC_DCHECK_GE(max_bitrate_bps, min_bitrate_bps);
rtc::Event thread_sync_event(false /* manual_reset */, false);
@@ -694,8 +708,11 @@
config_.max_bitrate_bps = max_bitrate_bps;
config_.bitrate_priority = bitrate_priority;
// This either updates the current observer or adds a new observer.
- bitrate_allocator_->AddObserver(this, min_bitrate_bps, max_bitrate_bps, 0,
- true, config_.track_id, bitrate_priority);
+ bitrate_allocator_->AddObserver(
+ this, MediaStreamAllocationConfig{
+ static_cast<uint32_t>(min_bitrate_bps),
+ static_cast<uint32_t>(max_bitrate_bps), 0, true,
+ config_.track_id, bitrate_priority, has_packet_feedback});
thread_sync_event.Set();
});
thread_sync_event.Wait(rtc::Event::kForever);
diff --git a/audio/audio_send_stream.h b/audio/audio_send_stream.h
index 1cda778..c51c7a3 100644
--- a/audio/audio_send_stream.h
+++ b/audio/audio_send_stream.h
@@ -119,7 +119,8 @@
void ConfigureBitrateObserver(int min_bitrate_bps,
int max_bitrate_bps,
- double bitrate_priority);
+ double bitrate_priority,
+ bool has_packet_feedback);
void RemoveBitrateObserver();
void RegisterCngPayloadType(int payload_type, int clockrate_hz);
@@ -157,6 +158,7 @@
struct ExtensionIds {
int audio_level = 0;
int transport_sequence_number = 0;
+ int mid = 0;
};
static ExtensionIds FindExtensionIds(
const std::vector<RtpExtension>& extensions);
diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc
index d8ff0fd..8fb7e7e 100644
--- a/audio/audio_send_stream_unittest.cc
+++ b/audio/audio_send_stream_unittest.cc
@@ -75,10 +75,11 @@
class MockLimitObserver : public BitrateAllocator::LimitObserver {
public:
- MOCK_METHOD3(OnAllocationLimitsChanged,
+ MOCK_METHOD4(OnAllocationLimitsChanged,
void(uint32_t min_send_bitrate_bps,
uint32_t max_padding_bitrate_bps,
- uint32_t total_bitrate_bps));
+ uint32_t total_bitrate_bps,
+ bool has_packet_feedback));
};
std::unique_ptr<MockAudioEncoder> SetupAudioEncoderMock(
@@ -227,9 +228,6 @@
EXPECT_CALL(*channel_proxy_, SetRtcEventLog(testing::NotNull())).Times(1);
EXPECT_CALL(*channel_proxy_, SetRtcEventLog(testing::IsNull()))
.Times(1); // Destructor resets the event log
- EXPECT_CALL(*channel_proxy_, SetRtcpRttStats(&rtcp_rtt_stats_)).Times(1);
- EXPECT_CALL(*channel_proxy_, SetRtcpRttStats(testing::IsNull()))
- .Times(1); // Destructor resets the rtt stats.
}
void SetupMockForSetupSendCodec(bool expect_set_encoder_call) {
diff --git a/audio/channel.cc b/audio/channel.cc
index d50c161..df34ca2 100644
--- a/audio/channel.cc
+++ b/audio/channel.cc
@@ -26,7 +26,6 @@
#include "modules/audio_coding/codecs/audio_format_conversion.h"
#include "modules/audio_device/include/audio_device.h"
#include "modules/audio_processing/include/audio_processing.h"
-#include "modules/include/module_common_types.h"
#include "modules/pacing/packet_router.h"
#include "modules/rtp_rtcp/include/receive_statistics.h"
#include "modules/rtp_rtcp/include/rtp_payload_registry.h"
@@ -94,34 +93,6 @@
RTC_DISALLOW_COPY_AND_ASSIGN(RtcEventLogProxy);
};
-class RtcpRttStatsProxy final : public RtcpRttStats {
- public:
- RtcpRttStatsProxy() : rtcp_rtt_stats_(nullptr) {}
-
- void OnRttUpdate(int64_t rtt) override {
- rtc::CritScope lock(&crit_);
- if (rtcp_rtt_stats_)
- rtcp_rtt_stats_->OnRttUpdate(rtt);
- }
-
- int64_t LastProcessedRtt() const override {
- rtc::CritScope lock(&crit_);
- if (!rtcp_rtt_stats_)
- return 0;
- return rtcp_rtt_stats_->LastProcessedRtt();
- }
-
- void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) {
- rtc::CritScope lock(&crit_);
- rtcp_rtt_stats_ = rtcp_rtt_stats;
- }
-
- private:
- rtc::CriticalSection crit_;
- RtcpRttStats* rtcp_rtt_stats_ RTC_GUARDED_BY(crit_);
- RTC_DISALLOW_COPY_AND_ASSIGN(RtcpRttStatsProxy);
-};
-
class TransportFeedbackProxy : public TransportFeedbackObserver {
public:
TransportFeedbackProxy() : feedback_observer_(nullptr) {
@@ -390,23 +361,6 @@
_rtpRtcpModule->SetRemoteSSRC(ssrc);
}
-void Channel::OnIncomingCSRCChanged(uint32_t CSRC, bool added) {
- // TODO(saza): remove.
-}
-
-int32_t Channel::OnInitializeDecoder(int payload_type,
- const SdpAudioFormat& audio_format,
- uint32_t rate) {
- if (!audio_coding_->RegisterReceiveCodec(payload_type, audio_format)) {
- RTC_DLOG(LS_WARNING) << "Channel::OnInitializeDecoder() invalid codec (pt="
- << payload_type << ", " << audio_format
- << ") received -1";
- return -1;
- }
-
- return 0;
-}
-
int32_t Channel::OnReceivedPayloadData(const uint8_t* payloadData,
size_t payloadSize,
const WebRtcRTPHeader* rtpHeader) {
@@ -552,23 +506,27 @@
Channel::Channel(rtc::TaskQueue* encoder_queue,
ProcessThread* module_process_thread,
- AudioDeviceModule* audio_device_module)
+ AudioDeviceModule* audio_device_module,
+ RtcpRttStats* rtcp_rtt_stats)
: Channel(module_process_thread,
audio_device_module,
+ rtcp_rtt_stats,
0,
false,
- rtc::scoped_refptr<AudioDecoderFactory>()) {
+ rtc::scoped_refptr<AudioDecoderFactory>(),
+ rtc::nullopt) {
RTC_DCHECK(encoder_queue);
encoder_queue_ = encoder_queue;
}
Channel::Channel(ProcessThread* module_process_thread,
AudioDeviceModule* audio_device_module,
+ RtcpRttStats* rtcp_rtt_stats,
size_t jitter_buffer_max_packets,
bool jitter_buffer_fast_playout,
- rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
+ rtc::Optional<AudioCodecPairId> codec_pair_id)
: event_log_proxy_(new RtcEventLogProxy()),
- rtcp_rtt_stats_proxy_(new RtcpRttStatsProxy()),
rtp_payload_registry_(new RTPPayloadRegistry()),
rtp_receive_statistics_(
ReceiveStatistics::Create(Clock::GetRealTimeClock())),
@@ -610,6 +568,7 @@
RTC_DCHECK(audio_device_module);
AudioCodingModule::Config acm_config;
acm_config.decoder_factory = decoder_factory;
+ acm_config.neteq_config.codec_pair_id = codec_pair_id;
acm_config.neteq_config.max_packets_in_buffer = jitter_buffer_max_packets;
acm_config.neteq_config.enable_fast_accelerate = jitter_buffer_fast_playout;
acm_config.neteq_config.enable_muted_state = true;
@@ -630,7 +589,7 @@
configuration.transport_feedback_callback = feedback_observer_proxy_.get();
}
configuration.event_log = &(*event_log_proxy_);
- configuration.rtt_stats = &(*rtcp_rtt_stats_proxy_);
+ configuration.rtt_stats = rtcp_rtt_stats;
configuration.retransmission_rate_limiter =
retransmission_rate_limiter_.get();
@@ -1078,6 +1037,12 @@
return 0;
}
+void Channel::SetMid(const std::string& mid, int extension_id) {
+ int ret = SetSendRtpHeaderExtension(true, kRtpExtensionMid, extension_id);
+ RTC_DCHECK_EQ(0, ret);
+ _rtpRtcpModule->SetMid(mid);
+}
+
int Channel::GetRemoteSSRC(unsigned int& ssrc) {
ssrc = rtp_receiver_->SSRC();
return 0;
@@ -1323,10 +1288,6 @@
event_log_proxy_->SetEventLog(event_log);
}
-void Channel::SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) {
- rtcp_rtt_stats_proxy_->SetRtcpRttStats(rtcp_rtt_stats);
-}
-
void Channel::UpdateOverheadForEncoder() {
size_t overhead_per_packet =
transport_overhead_per_packet_ + rtp_overhead_per_packet_;
diff --git a/audio/channel.h b/audio/channel.h
index e59cae1..ab25aad 100644
--- a/audio/channel.h
+++ b/audio/channel.h
@@ -90,7 +90,6 @@
namespace voe {
class RtcEventLogProxy;
-class RtcpRttStatsProxy;
class RtpPacketSenderProxy;
class TransportFeedbackProxy;
class TransportSequenceNumberProxy;
@@ -151,13 +150,16 @@
// Used for send streams.
Channel(rtc::TaskQueue* encoder_queue,
ProcessThread* module_process_thread,
- AudioDeviceModule* audio_device_module);
+ AudioDeviceModule* audio_device_module,
+ RtcpRttStats* rtcp_rtt_stats);
// Used for receive streams.
Channel(ProcessThread* module_process_thread,
AudioDeviceModule* audio_device_module,
+ RtcpRttStats* rtcp_rtt_stats,
size_t jitter_buffer_max_packets,
bool jitter_buffer_fast_playout,
- rtc::scoped_refptr<AudioDecoderFactory> decoder_factory);
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
+ rtc::Optional<AudioCodecPairId> codec_pair_id);
virtual ~Channel();
void SetSink(AudioSinkInterface* sink);
@@ -217,6 +219,7 @@
// RTP+RTCP
int SetLocalSSRC(unsigned int ssrc);
+ void SetMid(const std::string& mid, int extension_id);
int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id);
void EnableSendTransportSequenceNumber(int id);
@@ -246,11 +249,7 @@
const WebRtcRTPHeader* rtpHeader) override;
// From RtpFeedback in the RTP/RTCP module
- int32_t OnInitializeDecoder(int payload_type,
- const SdpAudioFormat& audio_format,
- uint32_t rate) override;
void OnIncomingSSRCChanged(uint32_t ssrc) override;
- void OnIncomingCSRCChanged(uint32_t CSRC, bool added) override;
// From Transport (called by the RTP/RTCP module)
bool SendRtp(const uint8_t* data,
@@ -287,7 +286,6 @@
// Set a RtcEventLog logging object.
void SetRtcEventLog(RtcEventLog* event_log);
- void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);
void SetTransportOverhead(size_t transport_overhead_per_packet);
// From OverheadObserver in the RTP/RTCP module
@@ -343,7 +341,6 @@
ChannelState channel_state_;
std::unique_ptr<voe::RtcEventLogProxy> event_log_proxy_;
- std::unique_ptr<voe::RtcpRttStatsProxy> rtcp_rtt_stats_proxy_;
std::unique_ptr<RTPPayloadRegistry> rtp_payload_registry_;
std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
diff --git a/audio/channel_proxy.cc b/audio/channel_proxy.cc
index 1a546f6..b8d520d 100644
--- a/audio/channel_proxy.cc
+++ b/audio/channel_proxy.cc
@@ -53,6 +53,11 @@
RTC_DCHECK_EQ(0, error);
}
+void ChannelProxy::SetMid(const std::string& mid, int extension_id) {
+ RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
+ channel_->SetMid(mid, extension_id);
+}
+
void ChannelProxy::SetRTCP_CNAME(const std::string& c_name) {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
// Note: VoERTP_RTCP::SetRTCP_CNAME() accepts a char[256] array.
@@ -283,11 +288,6 @@
}
}
-void ChannelProxy::SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats) {
- RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
- channel_->SetRtcpRttStats(rtcp_rtt_stats);
-}
-
bool ChannelProxy::GetRecCodec(CodecInst* codec_inst) const {
RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
return channel_->GetRecCodec(*codec_inst) == 0;
diff --git a/audio/channel_proxy.h b/audio/channel_proxy.h
index e11bd2d..eadd686 100644
--- a/audio/channel_proxy.h
+++ b/audio/channel_proxy.h
@@ -62,6 +62,7 @@
virtual void SetRTCPStatus(bool enable);
virtual void SetLocalSSRC(uint32_t ssrc);
+ virtual void SetMid(const std::string& mid, int extension_id);
virtual void SetRTCP_CNAME(const std::string& c_name);
virtual void SetNACKStatus(bool enable, int max_packets);
virtual void SetSendAudioLevelIndicationStatus(bool enable, int id);
@@ -110,7 +111,6 @@
RtpReceiver** rtp_receiver) const;
virtual uint32_t GetPlayoutTimestamp() const;
virtual void SetMinimumPlayoutDelay(int delay_ms);
- virtual void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats);
virtual bool GetRecCodec(CodecInst* codec_inst) const;
virtual void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate);
virtual void OnRecoverableUplinkPacketLossRate(
diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h
index 2f6754e..e12b034 100644
--- a/audio/mock_voe_channel_proxy.h
+++ b/audio/mock_voe_channel_proxy.h
@@ -69,7 +69,6 @@
MOCK_METHOD2(ReceivedRTCPPacket, bool(const uint8_t* packet, size_t length));
MOCK_METHOD1(SetChannelOutputVolumeScaling, void(float scaling));
MOCK_METHOD1(SetRtcEventLog, void(RtcEventLog* event_log));
- MOCK_METHOD1(SetRtcpRttStats, void(RtcpRttStats* rtcp_rtt_stats));
MOCK_METHOD2(GetAudioFrameWithInfo,
AudioMixer::Source::AudioFrameInfo(int sample_rate_hz,
AudioFrame* audio_frame));
diff --git a/audio/remix_resample.cc b/audio/remix_resample.cc
index 52a491f..69038cd 100644
--- a/audio/remix_resample.cc
+++ b/audio/remix_resample.cc
@@ -10,11 +10,11 @@
#include "audio/remix_resample.h"
+#include "api/audio/audio_frame.h"
#include "audio/utility/audio_frame_operations.h"
#include "common_audio/resampler/include/push_resampler.h"
#include "common_audio/signal_processing/include/signal_processing_library.h"
#include "common_types.h" // NOLINT(build/include)
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
diff --git a/audio/remix_resample.h b/audio/remix_resample.h
index ddd8086..a45270b 100644
--- a/audio/remix_resample.h
+++ b/audio/remix_resample.h
@@ -11,12 +11,10 @@
#ifndef AUDIO_REMIX_RESAMPLE_H_
#define AUDIO_REMIX_RESAMPLE_H_
+#include "api/audio/audio_frame.h"
#include "common_audio/resampler/include/push_resampler.h"
namespace webrtc {
-
-class AudioFrame;
-
namespace voe {
// Upmix or downmix and resample the audio to |dst_frame|. Expects |dst_frame|
diff --git a/audio/remix_resample_unittest.cc b/audio/remix_resample_unittest.cc
index 753584b..1d8cce7 100644
--- a/audio/remix_resample_unittest.cc
+++ b/audio/remix_resample_unittest.cc
@@ -12,8 +12,8 @@
#include "audio/remix_resample.h"
#include "common_audio/resampler/include/push_resampler.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/arraysize.h"
+#include "rtc_base/checks.h"
#include "rtc_base/format_macros.h"
#include "test/gtest.h"
diff --git a/audio/transport_feedback_packet_loss_tracker.h b/audio/transport_feedback_packet_loss_tracker.h
index 7e73210..4ad4902 100644
--- a/audio/transport_feedback_packet_loss_tracker.h
+++ b/audio/transport_feedback_packet_loss_tracker.h
@@ -15,7 +15,6 @@
#include <vector>
#include "api/optional.h"
-#include "modules/include/module_common_types.h"
namespace webrtc {
diff --git a/audio/utility/BUILD.gn b/audio/utility/BUILD.gn
index aa8445c..fe39b7f 100644
--- a/audio/utility/BUILD.gn
+++ b/audio/utility/BUILD.gn
@@ -23,8 +23,7 @@
deps = [
"../..:webrtc_common",
"../../:typedefs",
- "../../modules:module_api",
- "../../modules/audio_coding:audio_format_conversion",
+ "../../api/audio:audio_frame_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
]
@@ -38,7 +37,6 @@
]
deps = [
":audio_frame_operations",
- "../../modules:module_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../test:test_support",
diff --git a/audio/utility/audio_frame_operations.cc b/audio/utility/audio_frame_operations.cc
index a7c7782..ed7b7a8 100644
--- a/audio/utility/audio_frame_operations.cc
+++ b/audio/utility/audio_frame_operations.cc
@@ -10,9 +10,9 @@
#include "audio/utility/audio_frame_operations.h"
+#include <string.h>
#include <algorithm>
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/numerics/safe_conversions.h"
diff --git a/audio/utility/audio_frame_operations.h b/audio/utility/audio_frame_operations.h
index cd55f19..65a2bad 100644
--- a/audio/utility/audio_frame_operations.h
+++ b/audio/utility/audio_frame_operations.h
@@ -13,12 +13,11 @@
#include <stddef.h>
+#include "api/audio/audio_frame.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
-class AudioFrame;
-
// TODO(andrew): consolidate this with utility.h and audio_frame_manipulator.h.
// Change reference parameters to pointers. Consider using a namespace rather
// than a class.
diff --git a/audio/utility/audio_frame_operations_unittest.cc b/audio/utility/audio_frame_operations_unittest.cc
index 6d23731..1d08d7e 100644
--- a/audio/utility/audio_frame_operations_unittest.cc
+++ b/audio/utility/audio_frame_operations_unittest.cc
@@ -9,7 +9,6 @@
*/
#include "audio/utility/audio_frame_operations.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "test/gtest.h"
diff --git a/common_audio/BUILD.gn b/common_audio/BUILD.gn
index 13b1e55..057b11c 100644
--- a/common_audio/BUILD.gn
+++ b/common_audio/BUILD.gn
@@ -68,6 +68,8 @@
"../rtc_base:checks",
"../rtc_base:gtest_prod",
"../rtc_base:rtc_base_approved",
+ "../rtc_base/memory:aligned_array",
+ "../rtc_base/memory:aligned_malloc",
"../system_wrappers",
"../system_wrappers:cpu_features_api",
]
@@ -131,7 +133,7 @@
} else {
sources += [ "signal_processing/filter_ar_fast_q12.c" ]
}
- deps += [ "../system_wrappers:asm_defines" ]
+ deps += [ "../rtc_base/system:asm_defines" ]
}
}
@@ -259,6 +261,7 @@
"../:typedefs",
"../rtc_base:gtest_prod",
"../rtc_base:rtc_base_approved",
+ "../rtc_base/memory:aligned_malloc",
"../system_wrappers",
]
}
@@ -300,7 +303,7 @@
"resampler/sinc_resampler_sse.cc",
]
- if (is_posix) {
+ if (is_posix || is_fuchsia) {
cflags = [ "-msse2" ]
}
@@ -313,7 +316,7 @@
":sinc_resampler",
"../rtc_base:checks",
"../rtc_base:rtc_base_approved",
- "../system_wrappers",
+ "../rtc_base/memory:aligned_malloc",
]
}
}
@@ -354,7 +357,7 @@
":sinc_resampler",
"../rtc_base:checks",
"../rtc_base:rtc_base_approved",
- "../system_wrappers",
+ "../rtc_base/memory:aligned_malloc",
]
}
diff --git a/common_audio/fir_filter_neon.cc b/common_audio/fir_filter_neon.cc
index d9f91b7..e27f21b 100644
--- a/common_audio/fir_filter_neon.cc
+++ b/common_audio/fir_filter_neon.cc
@@ -14,7 +14,7 @@
#include <string.h>
#include "rtc_base/checks.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
namespace webrtc {
diff --git a/common_audio/fir_filter_neon.h b/common_audio/fir_filter_neon.h
index 5696df8..1ffefd8 100644
--- a/common_audio/fir_filter_neon.h
+++ b/common_audio/fir_filter_neon.h
@@ -14,7 +14,7 @@
#include <memory>
#include "common_audio/fir_filter.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
namespace webrtc {
diff --git a/common_audio/fir_filter_sse.cc b/common_audio/fir_filter_sse.cc
index 3302d56..0da23fc 100644
--- a/common_audio/fir_filter_sse.cc
+++ b/common_audio/fir_filter_sse.cc
@@ -15,7 +15,7 @@
#include <xmmintrin.h>
#include "rtc_base/checks.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
namespace webrtc {
diff --git a/common_audio/fir_filter_sse.h b/common_audio/fir_filter_sse.h
index 6506024..7707f93 100644
--- a/common_audio/fir_filter_sse.h
+++ b/common_audio/fir_filter_sse.h
@@ -14,7 +14,7 @@
#include <memory>
#include "common_audio/fir_filter.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
namespace webrtc {
diff --git a/common_audio/lapped_transform.h b/common_audio/lapped_transform.h
index fe3a8cd..c97cd16 100644
--- a/common_audio/lapped_transform.h
+++ b/common_audio/lapped_transform.h
@@ -16,7 +16,7 @@
#include "common_audio/blocker.h"
#include "common_audio/real_fourier.h"
-#include "system_wrappers/include/aligned_array.h"
+#include "rtc_base/memory/aligned_array.h"
namespace webrtc {
diff --git a/common_audio/real_fourier.cc b/common_audio/real_fourier.cc
index cb0a005..f01c8eb 100644
--- a/common_audio/real_fourier.cc
+++ b/common_audio/real_fourier.cc
@@ -14,10 +14,6 @@
#include "common_audio/signal_processing/include/signal_processing_library.h"
#include "rtc_base/checks.h"
-#ifdef RTC_USE_OPENMAX_DL
-#include "common_audio/real_fourier_openmax.h"
-#endif
-
namespace webrtc {
using std::complex;
@@ -25,11 +21,7 @@
const size_t RealFourier::kFftBufferAlignment = 32;
std::unique_ptr<RealFourier> RealFourier::Create(int fft_order) {
-#if defined(RTC_USE_OPENMAX_DL)
- return std::unique_ptr<RealFourier>(new RealFourierOpenmax(fft_order));
-#else
return std::unique_ptr<RealFourier>(new RealFourierOoura(fft_order));
-#endif
}
int RealFourier::FftOrder(size_t length) {
diff --git a/common_audio/real_fourier.h b/common_audio/real_fourier.h
index 4c69c3c..d16149b 100644
--- a/common_audio/real_fourier.h
+++ b/common_audio/real_fourier.h
@@ -14,7 +14,7 @@
#include <complex>
#include <memory>
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
// Uniform interface class for the real DFT and its inverse, for power-of-2
// input lengths. Also contains helper functions for buffer allocation, taking
diff --git a/common_audio/real_fourier_openmax.cc b/common_audio/real_fourier_openmax.cc
index 6c5c9ce..3c4d0e5 100644
--- a/common_audio/real_fourier_openmax.cc
+++ b/common_audio/real_fourier_openmax.cc
@@ -8,6 +8,9 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+// TODO(http://bugs.webrtc.org/9071): Required by downstream projects.
+#ifdef RTC_USE_OPENMAX_DL
+
#include "common_audio/real_fourier_openmax.h"
#include <cstdlib>
@@ -67,3 +70,4 @@
} // namespace webrtc
+#endif // 0
diff --git a/common_audio/real_fourier_openmax.h b/common_audio/real_fourier_openmax.h
index af91dde..e132280 100644
--- a/common_audio/real_fourier_openmax.h
+++ b/common_audio/real_fourier_openmax.h
@@ -11,9 +11,8 @@
#ifndef COMMON_AUDIO_REAL_FOURIER_OPENMAX_H_
#define COMMON_AUDIO_REAL_FOURIER_OPENMAX_H_
-#ifndef RTC_USE_OPENMAX_DL
-#error "Only include this header if RTC_USE_OPENMAX_DL is defined."
-#endif
+// TODO(http://bugs.webrtc.org/9071): Required by downstream projects.
+#ifdef RTC_USE_OPENMAX_DL
#include <complex>
@@ -44,4 +43,6 @@
} // namespace webrtc
+#endif // RTC_USE_OPENMAX_DL
+
#endif // COMMON_AUDIO_REAL_FOURIER_OPENMAX_H_
diff --git a/common_audio/real_fourier_unittest.cc b/common_audio/real_fourier_unittest.cc
index 5ac39b2..e6ec012 100644
--- a/common_audio/real_fourier_unittest.cc
+++ b/common_audio/real_fourier_unittest.cc
@@ -72,11 +72,7 @@
const RealFourier::fft_cplx_scoper cplx_buffer_;
};
-using FftTypes = ::testing::Types<
-#if defined(RTC_USE_OPENMAX_DL)
- RealFourierOpenmax,
-#endif
- RealFourierOoura>;
+using FftTypes = ::testing::Types<RealFourierOoura>;
TYPED_TEST_CASE(RealFourierTest, FftTypes);
TYPED_TEST(RealFourierTest, SimpleForwardTransform) {
diff --git a/common_audio/resampler/sinc_resampler.h b/common_audio/resampler/sinc_resampler.h
index d011224..774a9b7 100644
--- a/common_audio/resampler/sinc_resampler.h
+++ b/common_audio/resampler/sinc_resampler.h
@@ -18,7 +18,7 @@
#include "rtc_base/constructormagic.h"
#include "rtc_base/gtest_prod_util.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/common_audio/signal_processing/complex_bit_reverse_arm.S b/common_audio/signal_processing/complex_bit_reverse_arm.S
index c70349a..be8e181 100644
--- a/common_audio/signal_processing/complex_bit_reverse_arm.S
+++ b/common_audio/signal_processing/complex_bit_reverse_arm.S
@@ -12,7 +12,7 @@
@ for ARMv5 platforms.
@ Reference C code is in file complex_bit_reverse.c. Bit-exact.
-#include "system_wrappers/include/asm_defines.h"
+#include "rtc_base/system/asm_defines.h"
GLOBAL_FUNCTION WebRtcSpl_ComplexBitReverse
.align 2
diff --git a/common_audio/signal_processing/downsample_fast.c b/common_audio/signal_processing/downsample_fast.c
index 9a2ea05..80fdc58 100644
--- a/common_audio/signal_processing/downsample_fast.c
+++ b/common_audio/signal_processing/downsample_fast.c
@@ -42,8 +42,13 @@
out_s32 = 2048; // Round value, 0.5 in Q12.
for (j = 0; j < coefficients_length; j++) {
- rtc_MsanCheckInitialized(&data_in[i - j], sizeof(data_in[0]), 1);
- out_s32 += coefficients[j] * data_in[i - j]; // Q12.
+ // Negative overflow is permitted here, because this is
+ // auto-regressive filters, and the state for each batch run is
+ // stored in the "negative" positions of the output vector.
+ rtc_MsanCheckInitialized(&data_in[(ptrdiff_t) i - (ptrdiff_t) j],
+ sizeof(data_in[0]), 1);
+ // out_s32 is in Q12 domain.
+ out_s32 += coefficients[j] * data_in[(ptrdiff_t) i - (ptrdiff_t) j];
}
out_s32 >>= 12; // Q0.
diff --git a/common_audio/signal_processing/filter_ar.c b/common_audio/signal_processing/filter_ar.c
index 49d5d61..2471cd1 100644
--- a/common_audio/signal_processing/filter_ar.c
+++ b/common_audio/signal_processing/filter_ar.c
@@ -17,6 +17,8 @@
#include "common_audio/signal_processing/include/signal_processing_library.h"
+#include "rtc_base/checks.h"
+
size_t WebRtcSpl_FilterAR(const int16_t* a,
size_t a_length,
const int16_t* x,
@@ -40,8 +42,10 @@
{
// Calculate filtered[i] and filtered_low[i]
const int16_t* a_ptr = &a[1];
- int16_t* filtered_ptr = &filtered[i - 1];
- int16_t* filtered_low_ptr = &filtered_low[i - 1];
+ // The index can become negative, but the arrays will never be indexed
+ // with it when negative. Nevertheless, the index cannot be a size_t
+ // because of this.
+ int filtered_ix = (int)i - 1;
int16_t* state_ptr = &state[state_length - 1];
int16_t* state_low_ptr = &state_low[state_length - 1];
@@ -51,8 +55,10 @@
stop = (i < a_length) ? i + 1 : a_length;
for (j = 1; j < stop; j++)
{
- o -= *a_ptr * *filtered_ptr--;
- oLOW -= *a_ptr++ * *filtered_low_ptr--;
+ RTC_DCHECK_GE(filtered_ix, 0);
+ o -= *a_ptr * filtered[filtered_ix];
+ oLOW -= *a_ptr++ * filtered_low[filtered_ix];
+ --filtered_ix;
}
for (j = i + 1; j < a_length; j++)
{
diff --git a/common_audio/signal_processing/filter_ar_fast_q12.c b/common_audio/signal_processing/filter_ar_fast_q12.c
index df9e518..8b8bdb1 100644
--- a/common_audio/signal_processing/filter_ar_fast_q12.c
+++ b/common_audio/signal_processing/filter_ar_fast_q12.c
@@ -8,6 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "stddef.h"
+
#include "rtc_base/checks.h"
#include "common_audio/signal_processing/include/signal_processing_library.h"
@@ -29,7 +31,10 @@
int64_t sum = 0;
for (j = coefficients_length - 1; j > 0; j--) {
- sum += coefficients[j] * data_out[i - j];
+ // Negative overflow is permitted here, because this is
+ // auto-regressive filters, and the state for each batch run is
+ // stored in the "negative" positions of the output vector.
+ sum += coefficients[j] * data_out[(ptrdiff_t) i - (ptrdiff_t) j];
}
output = coefficients[0] * data_in[i];
diff --git a/common_audio/signal_processing/filter_ar_fast_q12_armv7.S b/common_audio/signal_processing/filter_ar_fast_q12_armv7.S
index c6397c2..60319d2 100644
--- a/common_audio/signal_processing/filter_ar_fast_q12_armv7.S
+++ b/common_audio/signal_processing/filter_ar_fast_q12_armv7.S
@@ -35,7 +35,7 @@
@ r11: Scratch
@ r12: &coefficients[j]
-#include "system_wrappers/include/asm_defines.h"
+#include "rtc_base/system/asm_defines.h"
GLOBAL_FUNCTION WebRtcSpl_FilterARFastQ12
.align 2
diff --git a/common_audio/signal_processing/filter_ma_fast_q12.c b/common_audio/signal_processing/filter_ma_fast_q12.c
index 9596ef1..329d47e 100644
--- a/common_audio/signal_processing/filter_ma_fast_q12.c
+++ b/common_audio/signal_processing/filter_ma_fast_q12.c
@@ -37,7 +37,10 @@
for (j = 0; j < B_length; j++)
{
- o += B[j] * in_ptr[i - j];
+ // Negative overflow is permitted here, because this is
+ // auto-regressive filters, and the state for each batch run is
+ // stored in the "negative" positions of the output vector.
+ o += B[j] * in_ptr[(ptrdiff_t) i - (ptrdiff_t) j];
}
// If output is higher than 32768, saturate it. Same with negative side
diff --git a/common_audio/signal_processing/include/signal_processing_library.h b/common_audio/signal_processing/include/signal_processing_library.h
index 73cdc0d..27fea6f 100644
--- a/common_audio/signal_processing/include/signal_processing_library.h
+++ b/common_audio/signal_processing/include/signal_processing_library.h
@@ -110,7 +110,7 @@
// C code will be assigned.
// Note that this function MUST be called in any application that uses SPL
// functions.
-void WebRtcSpl_Init();
+void WebRtcSpl_Init(void);
int16_t WebRtcSpl_GetScalingSquare(int16_t* in_vector,
size_t in_vector_length,
diff --git a/common_audio/signal_processing/spl_init.c b/common_audio/signal_processing/spl_init.c
index 0f41bc1..80e9197 100644
--- a/common_audio/signal_processing/spl_init.c
+++ b/common_audio/signal_processing/spl_init.c
@@ -30,7 +30,7 @@
#if (!defined(WEBRTC_HAS_NEON)) && !defined(MIPS32_LE)
/* Initialize function pointers to the generic C version. */
-static void InitPointersToC() {
+static void InitPointersToC(void) {
WebRtcSpl_MaxAbsValueW16 = WebRtcSpl_MaxAbsValueW16C;
WebRtcSpl_MaxAbsValueW32 = WebRtcSpl_MaxAbsValueW32C;
WebRtcSpl_MaxValueW16 = WebRtcSpl_MaxValueW16C;
@@ -46,7 +46,7 @@
#if defined(WEBRTC_HAS_NEON)
/* Initialize function pointers to the Neon version. */
-static void InitPointersToNeon() {
+static void InitPointersToNeon(void) {
WebRtcSpl_MaxAbsValueW16 = WebRtcSpl_MaxAbsValueW16Neon;
WebRtcSpl_MaxAbsValueW32 = WebRtcSpl_MaxAbsValueW32Neon;
WebRtcSpl_MaxValueW16 = WebRtcSpl_MaxValueW16Neon;
@@ -62,7 +62,7 @@
#if defined(MIPS32_LE)
/* Initialize function pointers to the MIPS version. */
-static void InitPointersToMIPS() {
+static void InitPointersToMIPS(void) {
WebRtcSpl_MaxAbsValueW16 = WebRtcSpl_MaxAbsValueW16_mips;
WebRtcSpl_MaxValueW16 = WebRtcSpl_MaxValueW16_mips;
WebRtcSpl_MaxValueW32 = WebRtcSpl_MaxValueW32_mips;
@@ -128,6 +128,6 @@
*/
#endif /* WEBRTC_POSIX */
-void WebRtcSpl_Init() {
+void WebRtcSpl_Init(void) {
once(InitFunctionPointers);
}
diff --git a/common_audio/signal_processing/spl_sqrt_floor_arm.S b/common_audio/signal_processing/spl_sqrt_floor_arm.S
index 29e6d4d..228e68e 100644
--- a/common_audio/signal_processing/spl_sqrt_floor_arm.S
+++ b/common_audio/signal_processing/spl_sqrt_floor_arm.S
@@ -32,7 +32,7 @@
@ Output: r0 = INT (SQRT (r0)), precision is 16 bits
@ Registers touched: r1, r2
-#include "system_wrappers/include/asm_defines.h"
+#include "rtc_base/system/asm_defines.h"
GLOBAL_FUNCTION WebRtcSpl_SqrtFloor
.align 2
diff --git a/common_audio/vad/include/webrtc_vad.h b/common_audio/vad/include/webrtc_vad.h
index 7d71b9b..353dbf0 100644
--- a/common_audio/vad/include/webrtc_vad.h
+++ b/common_audio/vad/include/webrtc_vad.h
@@ -27,7 +27,7 @@
#endif
// Creates an instance to the VAD structure.
-VadInst* WebRtcVad_Create();
+VadInst* WebRtcVad_Create(void);
// Frees the dynamic memory of a specified VAD instance.
//
diff --git a/common_audio/wav_file.cc b/common_audio/wav_file.cc
index 37f249e..1217e40 100644
--- a/common_audio/wav_file.cc
+++ b/common_audio/wav_file.cc
@@ -18,6 +18,7 @@
#include "common_audio/include/audio_util.h"
#include "common_audio/wav_header.h"
#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
namespace webrtc {
@@ -47,8 +48,21 @@
}
WavReader::WavReader(const std::string& filename)
- : file_handle_(fopen(filename.c_str(), "rb")) {
- RTC_CHECK(file_handle_) << "Could not open wav file for reading.";
+ : WavReader(rtc::OpenPlatformFileReadOnly(filename)) {}
+
+WavReader::WavReader(rtc::PlatformFile file) {
+ RTC_CHECK_NE(file, rtc::kInvalidPlatformFileValue)
+ << "Invalid file. Could not create file handle for wav file.";
+ file_handle_ = rtc::FdopenPlatformFile(file, "rb");
+ if (!file_handle_) {
+ RTC_LOG(LS_ERROR) << "Could not open wav file for reading: " << errno;
+ // Even though we failed to open a FILE*, the file is still open
+ // and needs to be closed.
+ if (!rtc::ClosePlatformFile(file)) {
+ RTC_LOG(LS_ERROR) << "Can't close file.";
+ }
+ FATAL() << "Could not open wav file for reading.";
+ }
ReadableWavFile readable(file_handle_);
WavFormat format;
@@ -110,13 +124,31 @@
file_handle_ = nullptr;
}
-WavWriter::WavWriter(const std::string& filename, int sample_rate,
+WavWriter::WavWriter(const std::string& filename,
+ int sample_rate,
size_t num_channels)
- : sample_rate_(sample_rate),
- num_channels_(num_channels),
- num_samples_(0),
- file_handle_(fopen(filename.c_str(), "wb")) {
- RTC_CHECK(file_handle_) << "Could not open wav file for writing.";
+ // Unlike plain fopen, CreatePlatformFile takes care of filename utf8 ->
+ // wchar conversion on windows.
+ : WavWriter(rtc::CreatePlatformFile(filename), sample_rate, num_channels) {}
+
+WavWriter::WavWriter(rtc::PlatformFile file,
+ int sample_rate,
+ size_t num_channels)
+ : sample_rate_(sample_rate), num_channels_(num_channels), num_samples_(0) {
+ // Handle errors from the CreatePlatformFile call in above constructor.
+ RTC_CHECK_NE(file, rtc::kInvalidPlatformFileValue)
+ << "Invalid file. Could not create wav file.";
+ file_handle_ = rtc::FdopenPlatformFile(file, "wb");
+ if (!file_handle_) {
+ RTC_LOG(LS_ERROR) << "Could not open wav file for writing.";
+ // Even though we failed to open a FILE*, the file is still open
+ // and needs to be closed.
+ if (!rtc::ClosePlatformFile(file)) {
+ RTC_LOG(LS_ERROR) << "Can't close file.";
+ }
+ FATAL() << "Could not open wav file for writing.";
+ }
+
RTC_CHECK(CheckWavParameters(num_channels_, sample_rate_, kWavFormat,
kBytesPerSample, num_samples_));
diff --git a/common_audio/wav_file.h b/common_audio/wav_file.h
index f7afe92..befe733 100644
--- a/common_audio/wav_file.h
+++ b/common_audio/wav_file.h
@@ -18,6 +18,7 @@
#include <string>
#include "rtc_base/constructormagic.h"
+#include "rtc_base/platform_file.h"
namespace webrtc {
@@ -41,6 +42,9 @@
// Open a new WAV file for writing.
WavWriter(const std::string& filename, int sample_rate, size_t num_channels);
+ // Open a new WAV file for writing.
+ WavWriter(rtc::PlatformFile file, int sample_rate, size_t num_channels);
+
// Close the WAV file, after writing its header.
~WavWriter() override;
@@ -70,6 +74,9 @@
// Opens an existing WAV file for reading.
explicit WavReader(const std::string& filename);
+ // Opens an existing WAV file for reading.
+ explicit WavReader(rtc::PlatformFile file);
+
// Close the WAV file.
~WavReader() override;
diff --git a/common_audio/wav_file_unittest.cc b/common_audio/wav_file_unittest.cc
index 7113b47..248273b 100644
--- a/common_audio/wav_file_unittest.cc
+++ b/common_audio/wav_file_unittest.cc
@@ -174,4 +174,72 @@
}
}
+// Write a tiny WAV file with the the std::FILE interface and verify the
+// result.
+TEST(WavWriterTest, CPPFileDescriptor) {
+ const std::string outfile = test::OutputPath() + "wavtest1.wav";
+ static constexpr size_t kNumSamples = 3;
+ {
+ WavWriter w(rtc::CreatePlatformFile(outfile), 14099, 1);
+ EXPECT_EQ(14099, w.sample_rate());
+ EXPECT_EQ(1u, w.num_channels());
+ EXPECT_EQ(0u, w.num_samples());
+ w.WriteSamples(kSamples, kNumSamples);
+ EXPECT_EQ(kNumSamples, w.num_samples());
+ }
+ // Write some extra "metadata" to the file that should be silently ignored
+ // by WavReader. We don't use WavWriter directly for this because it doesn't
+ // support metadata.
+ static constexpr uint8_t kMetadata[] = {101, 202};
+ {
+ FILE* f = fopen(outfile.c_str(), "ab");
+ ASSERT_TRUE(f);
+ ASSERT_EQ(1u, fwrite(kMetadata, sizeof(kMetadata), 1, f));
+ fclose(f);
+ }
+ static const uint8_t kExpectedContents[] = {
+ // clang-format off
+ 'R', 'I', 'F', 'F',
+ 42, 0, 0, 0, // size of whole file - 8: 6 + 44 - 8
+ 'W', 'A', 'V', 'E',
+ 'f', 'm', 't', ' ',
+ 16, 0, 0, 0, // size of fmt block - 8: 24 - 8
+ 1, 0, // format: PCM (1)
+ 1, 0, // channels: 1
+ 0x13, 0x37, 0, 0, // sample rate: 14099
+ 0x26, 0x6e, 0, 0, // byte rate: 2 * 14099
+ 2, 0, // block align: NumChannels * BytesPerSample
+ 16, 0, // bits per sample: 2 * 8
+ 'd', 'a', 't', 'a',
+ 6, 0, 0, 0, // size of payload: 6
+ 0, 0, // first sample: 0.0
+ 10, 0, // second sample: 10.0
+ 0xff, 0x7f, // third sample: 4e4 (saturated)
+ kMetadata[0], kMetadata[1],
+ // clang-format on
+ };
+ static constexpr size_t kContentSize =
+ kWavHeaderSize + kNumSamples * sizeof(int16_t) + sizeof(kMetadata);
+ static_assert(sizeof(kExpectedContents) == kContentSize, "");
+ EXPECT_EQ(kContentSize, test::GetFileSize(outfile));
+ FILE* f = fopen(outfile.c_str(), "rb");
+ ASSERT_TRUE(f);
+ uint8_t contents[kContentSize];
+ ASSERT_EQ(1u, fread(contents, kContentSize, 1, f));
+ EXPECT_EQ(0, fclose(f));
+ EXPECT_EQ(0, memcmp(kExpectedContents, contents, kContentSize));
+
+ {
+ WavReader r(rtc::OpenPlatformFileReadOnly(outfile));
+ EXPECT_EQ(14099, r.sample_rate());
+ EXPECT_EQ(1u, r.num_channels());
+ EXPECT_EQ(kNumSamples, r.num_samples());
+ static constexpr float kTruncatedSamples[] = {0.0, 10.0, 32767.0};
+ float samples[kNumSamples];
+ EXPECT_EQ(kNumSamples, r.ReadSamples(kNumSamples, samples));
+ EXPECT_EQ(0, memcmp(kTruncatedSamples, samples, sizeof(samples)));
+ EXPECT_EQ(0u, r.ReadSamples(kNumSamples, samples));
+ }
+}
+
} // namespace webrtc
diff --git a/common_audio/wav_header_unittest.cc b/common_audio/wav_header_unittest.cc
index c6f605f..8b30530 100644
--- a/common_audio/wav_header_unittest.cc
+++ b/common_audio/wav_header_unittest.cc
@@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <string.h>
#include <limits>
#include "common_audio/wav_header.h"
diff --git a/common_types.h b/common_types.h
index ee4ddf7..f717094 100644
--- a/common_types.h
+++ b/common_types.h
@@ -13,12 +13,13 @@
#include <stddef.h>
#include <string.h>
-#include <ostream>
#include <string>
#include <vector>
#include "api/array_view.h"
#include "api/optional.h"
+// TODO(sprang): Remove this include when all usage includes it directly.
+#include "api/video/video_bitrate_allocation.h"
#include "rtc_base/checks.h"
#include "rtc_base/deprecation.h"
#include "typedefs.h" // NOLINT(build/include)
@@ -29,14 +30,6 @@
#pragma warning(disable : 4351)
#endif
-#if defined(WEBRTC_EXPORT)
-#define WEBRTC_DLLEXPORT _declspec(dllexport)
-#elif defined(WEBRTC_DLL)
-#define WEBRTC_DLLEXPORT _declspec(dllimport)
-#else
-#define WEBRTC_DLLEXPORT
-#endif
-
#ifndef NULL
#define NULL 0
#endif
@@ -55,34 +48,6 @@
namespace webrtc {
-class RewindableStream {
- public:
- virtual ~RewindableStream() {}
- virtual int Rewind() = 0;
-};
-
-class InStream : public RewindableStream {
- public:
- // Reads |len| bytes from file to |buf|. Returns the number of bytes read
- // or -1 on error.
- virtual int Read(void* buf, size_t len) = 0;
-};
-
-class OutStream : public RewindableStream {
- public:
- // Writes |len| bytes from |buf| to file. The actual writing may happen
- // some time later. Call Flush() to force a write.
- virtual bool Write(const void* buf, size_t len) = 0;
-};
-
-// For the deprecated MediaFile module.
-enum FileFormats {
- kFileFormatWavFile = 1,
- kFileFormatPcm16kHzFile = 7,
- kFileFormatPcm8kHzFile = 8,
- kFileFormatPcm32kHzFile = 9,
-};
-
enum FrameType {
kEmptyFrame = 0,
kAudioFrameSpeech = 1,
@@ -186,14 +151,6 @@
const RtcpPacketTypeCounter& packet_counter) = 0;
};
-// Rate statistics for a stream.
-struct BitrateStatistics {
- BitrateStatistics() : bitrate_bps(0), packet_rate(0) {}
-
- uint32_t bitrate_bps; // Bitrate in bits per second.
- uint32_t packet_rate; // Packet rate in packets per second.
-};
-
// Callback, used to notify an observer whenever new rates have been estimated.
class BitrateStatisticsObserver {
public:
@@ -268,35 +225,11 @@
}
bool operator!=(const CodecInst& other) const { return !(*this == other); }
-
- friend std::ostream& operator<<(std::ostream& os, const CodecInst& ci) {
- os << "{pltype: " << ci.pltype;
- os << ", plname: " << ci.plname;
- os << ", plfreq: " << ci.plfreq;
- os << ", pacsize: " << ci.pacsize;
- os << ", channels: " << ci.channels;
- os << ", rate: " << ci.rate << "}";
- return os;
- }
};
// RTP
enum { kRtpCsrcSize = 15 }; // RFC 3550 page 13
-enum PayloadFrequencies {
- kFreq8000Hz = 8000,
- kFreq16000Hz = 16000,
- kFreq32000Hz = 32000
-};
-
-// Degree of bandwidth reduction.
-enum VadModes {
- kVadConventional = 0, // lowest reduction
- kVadAggressiveLow,
- kVadAggressiveMid,
- kVadAggressiveHigh // highest reduction
-};
-
// NETEQ statistics.
struct NetworkStatistics {
// current jitter buffer size in ms
@@ -397,10 +330,6 @@
};
// Video codec
-enum { kMaxSimulcastStreams = 4 };
-enum { kMaxSpatialLayers = 5 };
-enum { kMaxTemporalStreams = 4 };
-
enum VideoCodecComplexity {
kComplexityNormal = 0,
kComplexityHigh = 1,
@@ -408,34 +337,36 @@
kComplexityMax = 3
};
-enum VP8ResilienceMode {
- kResilienceOff, // The stream produced by the encoder requires a
- // recovery frame (typically a key frame) to be
- // decodable after a packet loss.
- kResilientStream, // A stream produced by the encoder is resilient to
- // packet losses, but packets within a frame subsequent
- // to a loss can't be decoded.
- kResilientFrames // Same as kResilientStream but with added resilience
- // within a frame.
-};
-
-class TemporalLayersFactory;
// VP8 specific
struct VideoCodecVP8 {
+ bool operator==(const VideoCodecVP8& other) const;
+ bool operator!=(const VideoCodecVP8& other) const {
+ return !(*this == other);
+ }
VideoCodecComplexity complexity;
- VP8ResilienceMode resilience;
unsigned char numberOfTemporalLayers;
bool denoisingOn;
bool automaticResizeOn;
bool frameDroppingOn;
int keyFrameInterval;
- TemporalLayersFactory* tl_factory;
+};
+
+enum class InterLayerPredMode {
+ kOn, // Allow inter-layer prediction for all frames.
+ // Frame of low spatial layer can be used for
+ // prediction of next spatial layer frame.
+ kOff, // Encoder produces independent spatial layers.
+ kOnKeyPic // Allow inter-layer prediction only for frames
+ // within key picture.
};
// VP9 specific.
struct VideoCodecVP9 {
+ bool operator==(const VideoCodecVP9& other) const;
+ bool operator!=(const VideoCodecVP9& other) const {
+ return !(*this == other);
+ }
VideoCodecComplexity complexity;
- bool resilienceOn;
unsigned char numberOfTemporalLayers;
bool denoisingOn;
bool frameDroppingOn;
@@ -444,6 +375,7 @@
bool automaticResizeOn;
unsigned char numberOfSpatialLayers;
bool flexibleMode;
+ InterLayerPredMode interLayerPred;
};
// TODO(magjed): Move this and other H264 related classes out to their own file.
@@ -461,6 +393,10 @@
// H264 specific.
struct VideoCodecH264 {
+ bool operator==(const VideoCodecH264& other) const;
+ bool operator!=(const VideoCodecH264& other) const {
+ return !(*this == other);
+ }
bool frameDroppingOn;
int keyFrameInterval;
// These are NULL/0 if not externally negotiated.
@@ -496,6 +432,9 @@
};
struct SpatialLayer {
+ bool operator==(const SpatialLayer& other) const;
+ bool operator!=(const SpatialLayer& other) const { return !(*this == other); }
+
unsigned short width;
unsigned short height;
unsigned char numberOfTemporalLayers;
@@ -576,45 +515,8 @@
VideoCodecUnion codec_specific_;
};
-class BitrateAllocation {
- public:
- static const uint32_t kMaxBitrateBps;
- BitrateAllocation();
-
- bool SetBitrate(size_t spatial_index,
- size_t temporal_index,
- uint32_t bitrate_bps);
-
- bool HasBitrate(size_t spatial_index, size_t temporal_index) const;
-
- uint32_t GetBitrate(size_t spatial_index, size_t temporal_index) const;
-
- // Whether the specific spatial layers has the bitrate set in any of its
- // temporal layers.
- bool IsSpatialLayerUsed(size_t spatial_index) const;
-
- // Get the sum of all the temporal layer for a specific spatial layer.
- uint32_t GetSpatialLayerSum(size_t spatial_index) const;
-
- uint32_t get_sum_bps() const { return sum_; } // Sum of all bitrates.
- uint32_t get_sum_kbps() const { return (sum_ + 500) / 1000; }
-
- inline bool operator==(const BitrateAllocation& other) const {
- return memcmp(bitrates_, other.bitrates_, sizeof(bitrates_)) == 0;
- }
- inline bool operator!=(const BitrateAllocation& other) const {
- return !(*this == other);
- }
-
- // Expensive, please use only in tests.
- std::string ToString() const;
- std::ostream& operator<<(std::ostream& os) const;
-
- private:
- uint32_t sum_;
- uint32_t bitrates_[kMaxSpatialLayers][kMaxTemporalStreams];
- bool has_bitrate_[kMaxSpatialLayers][kMaxTemporalStreams];
-};
+// TODO(sprang): Remove this when downstream projects have been updated.
+using BitrateAllocation = VideoBitrateAllocation;
// Bandwidth over-use detector options. These are used to drive
// experimentation with bandwidth estimation parameters.
diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn
index c20bd2b..7dfe70b 100644
--- a/modules/audio_coding/BUILD.gn
+++ b/modules/audio_coding/BUILD.gn
@@ -97,7 +97,6 @@
"include/audio_coding_module_typedefs.h",
]
deps = [
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
]
@@ -105,6 +104,7 @@
rtc_static_library("audio_coding") {
visibility += [ "*" ]
+ allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove.
sources = [
"acm2/acm_receiver.cc",
"acm2/acm_receiver.h",
@@ -136,18 +136,19 @@
}
deps = audio_coding_deps + [
+ "../../api/audio:audio_frame_api",
+ "..:module_api",
"../../common_audio:common_audio_c",
"../..:typedefs",
"../../rtc_base:deprecation",
"../../rtc_base:checks",
"../../system_wrappers:metrics_api",
- "..:module_api",
"../../api:array_view",
"../../api/audio_codecs:audio_codecs_api",
- "../../api/audio_codecs:builtin_audio_decoder_factory",
":audio_coding_module_typedefs",
":neteq",
":rent_a_codec",
+ "../../rtc_base:audio_format_to_string",
"../../rtc_base:rtc_base_approved",
"../../api:optional",
"../../logging:rtc_event_log_api",
@@ -220,6 +221,7 @@
rtc_static_library("g711") {
visibility += [ "*" ]
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/g711/audio_decoder_pcm.cc",
"codecs/g711/audio_decoder_pcm.h",
@@ -242,6 +244,7 @@
}
rtc_source_set("g711_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/g711/g711.c",
"codecs/g711/g711.h",
@@ -260,6 +263,7 @@
rtc_static_library("g722") {
visibility += [ "*" ]
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/g722/audio_decoder_g722.cc",
"codecs/g722/audio_decoder_g722.h",
@@ -283,6 +287,7 @@
}
rtc_source_set("g722_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/g722/g722_decode.c",
"codecs/g722/g722_enc_dec.h",
@@ -302,6 +307,7 @@
rtc_static_library("ilbc") {
visibility += webrtc_default_visibility
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/ilbc/audio_decoder_ilbc.cc",
"codecs/ilbc/audio_decoder_ilbc.h",
@@ -326,6 +332,7 @@
}
rtc_source_set("ilbc_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/ilbc/abs_quant.c",
"codecs/ilbc/abs_quant.h",
@@ -483,6 +490,7 @@
}
rtc_static_library("isac_common") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/audio_decoder_isac_t.h",
"codecs/isac/audio_decoder_isac_t_impl.h",
@@ -508,6 +516,7 @@
rtc_static_library("isac") {
visibility += [ "*" ]
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/main/include/audio_decoder_isac.h",
"codecs/isac/main/include/audio_encoder_isac.h",
@@ -525,6 +534,7 @@
}
rtc_static_library("isac_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/main/include/isac.h",
"codecs/isac/main/source/arith_routines.c",
@@ -602,6 +612,7 @@
rtc_static_library("isac_fix") {
visibility += [ "*" ]
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/fix/source/audio_decoder_isacfix.cc",
"codecs/isac/fix/source/audio_encoder_isacfix.cc",
@@ -625,6 +636,7 @@
}
rtc_source_set("isac_fix_common") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/fix/source/codec.h",
"codecs/isac/fix/source/entropy_coding.h",
@@ -633,6 +645,7 @@
"codecs/isac/fix/source/filterbank_internal.h",
"codecs/isac/fix/source/settings.h",
"codecs/isac/fix/source/structs.h",
+ "codecs/isac/fix/source/transform_tables.c",
]
public_configs = [ ":isac_fix_config" ]
deps = [
@@ -644,6 +657,7 @@
}
rtc_source_set("isac_fix_c_arm_asm") {
+ poisonous = [ "audio_codecs" ]
sources = []
if (current_cpu == "arm" && arm_version >= 7) {
sources += [
@@ -652,12 +666,13 @@
]
deps = [
":isac_fix_common",
- "../../system_wrappers:asm_defines",
+ "../../rtc_base/system:asm_defines",
]
}
}
rtc_source_set("isac_fix_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/fix/include/audio_decoder_isacfix.h",
"codecs/isac/fix/include/audio_encoder_isacfix.h",
@@ -698,7 +713,6 @@
"codecs/isac/fix/source/spectrum_ar_model_tables.c",
"codecs/isac/fix/source/spectrum_ar_model_tables.h",
"codecs/isac/fix/source/transform.c",
- "codecs/isac/fix/source/transform_tables.c",
]
public_configs = [ ":isac_fix_config" ]
@@ -760,6 +774,7 @@
if (rtc_build_with_neon) {
rtc_static_library("isac_neon") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/isac/fix/source/entropy_coding_neon.c",
"codecs/isac/fix/source/filterbanks_neon.c",
@@ -801,6 +816,7 @@
rtc_static_library("pcm16b") {
visibility += [ "*" ]
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/pcm16b/audio_decoder_pcm16b.cc",
"codecs/pcm16b/audio_decoder_pcm16b.h",
@@ -825,6 +841,7 @@
}
rtc_source_set("pcm16b_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/pcm16b/pcm16b.c",
"codecs/pcm16b/pcm16b.h",
@@ -839,6 +856,7 @@
rtc_static_library("webrtc_opus") {
visibility += webrtc_default_visibility
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/opus/audio_decoder_opus.cc",
"codecs/opus/audio_decoder_opus.h",
@@ -874,6 +892,7 @@
}
rtc_source_set("webrtc_opus_c") {
+ poisonous = [ "audio_codecs" ]
sources = [
"codecs/opus/opus_inst.h",
"codecs/opus/opus_interface.c",
@@ -971,6 +990,7 @@
"../../rtc_base:checks",
"../../rtc_base:protobuf_utils",
"../../rtc_base:rtc_base_approved",
+ "../../rtc_base/system:file_wrapper",
"../../system_wrappers",
"../../system_wrappers:field_trial_api",
]
@@ -1038,6 +1058,8 @@
"neteq/dtmf_tone_generator.h",
"neteq/expand.cc",
"neteq/expand.h",
+ "neteq/expand_uma_logger.cc",
+ "neteq/expand_uma_logger.h",
"neteq/include/neteq.h",
"neteq/merge.cc",
"neteq/merge.h",
@@ -1083,9 +1105,11 @@
"../..:webrtc_common",
"../../api:libjingle_peerconnection_api",
"../../api:optional",
+ "../../api/audio:audio_frame_api",
"../../api/audio_codecs:audio_codecs_api",
"../../common_audio",
"../../common_audio:common_audio_c",
+ "../../rtc_base:audio_format_to_string",
"../../rtc_base:checks",
"../../rtc_base:gtest_prod",
"../../rtc_base:rtc_base_approved",
@@ -1125,11 +1149,11 @@
deps = [
":neteq",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../api:libjingle_peerconnection_api",
"../../api:optional",
+ "../../api/audio:audio_frame_api",
"../../api/audio_codecs:audio_codecs_api",
"../../api/audio_codecs:builtin_audio_decoder_factory",
"../../rtc_base:checks",
@@ -1164,7 +1188,6 @@
deps = [
":pcm16b",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../api:array_view",
@@ -1206,6 +1229,8 @@
"neteq/tools/neteq_delay_analyzer.h",
"neteq/tools/neteq_replacement_input.cc",
"neteq/tools/neteq_replacement_input.h",
+ "neteq/tools/neteq_stats_getter.cc",
+ "neteq/tools/neteq_stats_getter.h",
]
public_configs = [ ":neteq_tools_config" ]
@@ -1216,6 +1241,7 @@
}
deps = [
+ "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../api:array_view",
@@ -1375,8 +1401,11 @@
"../..:typedefs",
"../..:webrtc_common",
"../../api:optional",
+ "../../api/audio:audio_frame_api",
"../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
+ "../../rtc_base/synchronization:rw_lock_wrapper",
"../../system_wrappers",
"../../test:fileutils",
"../../test:test_support",
@@ -1433,6 +1462,7 @@
defines = audio_coding_defines
deps = audio_coding_deps + [
+ "..:module_api",
":audio_coding",
":audio_format_conversion",
"../../api/audio_codecs:audio_codecs_api",
@@ -1454,9 +1484,11 @@
defines = audio_coding_defines
deps = audio_coding_deps + [
+ "../../api/audio:audio_frame_api",
"../../rtc_base:checks",
":audio_coding",
":neteq_tools",
+ "../../api/audio_codecs:builtin_audio_decoder_factory",
"../../api/audio_codecs:audio_codecs_api",
"../../rtc_base:rtc_base_approved",
"../../test:test_support",
@@ -1484,6 +1516,9 @@
"../..:typedefs",
"../../:webrtc_common",
"../../api:optional",
+ "../../api/audio:audio_frame_api",
+ "../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
"../../system_wrappers:system_wrappers_default",
@@ -1516,6 +1551,9 @@
"../..:typedefs",
"../../:webrtc_common",
"../../api:optional",
+ "../../api/audio:audio_frame_api",
+ "../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
"../../system_wrappers:system_wrappers_default",
@@ -1594,7 +1632,6 @@
testonly = true
defines = []
deps = [
- "..:module_api",
"../..:typedefs",
"../../rtc_base:checks",
"../../test:fileutils",
@@ -1705,9 +1742,9 @@
":neteq",
":neteq_test_tools",
":pcm16b",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
+ "../../api/audio:audio_frame_api",
"../../api/audio_codecs:audio_codecs_api",
"../../api/audio_codecs:builtin_audio_decoder_factory",
"../../rtc_base:checks",
@@ -1734,7 +1771,6 @@
deps = [
":neteq",
":neteq_test_tools",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../api/audio_codecs:builtin_audio_decoder_factory",
@@ -1753,6 +1789,7 @@
"../..:typedefs",
":audio_coding",
":neteq_input_audio_tools",
+ "../../api/audio:audio_frame_api",
"../../api/audio_codecs/g711:audio_encoder_g711",
"../../api/audio_codecs/L16:audio_encoder_L16",
"../../api/audio_codecs/g722:audio_encoder_g722",
@@ -2214,6 +2251,7 @@
"..:module_api",
"../..:typedefs",
"../..:webrtc_common",
+ "../../api/audio:audio_frame_api",
"../../api/audio_codecs:audio_codecs_api",
"../../api/audio_codecs:builtin_audio_decoder_factory",
"../../api/audio_codecs:builtin_audio_encoder_factory",
diff --git a/modules/audio_coding/acm2/acm_receive_test.cc b/modules/audio_coding/acm2/acm_receive_test.cc
index 082506a..473b651 100644
--- a/modules/audio_coding/acm2/acm_receive_test.cc
+++ b/modules/audio_coding/acm2/acm_receive_test.cc
@@ -21,6 +21,7 @@
#include "modules/audio_coding/neteq/tools/audio_sink.h"
#include "modules/audio_coding/neteq/tools/packet.h"
#include "modules/audio_coding/neteq/tools/packet_source.h"
+#include "modules/include/module_common_types.h"
#include "test/gtest.h"
namespace webrtc {
diff --git a/modules/audio_coding/acm2/acm_receiver.cc b/modules/audio_coding/acm2/acm_receiver.cc
index 0d5dcae..41a23a7 100644
--- a/modules/audio_coding/acm2/acm_receiver.cc
+++ b/modules/audio_coding/acm2/acm_receiver.cc
@@ -22,10 +22,12 @@
#include "modules/audio_coding/acm2/call_statistics.h"
#include "modules/audio_coding/acm2/rent_a_codec.h"
#include "modules/audio_coding/neteq/include/neteq.h"
+#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/format_macros.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
+#include "rtc_base/strings/audio_format_to_string.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
@@ -260,7 +262,8 @@
neteq_->RegisterPayloadType(rtp_payload_type, audio_format);
if (!success) {
RTC_LOG(LERROR) << "AcmReceiver::AddCodec failed for payload type "
- << rtp_payload_type << ", decoder format " << audio_format;
+ << rtp_payload_type << ", decoder format "
+ << rtc::ToString(audio_format);
}
return success;
}
diff --git a/modules/audio_coding/acm2/acm_receiver.h b/modules/audio_coding/acm2/acm_receiver.h
index 5c6b36f..ce1e1f2 100644
--- a/modules/audio_coding/acm2/acm_receiver.h
+++ b/modules/audio_coding/acm2/acm_receiver.h
@@ -16,6 +16,7 @@
#include <string>
#include <vector>
+#include "api/audio/audio_frame.h"
#include "api/array_view.h"
#include "api/optional.h"
#include "common_audio/vad/include/webrtc_vad.h"
@@ -23,7 +24,6 @@
#include "modules/audio_coding/acm2/call_statistics.h"
#include "modules/audio_coding/include/audio_coding_module.h"
#include "modules/audio_coding/neteq/include/neteq.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/thread_annotations.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_coding/acm2/acm_receiver_unittest.cc b/modules/audio_coding/acm2/acm_receiver_unittest.cc
index 8d0b2f1..7877821 100644
--- a/modules/audio_coding/acm2/acm_receiver_unittest.cc
+++ b/modules/audio_coding/acm2/acm_receiver_unittest.cc
@@ -17,6 +17,7 @@
#include "modules/audio_coding/acm2/rent_a_codec.h"
#include "modules/audio_coding/include/audio_coding_module.h"
#include "modules/audio_coding/neteq/tools/rtp_generator.h"
+#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "system_wrappers/include/clock.h"
diff --git a/modules/audio_coding/acm2/acm_send_test.cc b/modules/audio_coding/acm2/acm_send_test.cc
index 307c906..09a6c80 100644
--- a/modules/audio_coding/acm2/acm_send_test.cc
+++ b/modules/audio_coding/acm2/acm_send_test.cc
@@ -15,6 +15,7 @@
#include <string.h>
#include "api/audio_codecs/audio_encoder.h"
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "modules/audio_coding/include/audio_coding_module.h"
#include "modules/audio_coding/neteq/tools/input_audio_file.h"
#include "modules/audio_coding/neteq/tools/packet.h"
@@ -28,7 +29,12 @@
int source_rate_hz,
int test_duration_ms)
: clock_(0),
- acm_(webrtc::AudioCodingModule::Create(&clock_)),
+ acm_(webrtc::AudioCodingModule::Create([this] {
+ AudioCodingModule::Config config;
+ config.clock = &clock_;
+ config.decoder_factory = CreateBuiltinAudioDecoderFactory();
+ return config;
+ }())),
audio_source_(audio_source),
source_rate_hz_(source_rate_hz),
input_block_size_samples_(
diff --git a/modules/audio_coding/acm2/acm_send_test.h b/modules/audio_coding/acm2/acm_send_test.h
index 6aea0f1..68ba9e1 100644
--- a/modules/audio_coding/acm2/acm_send_test.h
+++ b/modules/audio_coding/acm2/acm_send_test.h
@@ -14,6 +14,7 @@
#include <memory>
#include <vector>
+#include "api/audio/audio_frame.h"
#include "modules/audio_coding/include/audio_coding_module.h"
#include "modules/audio_coding/neteq/tools/packet_source.h"
#include "rtc_base/constructormagic.h"
diff --git a/modules/audio_coding/acm2/audio_coding_module.cc b/modules/audio_coding/acm2/audio_coding_module.cc
index 53b9177..4545810 100644
--- a/modules/audio_coding/acm2/audio_coding_module.cc
+++ b/modules/audio_coding/acm2/audio_coding_module.cc
@@ -12,11 +12,11 @@
#include <algorithm>
-#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "modules/audio_coding/acm2/acm_receiver.h"
#include "modules/audio_coding/acm2/acm_resampler.h"
#include "modules/audio_coding/acm2/codec_manager.h"
#include "modules/audio_coding/acm2/rent_a_codec.h"
+#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
@@ -1243,8 +1243,11 @@
} // namespace
-AudioCodingModule::Config::Config()
- : neteq_config(), clock(Clock::GetRealTimeClock()) {
+AudioCodingModule::Config::Config(
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
+ : neteq_config(),
+ clock(Clock::GetRealTimeClock()),
+ decoder_factory(decoder_factory) {
// Post-decode VAD is disabled by default in NetEq, however, Audio
// Conference Mixer relies on VAD decisions and fails without them.
neteq_config.enable_post_decode_vad = true;
@@ -1253,34 +1256,7 @@
AudioCodingModule::Config::Config(const Config&) = default;
AudioCodingModule::Config::~Config() = default;
-AudioCodingModule* AudioCodingModule::Create(int id) {
- RTC_UNUSED(id);
- return Create();
-}
-
-// Create module
-AudioCodingModule* AudioCodingModule::Create() {
- Config config;
- config.clock = Clock::GetRealTimeClock();
- config.decoder_factory = CreateBuiltinAudioDecoderFactory();
- return Create(config);
-}
-
-AudioCodingModule* AudioCodingModule::Create(Clock* clock) {
- Config config;
- config.clock = clock;
- config.decoder_factory = CreateBuiltinAudioDecoderFactory();
- return Create(config);
-}
-
AudioCodingModule* AudioCodingModule::Create(const Config& config) {
- if (!config.decoder_factory) {
- // TODO(ossu): Backwards compatibility. Will be removed after a deprecation
- // cycle.
- Config config_copy = config;
- config_copy.decoder_factory = CreateBuiltinAudioDecoderFactory();
- return new AudioCodingModuleImpl(config_copy);
- }
return new AudioCodingModuleImpl(config);
}
diff --git a/modules/audio_coding/acm2/audio_coding_module_unittest.cc b/modules/audio_coding/acm2/audio_coding_module_unittest.cc
index aaa4230..ff5431d 100644
--- a/modules/audio_coding/acm2/audio_coding_module_unittest.cc
+++ b/modules/audio_coding/acm2/audio_coding_module_unittest.cc
@@ -32,7 +32,6 @@
#include "modules/audio_coding/neteq/tools/output_wav_file.h"
#include "modules/audio_coding/neteq/tools/packet.h"
#include "modules/audio_coding/neteq/tools/rtp_file_source.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/messagedigest.h"
#include "rtc_base/numerics/safe_conversions.h"
@@ -166,7 +165,12 @@
void TearDown() {}
void SetUp() {
- acm_.reset(AudioCodingModule::Create(clock_));
+ acm_.reset(AudioCodingModule::Create([this] {
+ AudioCodingModule::Config config;
+ config.clock = clock_;
+ config.decoder_factory = CreateBuiltinAudioDecoderFactory();
+ return config;
+ }()));
rtp_utility_->Populate(&rtp_header_);
diff --git a/modules/audio_coding/acm2/call_statistics.h b/modules/audio_coding/acm2/call_statistics.h
index 9dd052f..9dced64 100644
--- a/modules/audio_coding/acm2/call_statistics.h
+++ b/modules/audio_coding/acm2/call_statistics.h
@@ -11,8 +11,8 @@
#ifndef MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_
#define MODULES_AUDIO_CODING_ACM2_CALL_STATISTICS_H_
+#include "api/audio/audio_frame.h"
#include "common_types.h" // NOLINT(build/include)
-#include "modules/include/module_common_types.h"
//
// This class is for book keeping of calls to ACM. It is not useful to log API
diff --git a/modules/audio_coding/acm2/rent_a_codec_unittest.cc b/modules/audio_coding/acm2/rent_a_codec_unittest.cc
index c949c1c..ca469e7 100644
--- a/modules/audio_coding/acm2/rent_a_codec_unittest.cc
+++ b/modules/audio_coding/acm2/rent_a_codec_unittest.cc
@@ -10,6 +10,7 @@
#include <memory>
+#include "common_types.h"
#include "modules/audio_coding/acm2/rent_a_codec.h"
#include "rtc_base/arraysize.h"
#include "test/gtest.h"
diff --git a/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc b/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc
index c437918..574b699 100644
--- a/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc
+++ b/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl_unittest.cc
@@ -314,12 +314,12 @@
auto ana_stats = states.audio_network_adaptor->GetStats();
- EXPECT_EQ(ana_stats.bitrate_action_counter, 2);
- EXPECT_EQ(ana_stats.channel_action_counter, 2);
- EXPECT_EQ(ana_stats.dtx_action_counter, 2);
- EXPECT_EQ(ana_stats.fec_action_counter, 2);
- EXPECT_EQ(ana_stats.frame_length_increase_counter, 1);
- EXPECT_EQ(ana_stats.frame_length_decrease_counter, 1);
+ EXPECT_EQ(ana_stats.bitrate_action_counter, 2u);
+ EXPECT_EQ(ana_stats.channel_action_counter, 2u);
+ EXPECT_EQ(ana_stats.dtx_action_counter, 2u);
+ EXPECT_EQ(ana_stats.fec_action_counter, 2u);
+ EXPECT_EQ(ana_stats.frame_length_increase_counter, 1u);
+ EXPECT_EQ(ana_stats.frame_length_decrease_counter, 1u);
EXPECT_EQ(ana_stats.uplink_packet_loss_fraction, 0.1f);
}
diff --git a/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h b/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h
index e40c832..b464eb6 100644
--- a/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h
+++ b/modules/audio_coding/audio_network_adaptor/debug_dump_writer.h
@@ -17,7 +17,7 @@
#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/ignore_wundef.h"
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#if WEBRTC_ENABLE_PROTOBUF
RTC_PUSH_IGNORING_WUNDEF()
#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
diff --git a/modules/audio_coding/codecs/cng/audio_encoder_cng.cc b/modules/audio_coding/codecs/cng/audio_encoder_cng.cc
index 78148ab..dc4c21e 100644
--- a/modules/audio_coding/codecs/cng/audio_encoder_cng.cc
+++ b/modules/audio_coding/codecs/cng/audio_encoder_cng.cc
@@ -44,19 +44,19 @@
}
AudioEncoderCng::AudioEncoderCng(Config&& config)
- : speech_encoder_(
- ([&] { RTC_CHECK(config.IsOk()) << "Invalid configuration."; }(),
- std::move(config.speech_encoder))),
+ : speech_encoder_((static_cast<void>([&] {
+ RTC_CHECK(config.IsOk()) << "Invalid configuration.";
+ }()),
+ std::move(config.speech_encoder))),
cng_payload_type_(config.payload_type),
num_cng_coefficients_(config.num_cng_coefficients),
sid_frame_interval_ms_(config.sid_frame_interval_ms),
last_frame_active_(true),
vad_(config.vad ? std::unique_ptr<Vad>(config.vad)
- : CreateVad(config.vad_mode)),
+ : CreateVad(config.vad_mode)),
cng_encoder_(new ComfortNoiseEncoder(SampleRateHz(),
sid_frame_interval_ms_,
- num_cng_coefficients_)) {
-}
+ num_cng_coefficients_)) {}
AudioEncoderCng::~AudioEncoderCng() = default;
diff --git a/modules/audio_coding/codecs/ilbc/cb_construct.c b/modules/audio_coding/codecs/ilbc/cb_construct.c
index e2ae361..1e9a704 100644
--- a/modules/audio_coding/codecs/ilbc/cb_construct.c
+++ b/modules/audio_coding/codecs/ilbc/cb_construct.c
@@ -21,6 +21,15 @@
#include "modules/audio_coding/codecs/ilbc/defines.h"
#include "modules/audio_coding/codecs/ilbc/gain_dequant.h"
#include "modules/audio_coding/codecs/ilbc/get_cd_vec.h"
+#include "rtc_base/sanitizer.h"
+
+// An arithmetic operation that is allowed to overflow. (It's still undefined
+// behavior, so not a good idea; this just makes UBSan ignore the violation, so
+// that our old code can continue to do what it's always been doing.)
+static inline int32_t RTC_NO_SANITIZE("signed-integer-overflow")
+ OverflowingAddS32S32ToS32(int32_t a, int32_t b) {
+ return a + b;
+}
/*----------------------------------------------------------------*
* Construct decoded vector from codebook and gains.
@@ -62,7 +71,7 @@
for (j=0;j<veclen;j++) {
a32 = (*gainPtr++) * cbvec0[j];
a32 += (*gainPtr++) * cbvec1[j];
- a32 += (*gainPtr) * cbvec2[j];
+ a32 = OverflowingAddS32S32ToS32(a32, (*gainPtr) * cbvec2[j]);
gainPtr -= 2;
decvector[j] = (int16_t)((a32 + 8192) >> 14);
}
diff --git a/modules/audio_coding/codecs/isac/fix/source/codec.h b/modules/audio_coding/codecs/isac/fix/source/codec.h
index 9b87c40..9876bd6 100644
--- a/modules/audio_coding/codecs/isac/fix/source/codec.h
+++ b/modules/audio_coding/codecs/isac/fix/source/codec.h
@@ -67,7 +67,7 @@
/* transform functions */
-void WebRtcIsacfix_InitTransform();
+void WebRtcIsacfix_InitTransform(void);
typedef void (*Time2Spec)(int16_t* inre1Q9,
int16_t* inre2Q9,
diff --git a/modules/audio_coding/codecs/isac/fix/source/filterbanks_neon.c b/modules/audio_coding/codecs/isac/fix/source/filterbanks_neon.c
index fd29ccb..a31cea6 100644
--- a/modules/audio_coding/codecs/isac/fix/source/filterbanks_neon.c
+++ b/modules/audio_coding/codecs/isac/fix/source/filterbanks_neon.c
@@ -15,6 +15,7 @@
#include <arm_neon.h>
+#include "modules/audio_coding/codecs/isac/fix/source/filterbank_internal.h"
#include "rtc_base/checks.h"
void WebRtcIsacfix_AllpassFilter2FixDec16Neon(
diff --git a/modules/audio_coding/codecs/isac/fix/source/lattice.c b/modules/audio_coding/codecs/isac/fix/source/lattice.c
index 1089549..7bbf4e0 100644
--- a/modules/audio_coding/codecs/isac/fix/source/lattice.c
+++ b/modules/audio_coding/codecs/isac/fix/source/lattice.c
@@ -279,8 +279,7 @@
for (i=0;i<HALF_SUBFRAMELEN;i++)
{
-
- tmp32 = lat_inQ25[i + temp1] * (1 << 1); // Q25->Q26
+ tmp32 = OverflowingLShiftS32(lat_inQ25[i + temp1], 1); // Q25->Q26
tmp32 = WEBRTC_SPL_MUL_16_32_RSFT16(inv_gain16, tmp32); //lat_in[]*inv_gain in (Q(18-sh)*Q26)>>16 = Q(28-sh)
tmp32 = WEBRTC_SPL_SHIFT_W32(tmp32, -(28-sh)); // lat_in[]*inv_gain in Q0
diff --git a/modules/audio_coding/codecs/isac/fix/source/lattice_armv7.S b/modules/audio_coding/codecs/isac/fix/source/lattice_armv7.S
index 67ca4a4..4c63227 100644
--- a/modules/audio_coding/codecs/isac/fix/source/lattice_armv7.S
+++ b/modules/audio_coding/codecs/isac/fix/source/lattice_armv7.S
@@ -25,8 +25,8 @@
@ r12: constant #16384
@ r6, r7, r8, r10, r11: scratch
-#include "system_wrappers/include/asm_defines.h"
#include "modules/audio_coding/codecs/isac/fix/source/settings.h"
+#include "rtc_base/system/asm_defines.h"
GLOBAL_FUNCTION WebRtcIsacfix_FilterArLoop
.align 2
diff --git a/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S b/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S
index d5b5541..0659468 100644
--- a/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S
+++ b/modules/audio_coding/codecs/isac/fix/source/pitch_filter_armv6.S
@@ -13,8 +13,8 @@
@
@ Output is bit-exact with the reference C code in pitch_filter.c.
-#include "system_wrappers/include/asm_defines.h"
#include "modules/audio_coding/codecs/isac/fix/source/settings.h"
+#include "rtc_base/system/asm_defines.h"
GLOBAL_FUNCTION WebRtcIsacfix_PitchFilterCore
.align 2
diff --git a/modules/audio_coding/codecs/isac/main/source/isac.c b/modules/audio_coding/codecs/isac/main/source/isac.c
index 79dc7e2..525e0f3 100644
--- a/modules/audio_coding/codecs/isac/main/source/isac.c
+++ b/modules/audio_coding/codecs/isac/main/source/isac.c
@@ -1266,8 +1266,10 @@
/* It might be less due to garbage. */
if ((numDecodedBytesUB != lenNextStream) &&
- (numDecodedBytesUB != (lenNextStream -
- encoded[numDecodedBytesLB + 1 + numDecodedBytesUB]))) {
+ (numDecodedBytesLB + 1 + numDecodedBytesUB >= lenEncodedBytes ||
+ numDecodedBytesUB !=
+ (lenNextStream -
+ encoded[numDecodedBytesLB + 1 + numDecodedBytesUB]))) {
instISAC->errorCode = ISAC_LENGTH_MISMATCH;
return -1;
}
diff --git a/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
index dfef682..bd34118 100644
--- a/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
+++ b/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
@@ -21,6 +21,7 @@
#include "modules/audio_coding/neteq/tools/audio_loop.h"
#include "rtc_base/checks.h"
#include "rtc_base/fakeclock.h"
+#include "rtc_base/ptr_util.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -54,43 +55,39 @@
}
struct AudioEncoderOpusStates {
- std::shared_ptr<MockAudioNetworkAdaptor*> mock_audio_network_adaptor;
+ MockAudioNetworkAdaptor* mock_audio_network_adaptor;
MockSmoothingFilter* mock_bitrate_smoother;
std::unique_ptr<AudioEncoderOpusImpl> encoder;
std::unique_ptr<rtc::ScopedFakeClock> fake_clock;
AudioEncoderOpusConfig config;
};
-AudioEncoderOpusStates CreateCodec(size_t num_channels) {
- AudioEncoderOpusStates states;
- states.mock_audio_network_adaptor =
- std::make_shared<MockAudioNetworkAdaptor*>(nullptr);
- states.fake_clock.reset(new rtc::ScopedFakeClock());
- states.fake_clock->SetTimeMicros(kInitialTimeUs);
- std::weak_ptr<MockAudioNetworkAdaptor*> mock_ptr(
- states.mock_audio_network_adaptor);
+std::unique_ptr<AudioEncoderOpusStates> CreateCodec(size_t num_channels) {
+ std::unique_ptr<AudioEncoderOpusStates> states =
+ rtc::MakeUnique<AudioEncoderOpusStates>();
+ states->mock_audio_network_adaptor = nullptr;
+ states->fake_clock.reset(new rtc::ScopedFakeClock());
+ states->fake_clock->SetTimeMicros(kInitialTimeUs);
+
+ MockAudioNetworkAdaptor** mock_ptr = &states->mock_audio_network_adaptor;
AudioEncoderOpusImpl::AudioNetworkAdaptorCreator creator =
[mock_ptr](const std::string&, RtcEventLog* event_log) {
std::unique_ptr<MockAudioNetworkAdaptor> adaptor(
new NiceMock<MockAudioNetworkAdaptor>());
EXPECT_CALL(*adaptor, Die());
- if (auto sp = mock_ptr.lock()) {
- *sp = adaptor.get();
- } else {
- RTC_NOTREACHED();
- }
+ *mock_ptr = adaptor.get();
return adaptor;
};
CodecInst codec_inst = kDefaultOpusSettings;
codec_inst.channels = num_channels;
- states.config = CreateConfig(codec_inst);
+ states->config = CreateConfig(codec_inst);
std::unique_ptr<MockSmoothingFilter> bitrate_smoother(
new MockSmoothingFilter());
- states.mock_bitrate_smoother = bitrate_smoother.get();
+ states->mock_bitrate_smoother = bitrate_smoother.get();
- states.encoder.reset(new AudioEncoderOpusImpl(
- states.config, codec_inst.pltype, std::move(creator),
+ states->encoder.reset(new AudioEncoderOpusImpl(
+ states->config, codec_inst.pltype, std::move(creator),
std::move(bitrate_smoother)));
return states;
}
@@ -145,77 +142,77 @@
TEST(AudioEncoderOpusTest, DefaultApplicationModeMono) {
auto states = CreateCodec(1);
EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip,
- states.encoder->application());
+ states->encoder->application());
}
TEST(AudioEncoderOpusTest, DefaultApplicationModeStereo) {
auto states = CreateCodec(2);
EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kAudio,
- states.encoder->application());
+ states->encoder->application());
}
TEST(AudioEncoderOpusTest, ChangeApplicationMode) {
auto states = CreateCodec(2);
EXPECT_TRUE(
- states.encoder->SetApplication(AudioEncoder::Application::kSpeech));
+ states->encoder->SetApplication(AudioEncoder::Application::kSpeech));
EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip,
- states.encoder->application());
+ states->encoder->application());
}
TEST(AudioEncoderOpusTest, ResetWontChangeApplicationMode) {
auto states = CreateCodec(2);
// Trigger a reset.
- states.encoder->Reset();
+ states->encoder->Reset();
// Verify that the mode is still kAudio.
EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kAudio,
- states.encoder->application());
+ states->encoder->application());
// Now change to kVoip.
EXPECT_TRUE(
- states.encoder->SetApplication(AudioEncoder::Application::kSpeech));
+ states->encoder->SetApplication(AudioEncoder::Application::kSpeech));
EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip,
- states.encoder->application());
+ states->encoder->application());
// Trigger a reset again.
- states.encoder->Reset();
+ states->encoder->Reset();
// Verify that the mode is still kVoip.
EXPECT_EQ(AudioEncoderOpusConfig::ApplicationMode::kVoip,
- states.encoder->application());
+ states->encoder->application());
}
TEST(AudioEncoderOpusTest, ToggleDtx) {
auto states = CreateCodec(2);
// Enable DTX
- EXPECT_TRUE(states.encoder->SetDtx(true));
- EXPECT_TRUE(states.encoder->GetDtx());
+ EXPECT_TRUE(states->encoder->SetDtx(true));
+ EXPECT_TRUE(states->encoder->GetDtx());
// Turn off DTX.
- EXPECT_TRUE(states.encoder->SetDtx(false));
- EXPECT_FALSE(states.encoder->GetDtx());
+ EXPECT_TRUE(states->encoder->SetDtx(false));
+ EXPECT_FALSE(states->encoder->GetDtx());
}
TEST(AudioEncoderOpusTest,
OnReceivedUplinkBandwidthWithoutAudioNetworkAdaptor) {
auto states = CreateCodec(1);
- // Constants are replicated from audio_states.encoderopus.cc.
+ // Constants are replicated from audio_states->encoderopus.cc.
const int kMinBitrateBps = 6000;
const int kMaxBitrateBps = 510000;
// Set a too low bitrate.
- states.encoder->OnReceivedUplinkBandwidth(kMinBitrateBps - 1, rtc::nullopt);
- EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(kMinBitrateBps - 1, rtc::nullopt);
+ EXPECT_EQ(kMinBitrateBps, states->encoder->GetTargetBitrate());
// Set a too high bitrate.
- states.encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps + 1, rtc::nullopt);
- EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps + 1, rtc::nullopt);
+ EXPECT_EQ(kMaxBitrateBps, states->encoder->GetTargetBitrate());
// Set the minimum rate.
- states.encoder->OnReceivedUplinkBandwidth(kMinBitrateBps, rtc::nullopt);
- EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(kMinBitrateBps, rtc::nullopt);
+ EXPECT_EQ(kMinBitrateBps, states->encoder->GetTargetBitrate());
// Set the maximum rate.
- states.encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps, rtc::nullopt);
- EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(kMaxBitrateBps, rtc::nullopt);
+ EXPECT_EQ(kMaxBitrateBps, states->encoder->GetTargetBitrate());
// Set rates from kMaxBitrateBps up to 32000 bps.
for (int rate = kMinBitrateBps; rate <= 32000; rate += 1000) {
- states.encoder->OnReceivedUplinkBandwidth(rate, rtc::nullopt);
- EXPECT_EQ(rate, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(rate, rtc::nullopt);
+ EXPECT_EQ(rate, states->encoder->GetTargetBitrate());
}
}
@@ -237,7 +234,7 @@
// Sets the packet loss rate to each number in the vector in turn, and verifies
// that the loss rate as reported by the encoder is |expected_return| for all
// of them.
-void TestSetPacketLossRate(AudioEncoderOpusStates* states,
+void TestSetPacketLossRate(const AudioEncoderOpusStates* states,
const std::vector<float>& losses,
float expected_return) {
// |kSampleIntervalMs| is chosen to ease the calculation since
@@ -262,17 +259,17 @@
// Note that the order of the following calls is critical.
// clang-format off
- TestSetPacketLossRate(&states, I(0.00f , 0.01f - eps), 0.00f);
- TestSetPacketLossRate(&states, I(0.01f + eps, 0.06f - eps), 0.01f);
- TestSetPacketLossRate(&states, I(0.06f + eps, 0.11f - eps), 0.05f);
- TestSetPacketLossRate(&states, I(0.11f + eps, 0.22f - eps), 0.10f);
- TestSetPacketLossRate(&states, I(0.22f + eps, 1.00f ), 0.20f);
+ TestSetPacketLossRate(states.get(), I(0.00f , 0.01f - eps), 0.00f);
+ TestSetPacketLossRate(states.get(), I(0.01f + eps, 0.06f - eps), 0.01f);
+ TestSetPacketLossRate(states.get(), I(0.06f + eps, 0.11f - eps), 0.05f);
+ TestSetPacketLossRate(states.get(), I(0.11f + eps, 0.22f - eps), 0.10f);
+ TestSetPacketLossRate(states.get(), I(0.22f + eps, 1.00f ), 0.20f);
- TestSetPacketLossRate(&states, I(1.00f , 0.18f + eps), 0.20f);
- TestSetPacketLossRate(&states, I(0.18f - eps, 0.09f + eps), 0.10f);
- TestSetPacketLossRate(&states, I(0.09f - eps, 0.04f + eps), 0.05f);
- TestSetPacketLossRate(&states, I(0.04f - eps, 0.01f + eps), 0.01f);
- TestSetPacketLossRate(&states, I(0.01f - eps, 0.00f ), 0.00f);
+ TestSetPacketLossRate(states.get(), I(1.00f , 0.18f + eps), 0.20f);
+ TestSetPacketLossRate(states.get(), I(0.18f - eps, 0.09f + eps), 0.10f);
+ TestSetPacketLossRate(states.get(), I(0.09f - eps, 0.04f + eps), 0.05f);
+ TestSetPacketLossRate(states.get(), I(0.04f - eps, 0.01f + eps), 0.01f);
+ TestSetPacketLossRate(states.get(), I(0.01f - eps, 0.00f ), 0.00f);
// clang-format on
}
@@ -282,85 +279,85 @@
// |supported_frame_lengths_ms| should contain only the frame length being
// used.
using ::testing::ElementsAre;
- EXPECT_THAT(states.encoder->supported_frame_lengths_ms(),
- ElementsAre(states.encoder->next_frame_length_ms()));
- states.encoder->SetReceiverFrameLengthRange(0, 12345);
- states.encoder->SetReceiverFrameLengthRange(21, 60);
- EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), ElementsAre(60));
- states.encoder->SetReceiverFrameLengthRange(20, 59);
- EXPECT_THAT(states.encoder->supported_frame_lengths_ms(), ElementsAre(20));
+ EXPECT_THAT(states->encoder->supported_frame_lengths_ms(),
+ ElementsAre(states->encoder->next_frame_length_ms()));
+ states->encoder->SetReceiverFrameLengthRange(0, 12345);
+ states->encoder->SetReceiverFrameLengthRange(21, 60);
+ EXPECT_THAT(states->encoder->supported_frame_lengths_ms(), ElementsAre(60));
+ states->encoder->SetReceiverFrameLengthRange(20, 59);
+ EXPECT_THAT(states->encoder->supported_frame_lengths_ms(), ElementsAre(20));
}
TEST(AudioEncoderOpusTest,
InvokeAudioNetworkAdaptorOnReceivedUplinkPacketLossFraction) {
auto states = CreateCodec(2);
- states.encoder->EnableAudioNetworkAdaptor("", nullptr);
+ states->encoder->EnableAudioNetworkAdaptor("", nullptr);
auto config = CreateEncoderRuntimeConfig();
- EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig())
+ EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig())
.WillOnce(Return(config));
// Since using mock audio network adaptor, any packet loss fraction is fine.
constexpr float kUplinkPacketLoss = 0.1f;
- EXPECT_CALL(**states.mock_audio_network_adaptor,
+ EXPECT_CALL(*states->mock_audio_network_adaptor,
SetUplinkPacketLossFraction(kUplinkPacketLoss));
- states.encoder->OnReceivedUplinkPacketLossFraction(kUplinkPacketLoss);
+ states->encoder->OnReceivedUplinkPacketLossFraction(kUplinkPacketLoss);
- CheckEncoderRuntimeConfig(states.encoder.get(), config);
+ CheckEncoderRuntimeConfig(states->encoder.get(), config);
}
TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) {
auto states = CreateCodec(2);
- states.encoder->EnableAudioNetworkAdaptor("", nullptr);
+ states->encoder->EnableAudioNetworkAdaptor("", nullptr);
auto config = CreateEncoderRuntimeConfig();
- EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig())
+ EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig())
.WillOnce(Return(config));
// Since using mock audio network adaptor, any target audio bitrate is fine.
constexpr int kTargetAudioBitrate = 30000;
constexpr int64_t kProbingIntervalMs = 3000;
- EXPECT_CALL(**states.mock_audio_network_adaptor,
+ EXPECT_CALL(*states->mock_audio_network_adaptor,
SetTargetAudioBitrate(kTargetAudioBitrate));
- EXPECT_CALL(*states.mock_bitrate_smoother,
+ EXPECT_CALL(*states->mock_bitrate_smoother,
SetTimeConstantMs(kProbingIntervalMs * 4));
- EXPECT_CALL(*states.mock_bitrate_smoother, AddSample(kTargetAudioBitrate));
- states.encoder->OnReceivedUplinkBandwidth(kTargetAudioBitrate,
- kProbingIntervalMs);
+ EXPECT_CALL(*states->mock_bitrate_smoother, AddSample(kTargetAudioBitrate));
+ states->encoder->OnReceivedUplinkBandwidth(kTargetAudioBitrate,
+ kProbingIntervalMs);
- CheckEncoderRuntimeConfig(states.encoder.get(), config);
+ CheckEncoderRuntimeConfig(states->encoder.get(), config);
}
TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedRtt) {
auto states = CreateCodec(2);
- states.encoder->EnableAudioNetworkAdaptor("", nullptr);
+ states->encoder->EnableAudioNetworkAdaptor("", nullptr);
auto config = CreateEncoderRuntimeConfig();
- EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig())
+ EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig())
.WillOnce(Return(config));
// Since using mock audio network adaptor, any rtt is fine.
constexpr int kRtt = 30;
- EXPECT_CALL(**states.mock_audio_network_adaptor, SetRtt(kRtt));
- states.encoder->OnReceivedRtt(kRtt);
+ EXPECT_CALL(*states->mock_audio_network_adaptor, SetRtt(kRtt));
+ states->encoder->OnReceivedRtt(kRtt);
- CheckEncoderRuntimeConfig(states.encoder.get(), config);
+ CheckEncoderRuntimeConfig(states->encoder.get(), config);
}
TEST(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedOverhead) {
auto states = CreateCodec(2);
- states.encoder->EnableAudioNetworkAdaptor("", nullptr);
+ states->encoder->EnableAudioNetworkAdaptor("", nullptr);
auto config = CreateEncoderRuntimeConfig();
- EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig())
+ EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig())
.WillOnce(Return(config));
// Since using mock audio network adaptor, any overhead is fine.
constexpr size_t kOverhead = 64;
- EXPECT_CALL(**states.mock_audio_network_adaptor, SetOverhead(kOverhead));
- states.encoder->OnReceivedOverhead(kOverhead);
+ EXPECT_CALL(*states->mock_audio_network_adaptor, SetOverhead(kOverhead));
+ states->encoder->OnReceivedOverhead(kOverhead);
- CheckEncoderRuntimeConfig(states.encoder.get(), config);
+ CheckEncoderRuntimeConfig(states->encoder.get(), config);
}
TEST(AudioEncoderOpusTest,
@@ -376,18 +373,18 @@
constexpr int64_t kSecondSampleTimeMs = 6931;
// First time, no filtering.
- states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1);
- EXPECT_FLOAT_EQ(0.01f, states.encoder->packet_loss_rate());
+ states->encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1);
+ EXPECT_FLOAT_EQ(0.01f, states->encoder->packet_loss_rate());
- states.fake_clock->AdvanceTime(
+ states->fake_clock->AdvanceTime(
rtc::TimeDelta::FromMilliseconds(kSecondSampleTimeMs));
- states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2);
+ states->encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2);
// Now the output of packet loss fraction smoother should be
// (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized
// packet loss rate to increase to 0.05. If no smoothing has been made, the
// optimized packet loss rate should have been increase to 0.1.
- EXPECT_FLOAT_EQ(0.05f, states.encoder->packet_loss_rate());
+ EXPECT_FLOAT_EQ(0.05f, states->encoder->packet_loss_rate());
}
TEST(AudioEncoderOpusTest, DoNotInvokeSetTargetBitrateIfOverheadUnknown) {
@@ -396,12 +393,12 @@
auto states = CreateCodec(2);
- states.encoder->OnReceivedUplinkBandwidth(kDefaultOpusSettings.rate * 2,
- rtc::nullopt);
+ states->encoder->OnReceivedUplinkBandwidth(kDefaultOpusSettings.rate * 2,
+ rtc::nullopt);
// Since |OnReceivedOverhead| has not been called, the codec bitrate should
// not change.
- EXPECT_EQ(kDefaultOpusSettings.rate, states.encoder->GetTargetBitrate());
+ EXPECT_EQ(kDefaultOpusSettings.rate, states->encoder->GetTargetBitrate());
}
TEST(AudioEncoderOpusTest, OverheadRemovedFromTargetAudioBitrate) {
@@ -411,15 +408,15 @@
auto states = CreateCodec(2);
constexpr size_t kOverheadBytesPerPacket = 64;
- states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket);
+ states->encoder->OnReceivedOverhead(kOverheadBytesPerPacket);
constexpr int kTargetBitrateBps = 40000;
- states.encoder->OnReceivedUplinkBandwidth(kTargetBitrateBps, rtc::nullopt);
+ states->encoder->OnReceivedUplinkBandwidth(kTargetBitrateBps, rtc::nullopt);
int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize);
EXPECT_EQ(kTargetBitrateBps -
8 * static_cast<int>(kOverheadBytesPerPacket) * packet_rate,
- states.encoder->GetTargetBitrate());
+ states->encoder->GetTargetBitrate());
}
TEST(AudioEncoderOpusTest, BitrateBounded) {
@@ -432,7 +429,7 @@
auto states = CreateCodec(2);
constexpr size_t kOverheadBytesPerPacket = 64;
- states.encoder->OnReceivedOverhead(kOverheadBytesPerPacket);
+ states->encoder->OnReceivedOverhead(kOverheadBytesPerPacket);
int packet_rate = rtc::CheckedDivExact(48000, kDefaultOpusSettings.pacsize);
@@ -440,15 +437,15 @@
// subtracted. The eventual codec rate should be bounded by |kMinBitrateBps|.
int target_bitrate =
kOverheadBytesPerPacket * 8 * packet_rate + kMinBitrateBps - 1;
- states.encoder->OnReceivedUplinkBandwidth(target_bitrate, rtc::nullopt);
- EXPECT_EQ(kMinBitrateBps, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(target_bitrate, rtc::nullopt);
+ EXPECT_EQ(kMinBitrateBps, states->encoder->GetTargetBitrate());
// Set a target rate that is greater than |kMaxBitrateBps| when overhead is
// subtracted. The eventual codec rate should be bounded by |kMaxBitrateBps|.
target_bitrate =
kOverheadBytesPerPacket * 8 * packet_rate + kMaxBitrateBps + 1;
- states.encoder->OnReceivedUplinkBandwidth(target_bitrate, rtc::nullopt);
- EXPECT_EQ(kMaxBitrateBps, states.encoder->GetTargetBitrate());
+ states->encoder->OnReceivedUplinkBandwidth(target_bitrate, rtc::nullopt);
+ EXPECT_EQ(kMaxBitrateBps, states->encoder->GetTargetBitrate());
}
// Verifies that the complexity adaptation in the config works as intended.
@@ -534,50 +531,50 @@
TEST(AudioEncoderOpusTest, EmptyConfigDoesNotAffectEncoderSettings) {
auto states = CreateCodec(2);
- states.encoder->EnableAudioNetworkAdaptor("", nullptr);
+ states->encoder->EnableAudioNetworkAdaptor("", nullptr);
auto config = CreateEncoderRuntimeConfig();
AudioEncoderRuntimeConfig empty_config;
- EXPECT_CALL(**states.mock_audio_network_adaptor, GetEncoderRuntimeConfig())
+ EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig())
.WillOnce(Return(config))
.WillOnce(Return(empty_config));
constexpr size_t kOverhead = 64;
- EXPECT_CALL(**states.mock_audio_network_adaptor, SetOverhead(kOverhead))
+ EXPECT_CALL(*states->mock_audio_network_adaptor, SetOverhead(kOverhead))
.Times(2);
- states.encoder->OnReceivedOverhead(kOverhead);
- states.encoder->OnReceivedOverhead(kOverhead);
+ states->encoder->OnReceivedOverhead(kOverhead);
+ states->encoder->OnReceivedOverhead(kOverhead);
- CheckEncoderRuntimeConfig(states.encoder.get(), config);
+ CheckEncoderRuntimeConfig(states->encoder.get(), config);
}
TEST(AudioEncoderOpusTest, UpdateUplinkBandwidthInAudioNetworkAdaptor) {
auto states = CreateCodec(2);
- states.encoder->EnableAudioNetworkAdaptor("", nullptr);
+ states->encoder->EnableAudioNetworkAdaptor("", nullptr);
std::array<int16_t, 480 * 2> audio;
audio.fill(0);
rtc::Buffer encoded;
- EXPECT_CALL(*states.mock_bitrate_smoother, GetAverage())
+ EXPECT_CALL(*states->mock_bitrate_smoother, GetAverage())
.WillOnce(Return(50000));
- EXPECT_CALL(**states.mock_audio_network_adaptor, SetUplinkBandwidth(50000));
- states.encoder->Encode(
+ EXPECT_CALL(*states->mock_audio_network_adaptor, SetUplinkBandwidth(50000));
+ states->encoder->Encode(
0, rtc::ArrayView<const int16_t>(audio.data(), audio.size()), &encoded);
// Repeat update uplink bandwidth tests.
for (int i = 0; i < 5; i++) {
// Don't update till it is time to update again.
- states.fake_clock->AdvanceTime(rtc::TimeDelta::FromMilliseconds(
- states.config.uplink_bandwidth_update_interval_ms - 1));
- states.encoder->Encode(
+ states->fake_clock->AdvanceTime(rtc::TimeDelta::FromMilliseconds(
+ states->config.uplink_bandwidth_update_interval_ms - 1));
+ states->encoder->Encode(
0, rtc::ArrayView<const int16_t>(audio.data(), audio.size()), &encoded);
// Update when it is time to update.
- EXPECT_CALL(*states.mock_bitrate_smoother, GetAverage())
+ EXPECT_CALL(*states->mock_bitrate_smoother, GetAverage())
.WillOnce(Return(40000));
- EXPECT_CALL(**states.mock_audio_network_adaptor, SetUplinkBandwidth(40000));
- states.fake_clock->AdvanceTime(rtc::TimeDelta::FromMilliseconds(1));
- states.encoder->Encode(
+ EXPECT_CALL(*states->mock_audio_network_adaptor, SetUplinkBandwidth(40000));
+ states->fake_clock->AdvanceTime(rtc::TimeDelta::FromMilliseconds(1));
+ states->encoder->Encode(
0, rtc::ArrayView<const int16_t>(audio.data(), audio.size()), &encoded);
}
}
@@ -586,25 +583,25 @@
auto states = CreateCodec(1);
constexpr int kNumPacketsToEncode = 2;
auto audio_frames =
- Create10msAudioBlocks(states.encoder, kNumPacketsToEncode * 20);
+ Create10msAudioBlocks(states->encoder, kNumPacketsToEncode * 20);
ASSERT_TRUE(audio_frames) << "Create10msAudioBlocks failed";
rtc::Buffer encoded;
uint32_t rtp_timestamp = 12345; // Just a number not important to this test.
- states.encoder->OnReceivedUplinkBandwidth(0, rtc::nullopt);
+ states->encoder->OnReceivedUplinkBandwidth(0, rtc::nullopt);
for (int packet_index = 0; packet_index < kNumPacketsToEncode;
packet_index++) {
// Make sure we are not encoding before we have enough data for
// a 20ms packet.
for (int index = 0; index < 1; index++) {
- states.encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(),
- &encoded);
+ states->encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(),
+ &encoded);
EXPECT_EQ(0u, encoded.size());
}
// Should encode now.
- states.encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(),
- &encoded);
+ states->encoder->Encode(rtp_timestamp, audio_frames->GetNextBlock(),
+ &encoded);
EXPECT_GT(encoded.size(), 0u);
encoded.Clear();
}
diff --git a/modules/audio_coding/include/audio_coding_module.h b/modules/audio_coding/include/audio_coding_module.h
index 12c98ee..3c193a4 100644
--- a/modules/audio_coding/include/audio_coding_module.h
+++ b/modules/audio_coding/include/audio_coding_module.h
@@ -21,7 +21,6 @@
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
#include "modules/audio_coding/neteq/include/neteq.h"
-#include "modules/include/module.h"
#include "rtc_base/deprecation.h"
#include "rtc_base/function_view.h"
#include "system_wrappers/include/clock.h"
@@ -66,7 +65,8 @@
public:
struct Config {
- Config();
+ explicit Config(
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory = nullptr);
Config(const Config&);
~Config();
@@ -75,17 +75,6 @@
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory;
};
- ///////////////////////////////////////////////////////////////////////////
- // Creation and destruction of a ACM.
- //
- // The second method is used for testing where a simulated clock can be
- // injected into ACM. ACM will take the ownership of the object clock and
- // delete it when destroyed.
- //
- // TODO(solenberg): Remove once downstream projects are updated.
- RTC_DEPRECATED static AudioCodingModule* Create(int id);
- static AudioCodingModule* Create();
- static AudioCodingModule* Create(Clock* clock);
static AudioCodingModule* Create(const Config& config);
virtual ~AudioCodingModule() = default;
@@ -288,9 +277,7 @@
//
// Input:
// -audio_frame : the input audio frame, containing raw audio
- // sampling frequency etc.,
- // c.f. module_common_types.h for definition of
- // AudioFrame.
+ // sampling frequency etc.
//
// Return value:
// >= 0 number of bytes encoded.
@@ -673,9 +660,7 @@
//
// Output:
// -audio_frame : output audio frame which contains raw audio data
- // and other relevant parameters, c.f.
- // module_common_types.h for the definition of
- // AudioFrame.
+ // and other relevant parameters.
// -muted : if true, the sample data in audio_frame is not
// populated, and must be interpreted as all zero.
//
diff --git a/modules/audio_coding/include/audio_coding_module_typedefs.h b/modules/audio_coding/include/audio_coding_module_typedefs.h
index ad71ef1..85a6bf9 100644
--- a/modules/audio_coding/include/audio_coding_module_typedefs.h
+++ b/modules/audio_coding/include/audio_coding_module_typedefs.h
@@ -13,7 +13,6 @@
#include <map>
-#include "modules/include/module_common_types.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/modules/audio_coding/neteq/background_noise.cc b/modules/audio_coding/neteq/background_noise.cc
index eda5c75..50ffa86 100644
--- a/modules/audio_coding/neteq/background_noise.cc
+++ b/modules/audio_coding/neteq/background_noise.cc
@@ -27,8 +27,7 @@
BackgroundNoise::BackgroundNoise(size_t num_channels)
: num_channels_(num_channels),
- channel_parameters_(new ChannelParameters[num_channels_]),
- mode_(NetEq::kBgnOn) {
+ channel_parameters_(new ChannelParameters[num_channels_]) {
Reset();
}
@@ -39,7 +38,6 @@
for (size_t channel = 0; channel < num_channels_; ++channel) {
channel_parameters_[channel].Reset();
}
- // Keep _bgnMode as it is.
}
void BackgroundNoise::Update(const AudioMultiVector& input,
diff --git a/modules/audio_coding/neteq/background_noise.h b/modules/audio_coding/neteq/background_noise.h
index 718f41d..a6f1395 100644
--- a/modules/audio_coding/neteq/background_noise.h
+++ b/modules/audio_coding/neteq/background_noise.h
@@ -68,11 +68,6 @@
// Accessors.
bool initialized() const { return initialized_; }
- NetEq::BackgroundNoiseMode mode() const { return mode_; }
-
- // Sets the mode of the background noise playout for cases when there is long
- // duration of packet loss.
- void set_mode(NetEq::BackgroundNoiseMode mode) { mode_ = mode; }
private:
static const int kThresholdIncrement = 229; // 0.0035 in Q16.
@@ -128,7 +123,6 @@
size_t num_channels_;
std::unique_ptr<ChannelParameters[]> channel_parameters_;
bool initialized_;
- NetEq::BackgroundNoiseMode mode_;
RTC_DISALLOW_COPY_AND_ASSIGN(BackgroundNoise);
};
diff --git a/modules/audio_coding/neteq/decision_logic.cc b/modules/audio_coding/neteq/decision_logic.cc
index 966d5c3..6ab2716 100644
--- a/modules/audio_coding/neteq/decision_logic.cc
+++ b/modules/audio_coding/neteq/decision_logic.cc
@@ -19,6 +19,7 @@
#include "modules/audio_coding/neteq/expand.h"
#include "modules/audio_coding/neteq/packet_buffer.h"
#include "modules/audio_coding/neteq/sync_buffer.h"
+#include "modules/include/module_common_types.h"
namespace webrtc {
diff --git a/modules/audio_coding/neteq/decision_logic_normal.cc b/modules/audio_coding/neteq/decision_logic_normal.cc
index 10f501a..1429bb7 100644
--- a/modules/audio_coding/neteq/decision_logic_normal.cc
+++ b/modules/audio_coding/neteq/decision_logic_normal.cc
@@ -20,7 +20,6 @@
#include "modules/audio_coding/neteq/expand.h"
#include "modules/audio_coding/neteq/packet_buffer.h"
#include "modules/audio_coding/neteq/sync_buffer.h"
-#include "modules/include/module_common_types.h"
namespace webrtc {
diff --git a/modules/audio_coding/neteq/decision_logic_unittest.cc b/modules/audio_coding/neteq/decision_logic_unittest.cc
index 1a7bab9..be1b854 100644
--- a/modules/audio_coding/neteq/decision_logic_unittest.cc
+++ b/modules/audio_coding/neteq/decision_logic_unittest.cc
@@ -26,7 +26,7 @@
int fs_hz = 8000;
int output_size_samples = fs_hz / 100; // Samples per 10 ms.
DecoderDatabase decoder_database(
- new rtc::RefCountedObject<MockAudioDecoderFactory>);
+ new rtc::RefCountedObject<MockAudioDecoderFactory>, rtc::nullopt);
TickTimer tick_timer;
PacketBuffer packet_buffer(10, &tick_timer);
DelayPeakDetector delay_peak_detector(&tick_timer);
diff --git a/modules/audio_coding/neteq/decoder_database.cc b/modules/audio_coding/neteq/decoder_database.cc
index 5ddaf04..5b940ae 100644
--- a/modules/audio_coding/neteq/decoder_database.cc
+++ b/modules/audio_coding/neteq/decoder_database.cc
@@ -15,40 +15,51 @@
#include "api/audio_codecs/audio_decoder.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
+#include "rtc_base/strings/audio_format_to_string.h"
namespace webrtc {
DecoderDatabase::DecoderDatabase(
- const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory)
+ const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory,
+ rtc::Optional<AudioCodecPairId> codec_pair_id)
: active_decoder_type_(-1),
active_cng_decoder_type_(-1),
- decoder_factory_(decoder_factory) {}
+ decoder_factory_(decoder_factory),
+ codec_pair_id_(codec_pair_id) {}
DecoderDatabase::~DecoderDatabase() = default;
-DecoderDatabase::DecoderInfo::DecoderInfo(const SdpAudioFormat& audio_format,
- AudioDecoderFactory* factory,
- const std::string& codec_name)
+DecoderDatabase::DecoderInfo::DecoderInfo(
+ const SdpAudioFormat& audio_format,
+ rtc::Optional<AudioCodecPairId> codec_pair_id,
+ AudioDecoderFactory* factory,
+ const std::string& codec_name)
: name_(codec_name),
audio_format_(audio_format),
+ codec_pair_id_(codec_pair_id),
factory_(factory),
external_decoder_(nullptr),
cng_decoder_(CngDecoder::Create(audio_format)),
subtype_(SubtypeFromFormat(audio_format)) {}
-DecoderDatabase::DecoderInfo::DecoderInfo(const SdpAudioFormat& audio_format,
- AudioDecoderFactory* factory)
- : DecoderInfo(audio_format, factory, audio_format.name) {}
+DecoderDatabase::DecoderInfo::DecoderInfo(
+ const SdpAudioFormat& audio_format,
+ rtc::Optional<AudioCodecPairId> codec_pair_id,
+ AudioDecoderFactory* factory)
+ : DecoderInfo(audio_format, codec_pair_id, factory, audio_format.name) {}
-DecoderDatabase::DecoderInfo::DecoderInfo(NetEqDecoder ct,
- AudioDecoderFactory* factory)
- : DecoderInfo(*NetEqDecoderToSdpAudioFormat(ct), factory) {}
+DecoderDatabase::DecoderInfo::DecoderInfo(
+ NetEqDecoder ct,
+ rtc::Optional<AudioCodecPairId> codec_pair_id,
+ AudioDecoderFactory* factory)
+ : DecoderInfo(*NetEqDecoderToSdpAudioFormat(ct), codec_pair_id, factory) {}
DecoderDatabase::DecoderInfo::DecoderInfo(const SdpAudioFormat& audio_format,
AudioDecoder* ext_dec,
const std::string& codec_name)
: name_(codec_name),
audio_format_(audio_format),
+ codec_pair_id_(rtc::nullopt),
factory_(nullptr),
external_decoder_(ext_dec),
subtype_(Subtype::kNormal) {
@@ -83,9 +94,9 @@
// TODO(ossu): Keep a check here for now, since a number of tests create
// DecoderInfos without factories.
RTC_DCHECK(factory_);
- decoder_ = factory_->MakeAudioDecoder(audio_format_, rtc::nullopt);
+ decoder_ = factory_->MakeAudioDecoder(audio_format_, codec_pair_id_);
}
- RTC_DCHECK(decoder_) << "Failed to create: " << audio_format_;
+ RTC_DCHECK(decoder_) << "Failed to create: " << rtc::ToString(audio_format_);
return decoder_.get();
}
@@ -156,7 +167,8 @@
RTC_DCHECK_LE(rtp_payload_type, 0x7f);
if (decoders_.count(rtp_payload_type) == 0) {
decoders_.insert(std::make_pair(
- rtp_payload_type, DecoderInfo(audio_format, decoder_factory_.get())));
+ rtp_payload_type,
+ DecoderInfo(audio_format, codec_pair_id_, decoder_factory_.get())));
} else {
// The mapping for this payload type hasn't changed.
}
@@ -178,7 +190,7 @@
if (!opt_format) {
return kCodecNotSupported;
}
- DecoderInfo info(*opt_format, decoder_factory_, name);
+ DecoderInfo info(*opt_format, codec_pair_id_, decoder_factory_, name);
if (!info.CanGetDecoder()) {
return kCodecNotSupported;
}
@@ -197,7 +209,8 @@
return kInvalidRtpPayloadType;
}
const auto ret = decoders_.insert(std::make_pair(
- rtp_payload_type, DecoderInfo(audio_format, decoder_factory_.get())));
+ rtp_payload_type,
+ DecoderInfo(audio_format, codec_pair_id_, decoder_factory_.get())));
if (ret.second == false) {
// Database already contains a decoder with type |rtp_payload_type|.
return kDecoderExists;
@@ -217,7 +230,8 @@
}
const auto opt_db_format = NetEqDecoderToSdpAudioFormat(codec_type);
- const SdpAudioFormat format = opt_db_format.value_or({"arbitrary", 0, 0});
+ const SdpAudioFormat format =
+ opt_db_format.value_or(SdpAudioFormat("arbitrary", 0, 0));
std::pair<DecoderMap::iterator, bool> ret;
DecoderInfo info(format, decoder, codec_name);
diff --git a/modules/audio_coding/neteq/decoder_database.h b/modules/audio_coding/neteq/decoder_database.h
index 5f0d173..f769e39 100644
--- a/modules/audio_coding/neteq/decoder_database.h
+++ b/modules/audio_coding/neteq/decoder_database.h
@@ -43,11 +43,14 @@
class DecoderInfo {
public:
DecoderInfo(const SdpAudioFormat& audio_format,
+ rtc::Optional<AudioCodecPairId> codec_pair_id,
AudioDecoderFactory* factory,
const std::string& codec_name);
explicit DecoderInfo(const SdpAudioFormat& audio_format,
+ rtc::Optional<AudioCodecPairId> codec_pair_id,
AudioDecoderFactory* factory = nullptr);
explicit DecoderInfo(NetEqDecoder ct,
+ rtc::Optional<AudioCodecPairId> codec_pair_id,
AudioDecoderFactory* factory = nullptr);
DecoderInfo(const SdpAudioFormat& audio_format,
AudioDecoder* ext_dec,
@@ -108,6 +111,7 @@
const std::string name_;
const SdpAudioFormat audio_format_;
+ const rtc::Optional<AudioCodecPairId> codec_pair_id_;
AudioDecoderFactory* const factory_;
mutable std::unique_ptr<AudioDecoder> decoder_;
@@ -138,7 +142,8 @@
static const uint8_t kRtpPayloadTypeError = 0xFF;
DecoderDatabase(
- const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory);
+ const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory,
+ rtc::Optional<AudioCodecPairId> codec_pair_id);
virtual ~DecoderDatabase();
@@ -242,6 +247,7 @@
int active_cng_decoder_type_;
mutable std::unique_ptr<ComfortNoiseDecoder> active_cng_decoder_;
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
+ const rtc::Optional<AudioCodecPairId> codec_pair_id_;
RTC_DISALLOW_COPY_AND_ASSIGN(DecoderDatabase);
};
diff --git a/modules/audio_coding/neteq/decoder_database_unittest.cc b/modules/audio_coding/neteq/decoder_database_unittest.cc
index 7f9b38e..a6b9689 100644
--- a/modules/audio_coding/neteq/decoder_database_unittest.cc
+++ b/modules/audio_coding/neteq/decoder_database_unittest.cc
@@ -28,7 +28,8 @@
namespace webrtc {
TEST(DecoderDatabase, CreateAndDestroy) {
- DecoderDatabase db(new rtc::RefCountedObject<MockAudioDecoderFactory>);
+ DecoderDatabase db(new rtc::RefCountedObject<MockAudioDecoderFactory>,
+ rtc::nullopt);
EXPECT_EQ(0, db.Size());
EXPECT_TRUE(db.Empty());
}
@@ -41,7 +42,7 @@
EXPECT_EQ("pcmu", format.name);
return true;
}));
- DecoderDatabase db(factory);
+ DecoderDatabase db(factory, rtc::nullopt);
const uint8_t kPayloadType = 0;
const std::string kCodecName = "Robert\'); DROP TABLE Students;";
EXPECT_EQ(
@@ -66,7 +67,7 @@
EXPECT_EQ("pcma", format.name);
return true;
}));
- DecoderDatabase db(factory);
+ DecoderDatabase db(factory, rtc::nullopt);
const std::string kCodecName1 = "Robert\'); DROP TABLE Students;";
const std::string kCodecName2 = "https://xkcd.com/327/";
EXPECT_EQ(DecoderDatabase::kOK,
@@ -96,7 +97,7 @@
EXPECT_EQ("pcmu", format.name);
dec->reset(decoder);
}));
- DecoderDatabase db(factory);
+ DecoderDatabase db(factory, rtc::nullopt);
const uint8_t kPayloadType = 0;
const std::string kCodecName = "Robert\'); DROP TABLE Students;";
EXPECT_EQ(
@@ -113,7 +114,7 @@
}
TEST(DecoderDatabase, GetDecoder) {
- DecoderDatabase db(CreateBuiltinAudioDecoderFactory());
+ DecoderDatabase db(CreateBuiltinAudioDecoderFactory(), rtc::nullopt);
const uint8_t kPayloadType = 0;
const std::string kCodecName = "Robert\'); DROP TABLE Students;";
EXPECT_EQ(DecoderDatabase::kOK,
@@ -131,7 +132,7 @@
EXPECT_EQ("pcmu", format.name);
return true;
}));
- DecoderDatabase db(factory);
+ DecoderDatabase db(factory, rtc::nullopt);
const uint8_t kPayloadTypePcmU = 0;
const uint8_t kPayloadTypeCng = 13;
const uint8_t kPayloadTypeDtmf = 100;
@@ -166,7 +167,8 @@
}
TEST(DecoderDatabase, ExternalDecoder) {
- DecoderDatabase db(new rtc::RefCountedObject<MockAudioDecoderFactory>);
+ DecoderDatabase db(new rtc::RefCountedObject<MockAudioDecoderFactory>,
+ rtc::nullopt);
const uint8_t kPayloadType = 0;
const std::string kCodecName = "Robert\'); DROP TABLE Students;";
MockAudioDecoder decoder;
@@ -203,7 +205,7 @@
EXPECT_EQ("pcmu", format.name);
return true;
}));
- DecoderDatabase db(factory);
+ DecoderDatabase db(factory, rtc::nullopt);
// Load a number of payloads into the database. Payload types are 0, 1, ...,
// while the decoder type is the same for all payload types (this does not
// matter for the test).
@@ -243,7 +245,7 @@
// Test the methods for setting and getting active speech and CNG decoders.
TEST(DecoderDatabase, IF_ISAC(ActiveDecoders)) {
- DecoderDatabase db(CreateBuiltinAudioDecoderFactory());
+ DecoderDatabase db(CreateBuiltinAudioDecoderFactory(), rtc::nullopt);
// Load payload types.
ASSERT_EQ(DecoderDatabase::kOK,
db.RegisterPayload(0, NetEqDecoder::kDecoderPCMu, "pcmu"));
diff --git a/modules/audio_coding/neteq/expand.cc b/modules/audio_coding/neteq/expand.cc
index 3fb09ea..03bcc77 100644
--- a/modules/audio_coding/neteq/expand.cc
+++ b/modules/audio_coding/neteq/expand.cc
@@ -911,46 +911,10 @@
// Unmute the background noise.
int16_t bgn_mute_factor = background_noise_->MuteFactor(channel);
- NetEq::BackgroundNoiseMode bgn_mode = background_noise_->mode();
- if (bgn_mode == NetEq::kBgnFade && too_many_expands &&
- bgn_mute_factor > 0) {
- // Fade BGN to zero.
- // Calculate muting slope, approximately -2^18 / fs_hz.
- int mute_slope;
- if (fs_hz_ == 8000) {
- mute_slope = -32;
- } else if (fs_hz_ == 16000) {
- mute_slope = -16;
- } else if (fs_hz_ == 32000) {
- mute_slope = -8;
- } else {
- mute_slope = -5;
- }
- // Use UnmuteSignal function with negative slope.
- // |bgn_mute_factor| is in Q14. |mute_slope| is in Q20.
- DspHelper::UnmuteSignal(noise_samples,
- num_noise_samples,
- &bgn_mute_factor,
- mute_slope,
- noise_samples);
- } else if (bgn_mute_factor < 16384) {
- // If mode is kBgnOn, or if kBgnFade has started fading,
- // use regular |mute_slope|.
- if (!stop_muting_ && bgn_mode != NetEq::kBgnOff &&
- !(bgn_mode == NetEq::kBgnFade && too_many_expands)) {
- DspHelper::UnmuteSignal(noise_samples,
- static_cast<int>(num_noise_samples),
- &bgn_mute_factor,
- mute_slope,
- noise_samples);
- } else {
- // kBgnOn and stop muting, or
- // kBgnOff (mute factor is always 0), or
- // kBgnFade has reached 0.
- WebRtcSpl_AffineTransformVector(noise_samples, noise_samples,
- bgn_mute_factor, 8192, 14,
- num_noise_samples);
- }
+ if (bgn_mute_factor < 16384) {
+ WebRtcSpl_AffineTransformVector(noise_samples, noise_samples,
+ bgn_mute_factor, 8192, 14,
+ num_noise_samples);
}
// Update mute_factor in BackgroundNoise class.
background_noise_->SetMuteFactor(channel, bgn_mute_factor);
diff --git a/modules/audio_coding/neteq/expand_uma_logger.cc b/modules/audio_coding/neteq/expand_uma_logger.cc
new file mode 100644
index 0000000..c656eed
--- /dev/null
+++ b/modules/audio_coding/neteq/expand_uma_logger.cc
@@ -0,0 +1,69 @@
+/* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_coding/neteq/expand_uma_logger.h"
+#include "rtc_base/checks.h"
+#include "system_wrappers/include/metrics.h"
+
+namespace webrtc {
+namespace {
+std::unique_ptr<TickTimer::Countdown> GetNewCountdown(
+ const TickTimer& tick_timer,
+ int logging_period_s) {
+ return tick_timer.GetNewCountdown((logging_period_s * 1000) /
+ tick_timer.ms_per_tick());
+}
+} // namespace
+
+ExpandUmaLogger::ExpandUmaLogger(std::string uma_name,
+ int logging_period_s,
+ const TickTimer* tick_timer)
+ : uma_name_(uma_name),
+ logging_period_s_(logging_period_s),
+ tick_timer_(*tick_timer),
+ timer_(GetNewCountdown(tick_timer_, logging_period_s_)) {
+ RTC_DCHECK(tick_timer);
+ RTC_DCHECK_GT(logging_period_s_, 0);
+}
+
+ExpandUmaLogger::~ExpandUmaLogger() = default;
+
+void ExpandUmaLogger::UpdateSampleCounter(uint64_t samples,
+ int sample_rate_hz) {
+ if ((last_logged_value_ && *last_logged_value_ > samples) ||
+ sample_rate_hz_ != sample_rate_hz) {
+ // Sanity checks. The incremental counter moved backwards, or sample rate
+ // changed.
+ last_logged_value_.reset();
+ }
+ last_value_ = samples;
+ sample_rate_hz_ = sample_rate_hz;
+ if (!last_logged_value_) {
+ last_logged_value_ = rtc::Optional<uint64_t>(samples);
+ }
+
+ if (!timer_->Finished()) {
+ // Not yet time to log.
+ return;
+ }
+
+ RTC_DCHECK(last_logged_value_);
+ RTC_DCHECK_GE(last_value_, *last_logged_value_);
+ const uint64_t diff = last_value_ - *last_logged_value_;
+ last_logged_value_ = rtc::Optional<uint64_t>(last_value_);
+ // Calculate rate in percent.
+ RTC_DCHECK_GT(sample_rate_hz, 0);
+ const int rate = (100 * diff) / (sample_rate_hz * logging_period_s_);
+ RTC_DCHECK_GE(rate, 0);
+ RTC_DCHECK_LE(rate, 100);
+ RTC_HISTOGRAM_PERCENTAGE_SPARSE(uma_name_, rate);
+ timer_ = GetNewCountdown(tick_timer_, logging_period_s_);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_coding/neteq/expand_uma_logger.h b/modules/audio_coding/neteq/expand_uma_logger.h
new file mode 100644
index 0000000..70af39b
--- /dev/null
+++ b/modules/audio_coding/neteq/expand_uma_logger.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_CODING_NETEQ_EXPAND_UMA_LOGGER_H_
+#define MODULES_AUDIO_CODING_NETEQ_EXPAND_UMA_LOGGER_H_
+
+#include <memory>
+#include <string>
+
+#include "api/optional.h"
+#include "modules/audio_coding/neteq/tick_timer.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+// This class is used to periodically log values to a UMA histogram. The caller
+// is expected to update this class with an incremental sample counter which
+// counts expand samples. At the end of each logging period, the class will
+// calculate the fraction of samples that were expand samples during that period
+// and report that in percent. The logging period must be strictly positive.
+// Does not take ownership of tick_timer and the pointer must refer to a valid
+// object that outlives the one constructed.
+class ExpandUmaLogger {
+ public:
+ ExpandUmaLogger(std::string uma_name,
+ int logging_period_s,
+ const TickTimer* tick_timer);
+
+ ~ExpandUmaLogger();
+
+ // In this call, value should be an incremental sample counter. The sample
+ // rate must be strictly positive.
+ void UpdateSampleCounter(uint64_t value, int sample_rate_hz);
+
+ private:
+ const std::string uma_name_;
+ const int logging_period_s_;
+ const TickTimer& tick_timer_;
+ std::unique_ptr<TickTimer::Countdown> timer_;
+ rtc::Optional<uint64_t> last_logged_value_;
+ uint64_t last_value_ = 0;
+ int sample_rate_hz_ = 0;
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(ExpandUmaLogger);
+};
+
+} // namespace webrtc
+#endif // MODULES_AUDIO_CODING_NETEQ_EXPAND_UMA_LOGGER_H_
diff --git a/modules/audio_coding/neteq/include/neteq.h b/modules/audio_coding/neteq/include/neteq.h
index f5bd8cd..310a227 100644
--- a/modules/audio_coding/neteq/include/neteq.h
+++ b/modules/audio_coding/neteq/include/neteq.h
@@ -16,6 +16,7 @@
#include <string>
#include <vector>
+#include "api/audio_codecs/audio_codec_pair_id.h"
#include "api/audio_codecs/audio_decoder.h"
#include "api/optional.h"
#include "api/rtp_headers.h"
@@ -83,33 +84,24 @@
// This is the interface class for NetEq.
class NetEq {
public:
- enum BackgroundNoiseMode {
- kBgnOn, // Default behavior with eternal noise.
- kBgnFade, // Noise fades to zero after some time.
- kBgnOff // Background noise is always zero.
- };
-
struct Config {
- Config()
- : sample_rate_hz(16000),
- enable_post_decode_vad(false),
- max_packets_in_buffer(50),
- // |max_delay_ms| has the same effect as calling SetMaximumDelay().
- max_delay_ms(2000),
- background_noise_mode(kBgnOff),
- playout_mode(kPlayoutOn),
- enable_fast_accelerate(false) {}
+ Config();
+ Config(const Config&);
+ Config(Config&&);
+ ~Config();
+ Config& operator=(const Config&);
+ Config& operator=(Config&&);
std::string ToString() const;
- int sample_rate_hz; // Initial value. Will change with input data.
- bool enable_post_decode_vad;
- size_t max_packets_in_buffer;
- int max_delay_ms;
- BackgroundNoiseMode background_noise_mode;
- NetEqPlayoutMode playout_mode;
- bool enable_fast_accelerate;
+ int sample_rate_hz = 16000; // Initial value. Will change with input data.
+ bool enable_post_decode_vad = false;
+ size_t max_packets_in_buffer = 50;
+ int max_delay_ms = 2000;
+ NetEqPlayoutMode playout_mode = kPlayoutOn;
+ bool enable_fast_accelerate = false;
bool enable_muted_state = false;
+ rtc::Optional<AudioCodecPairId> codec_pair_id;
};
enum ReturnCodes {
diff --git a/modules/audio_coding/neteq/mock/mock_decoder_database.h b/modules/audio_coding/neteq/mock/mock_decoder_database.h
index 049b693..a4240ce 100644
--- a/modules/audio_coding/neteq/mock/mock_decoder_database.h
+++ b/modules/audio_coding/neteq/mock/mock_decoder_database.h
@@ -23,7 +23,7 @@
public:
explicit MockDecoderDatabase(
rtc::scoped_refptr<AudioDecoderFactory> factory = nullptr)
- : DecoderDatabase(factory) {}
+ : DecoderDatabase(factory, rtc::nullopt) {}
virtual ~MockDecoderDatabase() { Die(); }
MOCK_METHOD0(Die, void());
MOCK_CONST_METHOD0(Empty,
diff --git a/modules/audio_coding/neteq/nack_tracker.cc b/modules/audio_coding/neteq/nack_tracker.cc
index d187883..c62cdf8 100644
--- a/modules/audio_coding/neteq/nack_tracker.cc
+++ b/modules/audio_coding/neteq/nack_tracker.cc
@@ -14,7 +14,6 @@
#include <algorithm> // For std::max.
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
namespace webrtc {
diff --git a/modules/audio_coding/neteq/nack_tracker.h b/modules/audio_coding/neteq/nack_tracker.h
index 4f88d91..66383ce 100644
--- a/modules/audio_coding/neteq/nack_tracker.h
+++ b/modules/audio_coding/neteq/nack_tracker.h
@@ -15,6 +15,7 @@
#include <map>
#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
+#include "modules/include/module_common_types.h"
#include "rtc_base/gtest_prod_util.h"
//
diff --git a/modules/audio_coding/neteq/neteq.cc b/modules/audio_coding/neteq/neteq.cc
index 8b74973..db12589 100644
--- a/modules/audio_coding/neteq/neteq.cc
+++ b/modules/audio_coding/neteq/neteq.cc
@@ -11,19 +11,26 @@
#include "modules/audio_coding/neteq/include/neteq.h"
#include <memory>
-#include <sstream>
#include "modules/audio_coding/neteq/neteq_impl.h"
+#include "rtc_base/strings/string_builder.h"
namespace webrtc {
+NetEq::Config::Config() = default;
+NetEq::Config::Config(const Config&) = default;
+NetEq::Config::Config(Config&&) = default;
+NetEq::Config::~Config() = default;
+NetEq::Config& NetEq::Config::operator=(const Config&) = default;
+NetEq::Config& NetEq::Config::operator=(Config&&) = default;
+
std::string NetEq::Config::ToString() const {
- std::stringstream ss;
+ char buf[1024];
+ rtc::SimpleStringBuilder ss(buf);
ss << "sample_rate_hz=" << sample_rate_hz
<< ", enable_post_decode_vad="
<< (enable_post_decode_vad ? "true" : "false")
<< ", max_packets_in_buffer=" << max_packets_in_buffer
- << ", background_noise_mode=" << background_noise_mode
<< ", playout_mode=" << playout_mode
<< ", enable_fast_accelerate="
<< (enable_fast_accelerate ? " true": "false")
diff --git a/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc b/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc
index ec16627..03f5aa3 100644
--- a/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc
+++ b/modules/audio_coding/neteq/neteq_external_decoder_unittest.cc
@@ -12,13 +12,13 @@
#include <memory>
+#include "api/audio/audio_frame.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/neteq/mock/mock_external_decoder_pcm16b.h"
#include "modules/audio_coding/neteq/tools/input_audio_file.h"
#include "modules/audio_coding/neteq/tools/neteq_external_decoder_test.h"
#include "modules/audio_coding/neteq/tools/rtp_generator.h"
-#include "modules/include/module_common_types.h"
#include "test/gmock.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_coding/neteq/neteq_impl.cc b/modules/audio_coding/neteq/neteq_impl.cc
index b107626..80bfdaf 100644
--- a/modules/audio_coding/neteq/neteq_impl.cc
+++ b/modules/audio_coding/neteq/neteq_impl.cc
@@ -41,11 +41,11 @@
#include "modules/audio_coding/neteq/sync_buffer.h"
#include "modules/audio_coding/neteq/tick_timer.h"
#include "modules/audio_coding/neteq/timestamp_scaler.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/sanitizer.h"
+#include "rtc_base/strings/audio_format_to_string.h"
#include "rtc_base/system/fallthrough.h"
#include "rtc_base/trace_event.h"
#include "system_wrappers/include/field_trial.h"
@@ -57,7 +57,8 @@
const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory)
: tick_timer(new TickTimer),
buffer_level_filter(new BufferLevelFilter),
- decoder_database(new DecoderDatabase(decoder_factory)),
+ decoder_database(
+ new DecoderDatabase(decoder_factory, config.codec_pair_id)),
delay_peak_detector(new DelayPeakDetector(tick_timer.get())),
delay_manager(new DelayManager(config.max_packets_in_buffer,
delay_peak_detector.get(),
@@ -100,11 +101,16 @@
reset_decoder_(false),
ssrc_(0),
first_packet_(true),
- background_noise_mode_(config.background_noise_mode),
playout_mode_(config.playout_mode),
enable_fast_accelerate_(config.enable_fast_accelerate),
nack_enabled_(false),
- enable_muted_state_(config.enable_muted_state) {
+ enable_muted_state_(config.enable_muted_state),
+ expand_uma_logger_("WebRTC.Audio.ExpandRatePercent",
+ 10, // Report once every 10 s.
+ tick_timer_.get()),
+ speech_expand_uma_logger_("WebRTC.Audio.SpeechExpandRatePercent",
+ 10, // Report once every 10 s.
+ tick_timer_.get()) {
RTC_LOG(LS_INFO) << "NetEq config: " << config.ToString();
int fs = config.sample_rate_hz;
if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) {
@@ -261,7 +267,8 @@
bool NetEqImpl::RegisterPayloadType(int rtp_payload_type,
const SdpAudioFormat& audio_format) {
RTC_LOG(LS_VERBOSE) << "NetEqImpl::RegisterPayloadType: payload type "
- << rtp_payload_type << ", codec " << audio_format;
+ << rtp_payload_type << ", codec "
+ << rtc::ToString(audio_format);
rtc::CritScope lock(&crit_sect_);
return decoder_database_->RegisterPayload(rtp_payload_type, audio_format) ==
DecoderDatabase::kOK;
@@ -834,6 +841,11 @@
last_decoded_timestamps_.clear();
tick_timer_->Increment();
stats_.IncreaseCounter(output_size_samples_, fs_hz_);
+ const auto lifetime_stats = stats_.GetLifetimeStatistics();
+ expand_uma_logger_.UpdateSampleCounter(lifetime_stats.concealed_samples,
+ fs_hz_);
+ speech_expand_uma_logger_.UpdateSampleCounter(
+ lifetime_stats.voice_concealed_samples, fs_hz_);
// Check for muted state.
if (enable_muted_state_ && expand_->Muted() && packet_buffer_->Empty()) {
@@ -2075,7 +2087,6 @@
// Delete BackgroundNoise object and create a new one.
background_noise_.reset(new BackgroundNoise(channels));
- background_noise_->set_mode(background_noise_mode_);
// Reset random vector.
random_vector_.Reset();
diff --git a/modules/audio_coding/neteq/neteq_impl.h b/modules/audio_coding/neteq/neteq_impl.h
index bdeb020..ce75ce0 100644
--- a/modules/audio_coding/neteq/neteq_impl.h
+++ b/modules/audio_coding/neteq/neteq_impl.h
@@ -15,15 +15,16 @@
#include <string>
#include "api/optional.h"
+#include "api/audio/audio_frame.h"
#include "modules/audio_coding/neteq/audio_multi_vector.h"
#include "modules/audio_coding/neteq/defines.h"
+#include "modules/audio_coding/neteq/expand_uma_logger.h"
#include "modules/audio_coding/neteq/include/neteq.h"
#include "modules/audio_coding/neteq/packet.h" // Declare PacketList.
#include "modules/audio_coding/neteq/random_vector.h"
#include "modules/audio_coding/neteq/rtcp.h"
#include "modules/audio_coding/neteq/statistics_calculator.h"
#include "modules/audio_coding/neteq/tick_timer.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/thread_annotations.h"
@@ -429,7 +430,6 @@
RTC_GUARDED_BY(crit_sect_);
uint32_t ssrc_ RTC_GUARDED_BY(crit_sect_);
bool first_packet_ RTC_GUARDED_BY(crit_sect_);
- const BackgroundNoiseMode background_noise_mode_ RTC_GUARDED_BY(crit_sect_);
NetEqPlayoutMode playout_mode_ RTC_GUARDED_BY(crit_sect_);
bool enable_fast_accelerate_ RTC_GUARDED_BY(crit_sect_);
std::unique_ptr<NackTracker> nack_ RTC_GUARDED_BY(crit_sect_);
@@ -440,6 +440,8 @@
std::unique_ptr<TickTimer::Stopwatch> generated_noise_stopwatch_
RTC_GUARDED_BY(crit_sect_);
std::vector<uint32_t> last_decoded_timestamps_ RTC_GUARDED_BY(crit_sect_);
+ ExpandUmaLogger expand_uma_logger_ RTC_GUARDED_BY(crit_sect_);
+ ExpandUmaLogger speech_expand_uma_logger_ RTC_GUARDED_BY(crit_sect_);
private:
RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl);
diff --git a/modules/audio_coding/neteq/neteq_impl_unittest.cc b/modules/audio_coding/neteq/neteq_impl_unittest.cc
index 12eabfa..de24cda 100644
--- a/modules/audio_coding/neteq/neteq_impl_unittest.cc
+++ b/modules/audio_coding/neteq/neteq_impl_unittest.cc
@@ -27,7 +27,6 @@
#include "modules/audio_coding/neteq/preemptive_expand.h"
#include "modules/audio_coding/neteq/sync_buffer.h"
#include "modules/audio_coding/neteq/timestamp_scaler.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -335,7 +334,7 @@
*dec = std::move(mock_decoder);
}));
- DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu,
+ DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, rtc::nullopt,
mock_decoder_factory);
// Expectations for decoder database.
diff --git a/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
index 334715f..b317099 100644
--- a/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
+++ b/modules/audio_coding/neteq/neteq_network_stats_unittest.cc
@@ -10,10 +10,10 @@
#include <memory>
+#include "api/audio/audio_frame.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/neteq/tools/neteq_external_decoder_test.h"
#include "modules/audio_coding/neteq/tools/rtp_generator.h"
-#include "modules/include/module_common_types.h"
#include "test/gmock.h"
namespace webrtc {
diff --git a/modules/audio_coding/neteq/neteq_stereo_unittest.cc b/modules/audio_coding/neteq/neteq_stereo_unittest.cc
index 1bef9c8..49facdd 100644
--- a/modules/audio_coding/neteq/neteq_stereo_unittest.cc
+++ b/modules/audio_coding/neteq/neteq_stereo_unittest.cc
@@ -15,13 +15,13 @@
#include <string>
#include <list>
+#include "api/audio/audio_frame.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/codecs/pcm16b/pcm16b.h"
#include "modules/audio_coding/neteq/include/neteq.h"
#include "modules/audio_coding/neteq/tools/input_audio_file.h"
#include "modules/audio_coding/neteq/tools/rtp_generator.h"
-#include "modules/include/module_common_types.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_coding/neteq/neteq_unittest.cc b/modules/audio_coding/neteq/neteq_unittest.cc
index ca93cf5..4cd3014 100644
--- a/modules/audio_coding/neteq/neteq_unittest.cc
+++ b/modules/audio_coding/neteq/neteq_unittest.cc
@@ -20,12 +20,12 @@
#include <string>
#include <vector>
+#include "api/audio/audio_frame.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/codecs/pcm16b/pcm16b.h"
#include "modules/audio_coding/neteq/tools/audio_loop.h"
#include "modules/audio_coding/neteq/tools/rtp_file_source.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/ignore_wundef.h"
#include "rtc_base/messagedigest.h"
#include "rtc_base/numerics/safe_conversions.h"
@@ -950,9 +950,6 @@
class NetEqBgnTest : public NetEqDecodingTest {
protected:
- virtual void TestCondition(double sum_squared_noise,
- bool should_be_faded) = 0;
-
void CheckBgn(int sampling_rate_hz) {
size_t expected_samples_per_channel = 0;
uint8_t payload_type = 0xFF; // Invalid.
@@ -1044,7 +1041,7 @@
for (size_t k = 0;
k < output.num_channels_ * output.samples_per_channel_; ++k)
sum_squared += output_data[k] * output_data[k];
- TestCondition(sum_squared, n > kFadingThreshold);
+ EXPECT_EQ(0, sum_squared);
} else {
EXPECT_EQ(AudioFrame::kPLC, output.speech_type_);
}
@@ -1053,53 +1050,7 @@
}
};
-class NetEqBgnTestOn : public NetEqBgnTest {
- protected:
- NetEqBgnTestOn() : NetEqBgnTest() {
- config_.background_noise_mode = NetEq::kBgnOn;
- }
-
- void TestCondition(double sum_squared_noise, bool /*should_be_faded*/) {
- EXPECT_NE(0, sum_squared_noise);
- }
-};
-
-class NetEqBgnTestOff : public NetEqBgnTest {
- protected:
- NetEqBgnTestOff() : NetEqBgnTest() {
- config_.background_noise_mode = NetEq::kBgnOff;
- }
-
- void TestCondition(double sum_squared_noise, bool /*should_be_faded*/) {
- EXPECT_EQ(0, sum_squared_noise);
- }
-};
-
-class NetEqBgnTestFade : public NetEqBgnTest {
- protected:
- NetEqBgnTestFade() : NetEqBgnTest() {
- config_.background_noise_mode = NetEq::kBgnFade;
- }
-
- void TestCondition(double sum_squared_noise, bool should_be_faded) {
- if (should_be_faded)
- EXPECT_EQ(0, sum_squared_noise);
- }
-};
-
-TEST_F(NetEqBgnTestOn, RunTest) {
- CheckBgn(8000);
- CheckBgn(16000);
- CheckBgn(32000);
-}
-
-TEST_F(NetEqBgnTestOff, RunTest) {
- CheckBgn(8000);
- CheckBgn(16000);
- CheckBgn(32000);
-}
-
-TEST_F(NetEqBgnTestFade, RunTest) {
+TEST_F(NetEqBgnTest, RunTest) {
CheckBgn(8000);
CheckBgn(16000);
CheckBgn(32000);
diff --git a/modules/audio_coding/neteq/packet_buffer_unittest.cc b/modules/audio_coding/neteq/packet_buffer_unittest.cc
index 0ddeb8a..1aaed8b 100644
--- a/modules/audio_coding/neteq/packet_buffer_unittest.cc
+++ b/modules/audio_coding/neteq/packet_buffer_unittest.cc
@@ -175,7 +175,8 @@
MockDecoderDatabase decoder_database;
auto factory = CreateBuiltinAudioDecoderFactory();
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu,
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(0))
.WillRepeatedly(Return(&info));
@@ -219,10 +220,12 @@
MockDecoderDatabase decoder_database;
auto factory = CreateBuiltinAudioDecoderFactory();
- const DecoderDatabase::DecoderInfo info0(NetEqDecoder::kDecoderPCMu, factory);
+ const DecoderDatabase::DecoderInfo info0(NetEqDecoder::kDecoderPCMu,
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(0))
.WillRepeatedly(Return(&info0));
- const DecoderDatabase::DecoderInfo info1(NetEqDecoder::kDecoderPCMa, factory);
+ const DecoderDatabase::DecoderInfo info1(NetEqDecoder::kDecoderPCMa,
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(1))
.WillRepeatedly(Return(&info1));
@@ -404,7 +407,8 @@
MockDecoderDatabase decoder_database;
auto factory = CreateBuiltinAudioDecoderFactory();
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu,
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(0))
.WillRepeatedly(Return(&info));
rtc::Optional<uint8_t> current_pt;
@@ -444,11 +448,11 @@
MockDecoderDatabase decoder_database;
auto factory = CreateBuiltinAudioDecoderFactory();
const DecoderDatabase::DecoderInfo info_cng(NetEqDecoder::kDecoderCNGnb,
- factory);
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(kCngPt))
.WillRepeatedly(Return(&info_cng));
const DecoderDatabase::DecoderInfo info_speech(NetEqDecoder::kDecoderPCM16Bwb,
- factory);
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(kSpeechPt))
.WillRepeatedly(Return(&info_speech));
@@ -545,7 +549,8 @@
list.push_back(gen.NextPacket(payload_len)); // Valid packet.
MockDecoderDatabase decoder_database;
auto factory = CreateBuiltinAudioDecoderFactory();
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu,
+ rtc::nullopt, factory);
EXPECT_CALL(decoder_database, GetDecoderInfo(0))
.WillRepeatedly(Return(&info));
rtc::Optional<uint8_t> current_pt;
diff --git a/modules/audio_coding/neteq/red_payload_splitter_unittest.cc b/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
index 4f511ad..077c8ea 100644
--- a/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
+++ b/modules/audio_coding/neteq/red_payload_splitter_unittest.cc
@@ -299,7 +299,7 @@
// easier to just register the payload types and let the actual implementation
// do its job.
DecoderDatabase decoder_database(
- new rtc::RefCountedObject<MockAudioDecoderFactory>);
+ new rtc::RefCountedObject<MockAudioDecoderFactory>, rtc::nullopt);
decoder_database.RegisterPayload(0, NetEqDecoder::kDecoderCNGnb, "cng-nb");
decoder_database.RegisterPayload(1, NetEqDecoder::kDecoderPCMu, "pcmu");
decoder_database.RegisterPayload(2, NetEqDecoder::kDecoderAVT, "avt");
diff --git a/modules/audio_coding/neteq/rtcp.cc b/modules/audio_coding/neteq/rtcp.cc
index 2885398..551eb5f 100644
--- a/modules/audio_coding/neteq/rtcp.cc
+++ b/modules/audio_coding/neteq/rtcp.cc
@@ -15,8 +15,6 @@
#include <algorithm>
-#include "modules/include/module_common_types.h"
-
namespace webrtc {
void Rtcp::Init(uint16_t start_sequence_number) {
diff --git a/modules/audio_coding/neteq/sync_buffer.h b/modules/audio_coding/neteq/sync_buffer.h
index ab9ff52..d880356 100644
--- a/modules/audio_coding/neteq/sync_buffer.h
+++ b/modules/audio_coding/neteq/sync_buffer.h
@@ -11,8 +11,8 @@
#ifndef MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_
#define MODULES_AUDIO_CODING_NETEQ_SYNC_BUFFER_H_
+#include "api/audio/audio_frame.h"
#include "modules/audio_coding/neteq/audio_multi_vector.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/constructormagic.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_coding/neteq/timestamp_scaler_unittest.cc b/modules/audio_coding/neteq/timestamp_scaler_unittest.cc
index b3c1bb0..1a7b71a 100644
--- a/modules/audio_coding/neteq/timestamp_scaler_unittest.cc
+++ b/modules/audio_coding/neteq/timestamp_scaler_unittest.cc
@@ -25,7 +25,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use PCMu, because it doesn't use scaled timestamps.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 0;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -46,7 +47,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use PCMu, because it doesn't use scaled timestamps.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderPCMu,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 0;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -72,7 +74,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use G722, which has a factor 2 scaling.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 17;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -97,7 +100,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use G722, which has a factor 2 scaling.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 17;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -127,9 +131,9 @@
auto factory = CreateBuiltinAudioDecoderFactory();
// Use G722, which has a factor 2 scaling.
const DecoderDatabase::DecoderInfo info_g722(NetEqDecoder::kDecoderG722,
- factory);
+ rtc::nullopt, factory);
const DecoderDatabase::DecoderInfo info_cng(NetEqDecoder::kDecoderCNGwb,
- factory);
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadTypeG722 = 17;
static const uint8_t kRtpPayloadTypeCng = 13;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadTypeG722))
@@ -170,7 +174,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use G722, which has a factor 2 scaling.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 17;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -199,7 +204,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use G722, which has a factor 2 scaling.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 17;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -232,7 +238,8 @@
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
// Use G722, which has a factor 2 scaling.
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderG722,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 17;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
@@ -272,7 +279,8 @@
TEST(TimestampScaler, TestOpusLargeStep) {
MockDecoderDatabase db;
auto factory = CreateBuiltinAudioDecoderFactory();
- const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderOpus, factory);
+ const DecoderDatabase::DecoderInfo info(NetEqDecoder::kDecoderOpus,
+ rtc::nullopt, factory);
static const uint8_t kRtpPayloadType = 17;
EXPECT_CALL(db, GetDecoderInfo(kRtpPayloadType))
.WillRepeatedly(Return(&info));
diff --git a/modules/audio_coding/neteq/tools/audio_sink.h b/modules/audio_coding/neteq/tools/audio_sink.h
index ecec51b..18ac6fc 100644
--- a/modules/audio_coding/neteq/tools/audio_sink.h
+++ b/modules/audio_coding/neteq/tools/audio_sink.h
@@ -11,7 +11,7 @@
#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_
#define MODULES_AUDIO_CODING_NETEQ_TOOLS_AUDIO_SINK_H_
-#include "modules/include/module_common_types.h"
+#include "api/audio/audio_frame.h"
#include "rtc_base/constructormagic.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_coding/neteq/tools/encode_neteq_input.h b/modules/audio_coding/neteq/tools/encode_neteq_input.h
index b44d4ac..13b39b3 100644
--- a/modules/audio_coding/neteq/tools/encode_neteq_input.h
+++ b/modules/audio_coding/neteq/tools/encode_neteq_input.h
@@ -15,7 +15,6 @@
#include "api/audio_codecs/audio_encoder.h"
#include "modules/audio_coding/neteq/tools/neteq_input.h"
-#include "modules/include/module_common_types.h"
namespace webrtc {
namespace test {
diff --git a/modules/audio_coding/neteq/tools/input_audio_file.cc b/modules/audio_coding/neteq/tools/input_audio_file.cc
index 31ebf98..330a874 100644
--- a/modules/audio_coding/neteq/tools/input_audio_file.cc
+++ b/modules/audio_coding/neteq/tools/input_audio_file.cc
@@ -56,13 +56,18 @@
RTC_CHECK_NE(EOF, file_size) << "Error returned when getting file position.";
// Find new position.
long new_pos = current_pos + sizeof(int16_t) * samples; // Samples to bytes.
- RTC_CHECK_GE(new_pos, 0)
- << "Trying to move to before the beginning of the file";
if (loop_at_end_) {
new_pos = new_pos % file_size; // Wrap around the end of the file.
+ if (new_pos < 0) {
+ // For negative values of new_pos, newpos % file_size will also be
+ // negative. To get the correct result it's needed to add file_size.
+ new_pos += file_size;
+ }
} else {
new_pos = new_pos > file_size ? file_size : new_pos; // Don't loop.
}
+ RTC_CHECK_GE(new_pos, 0)
+ << "Trying to move to before the beginning of the file";
// Move to new position relative to the beginning of the file.
RTC_CHECK_EQ(0, fseek(fp_, new_pos, SEEK_SET));
return true;
diff --git a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc
index 882f823..ba0b217 100644
--- a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc
+++ b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc
@@ -17,6 +17,7 @@
#include <limits>
#include <utility>
+#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
namespace webrtc {
diff --git a/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc b/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc
index 68dde52..2c23e5c 100644
--- a/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc
+++ b/modules/audio_coding/neteq/tools/neteq_external_decoder_test.cc
@@ -11,6 +11,7 @@
#include "modules/audio_coding/neteq/tools/neteq_external_decoder_test.h"
+#include "api/audio/audio_frame.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "rtc_base/format_macros.h"
#include "test/gtest.h"
diff --git a/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h b/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h
index aefa62e..b8670a3 100644
--- a/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h
+++ b/modules/audio_coding/neteq/tools/neteq_external_decoder_test.h
@@ -17,7 +17,6 @@
#include "api/audio_codecs/audio_decoder.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/neteq/include/neteq.h"
-#include "modules/include/module_common_types.h"
namespace webrtc {
namespace test {
diff --git a/modules/audio_coding/neteq/tools/neteq_performance_test.cc b/modules/audio_coding/neteq/tools/neteq_performance_test.cc
index 27ecdf4..80aa809 100644
--- a/modules/audio_coding/neteq/tools/neteq_performance_test.cc
+++ b/modules/audio_coding/neteq/tools/neteq_performance_test.cc
@@ -10,13 +10,13 @@
#include "modules/audio_coding/neteq/tools/neteq_performance_test.h"
+#include "api/audio/audio_frame.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/audio_coding/codecs/pcm16b/pcm16b.h"
#include "modules/audio_coding/neteq/include/neteq.h"
#include "modules/audio_coding/neteq/tools/audio_loop.h"
#include "modules/audio_coding/neteq/tools/rtp_generator.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "system_wrappers/include/clock.h"
#include "test/testsupport/fileutils.h"
@@ -103,7 +103,7 @@
return -1;
payload_len = WebRtcPcm16b_Encode(input_samples.data(),
input_samples.size(), input_payload);
- assert(payload_len == kInputBlockSizeSamples * sizeof(int16_t));
+ RTC_DCHECK_EQ(payload_len, kInputBlockSizeSamples * sizeof(int16_t));
}
// Get output audio, but don't do anything with it.
@@ -113,8 +113,7 @@
if (error != NetEq::kOK)
return -1;
- assert(out_frame.samples_per_channel_ ==
- static_cast<size_t>(kSampRateHz * 10 / 1000));
+ RTC_DCHECK_EQ(out_frame.samples_per_channel_, (kSampRateHz * 10) / 1000);
static const int kOutputBlockSizeMs = 10;
time_now_ms += kOutputBlockSizeMs;
diff --git a/modules/audio_coding/neteq/tools/neteq_quality_test.h b/modules/audio_coding/neteq/tools/neteq_quality_test.h
index 531a080..2b82b0a 100644
--- a/modules/audio_coding/neteq/tools/neteq_quality_test.h
+++ b/modules/audio_coding/neteq/tools/neteq_quality_test.h
@@ -19,7 +19,6 @@
#include "modules/audio_coding/neteq/tools/audio_sink.h"
#include "modules/audio_coding/neteq/tools/input_audio_file.h"
#include "modules/audio_coding/neteq/tools/rtp_generator.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/flags.h"
#include "test/gtest.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_coding/neteq/tools/neteq_rtpplay.cc b/modules/audio_coding/neteq/tools/neteq_rtpplay.cc
index 8c1fa38..7c071db 100644
--- a/modules/audio_coding/neteq/tools/neteq_rtpplay.cc
+++ b/modules/audio_coding/neteq/tools/neteq_rtpplay.cc
@@ -10,29 +10,24 @@
#include <errno.h>
#include <inttypes.h>
+#include <iostream>
#include <limits.h> // For ULONG_MAX returned by strtoul.
+#include <memory>
#include <stdio.h>
#include <stdlib.h> // For strtoul.
-#include <string.h>
-
-#include <algorithm>
-#include <ios>
-#include <iostream>
-#include <memory>
-#include <numeric>
#include <string>
#include "modules/audio_coding/neteq/include/neteq.h"
#include "modules/audio_coding/neteq/tools/fake_decode_from_file.h"
#include "modules/audio_coding/neteq/tools/input_audio_file.h"
#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h"
+#include "modules/audio_coding/neteq/tools/neteq_stats_getter.h"
#include "modules/audio_coding/neteq/tools/neteq_packet_source_input.h"
#include "modules/audio_coding/neteq/tools/neteq_replacement_input.h"
#include "modules/audio_coding/neteq/tools/neteq_test.h"
#include "modules/audio_coding/neteq/tools/output_audio_file.h"
#include "modules/audio_coding/neteq/tools/output_wav_file.h"
#include "modules/audio_coding/neteq/tools/rtp_file_source.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/flags.h"
#include "test/testsupport/fileutils.h"
@@ -309,166 +304,6 @@
rtc::Optional<uint32_t> last_ssrc_;
};
-class StatsGetter : public NetEqGetAudioCallback {
- public:
- // This struct is a replica of webrtc::NetEqNetworkStatistics, but with all
- // values stored in double precision.
- struct Stats {
- double current_buffer_size_ms = 0.0;
- double preferred_buffer_size_ms = 0.0;
- double jitter_peaks_found = 0.0;
- double packet_loss_rate = 0.0;
- double expand_rate = 0.0;
- double speech_expand_rate = 0.0;
- double preemptive_rate = 0.0;
- double accelerate_rate = 0.0;
- double secondary_decoded_rate = 0.0;
- double secondary_discarded_rate = 0.0;
- double clockdrift_ppm = 0.0;
- double added_zero_samples = 0.0;
- double mean_waiting_time_ms = 0.0;
- double median_waiting_time_ms = 0.0;
- double min_waiting_time_ms = 0.0;
- double max_waiting_time_ms = 0.0;
- };
-
- struct ConcealmentEvent {
- uint64_t duration_ms;
- size_t concealment_event_number;
- int64_t time_from_previous_event_end_ms;
-
- friend std::ostream& operator<<(std::ostream& stream,
- const ConcealmentEvent& concealment_event) {
- stream << "ConcealmentEvent duration_ms:" << concealment_event.duration_ms
- << " event_number:" << concealment_event.concealment_event_number
- << " time_from_previous_event_end_ms:"
- << concealment_event.time_from_previous_event_end_ms << "\n";
- return stream;
- }
- };
-
- // Takes a pointer to another callback object, which will be invoked after
- // this object finishes. This does not transfer ownership, and null is a
- // valid value.
- explicit StatsGetter(NetEqGetAudioCallback* other_callback)
- : other_callback_(other_callback) {}
-
- void BeforeGetAudio(NetEq* neteq) override {
- if (other_callback_) {
- other_callback_->BeforeGetAudio(neteq);
- }
- }
-
- void AfterGetAudio(int64_t time_now_ms,
- const AudioFrame& audio_frame,
- bool muted,
- NetEq* neteq) override {
- if (++counter_ >= 100) {
- counter_ = 0;
- NetEqNetworkStatistics stats;
- RTC_CHECK_EQ(neteq->NetworkStatistics(&stats), 0);
- stats_.push_back(stats);
- }
- const auto lifetime_stat = neteq->GetLifetimeStatistics();
- if (current_concealment_event_ != lifetime_stat.concealment_events) {
- if (last_event_end_time_ms_ > 0) {
- // Do not account for the first event to avoid start of the call
- // skewing.
- ConcealmentEvent concealment_event;
- uint64_t last_event_voice_concealed_samples =
- lifetime_stat.voice_concealed_samples -
- voice_concealed_samples_until_last_event_;
- RTC_CHECK_GT(last_event_voice_concealed_samples, 0);
- concealment_event.duration_ms = last_event_voice_concealed_samples /
- (audio_frame.sample_rate_hz_ / 1000);
- concealment_event.concealment_event_number = current_concealment_event_;
- concealment_event.time_from_previous_event_end_ms =
- time_now_ms - last_event_end_time_ms_;
- concealment_events_.emplace_back(concealment_event);
- voice_concealed_samples_until_last_event_ =
- lifetime_stat.voice_concealed_samples;
- }
- last_event_end_time_ms_ = time_now_ms;
- voice_concealed_samples_until_last_event_ =
- lifetime_stat.voice_concealed_samples;
- current_concealment_event_ = lifetime_stat.concealment_events;
- }
-
- if (other_callback_) {
- other_callback_->AfterGetAudio(time_now_ms, audio_frame, muted, neteq);
- }
- }
-
- double AverageSpeechExpandRate() const {
- double sum_speech_expand =
- std::accumulate(stats_.begin(), stats_.end(), double{0.0},
- [](double a, NetEqNetworkStatistics b) {
- return a + static_cast<double>(b.speech_expand_rate);
- });
- return sum_speech_expand / 16384.0 / stats_.size();
- }
-
- const std::vector<ConcealmentEvent>& concealment_events() {
- // Do not account for the last concealment event to avoid potential end
- // call skewing.
- return concealment_events_;
- }
-
- Stats AverageStats() const {
- Stats sum_stats = std::accumulate(
- stats_.begin(), stats_.end(), Stats(),
- [](Stats a, NetEqNetworkStatistics b) {
- a.current_buffer_size_ms += b.current_buffer_size_ms;
- a.preferred_buffer_size_ms += b.preferred_buffer_size_ms;
- a.jitter_peaks_found += b.jitter_peaks_found;
- a.packet_loss_rate += b.packet_loss_rate / 16384.0;
- a.expand_rate += b.expand_rate / 16384.0;
- a.speech_expand_rate += b.speech_expand_rate / 16384.0;
- a.preemptive_rate += b.preemptive_rate / 16384.0;
- a.accelerate_rate += b.accelerate_rate / 16384.0;
- a.secondary_decoded_rate += b.secondary_decoded_rate / 16384.0;
- a.secondary_discarded_rate += b.secondary_discarded_rate / 16384.0;
- a.clockdrift_ppm += b.clockdrift_ppm;
- a.added_zero_samples += b.added_zero_samples;
- a.mean_waiting_time_ms += b.mean_waiting_time_ms;
- a.median_waiting_time_ms += b.median_waiting_time_ms;
- a.min_waiting_time_ms =
- std::min(a.min_waiting_time_ms,
- static_cast<double>(b.min_waiting_time_ms));
- a.max_waiting_time_ms =
- std::max(a.max_waiting_time_ms,
- static_cast<double>(b.max_waiting_time_ms));
- return a;
- });
-
- sum_stats.current_buffer_size_ms /= stats_.size();
- sum_stats.preferred_buffer_size_ms /= stats_.size();
- sum_stats.jitter_peaks_found /= stats_.size();
- sum_stats.packet_loss_rate /= stats_.size();
- sum_stats.expand_rate /= stats_.size();
- sum_stats.speech_expand_rate /= stats_.size();
- sum_stats.preemptive_rate /= stats_.size();
- sum_stats.accelerate_rate /= stats_.size();
- sum_stats.secondary_decoded_rate /= stats_.size();
- sum_stats.secondary_discarded_rate /= stats_.size();
- sum_stats.clockdrift_ppm /= stats_.size();
- sum_stats.added_zero_samples /= stats_.size();
- sum_stats.mean_waiting_time_ms /= stats_.size();
- sum_stats.median_waiting_time_ms /= stats_.size();
-
- return sum_stats;
- }
-
- private:
- NetEqGetAudioCallback* other_callback_;
- size_t counter_ = 0;
- std::vector<NetEqNetworkStatistics> stats_;
- size_t current_concealment_event_ = 1;
- uint64_t voice_concealed_samples_until_last_event_ = 0;
- std::vector<ConcealmentEvent> concealment_events_;
- int64_t last_event_end_time_ms_ = 0;
-};
-
int RunTest(int argc, char* argv[]) {
std::string program_name = argv[0];
std::string usage = "Tool for decoding an RTP dump file using NetEq.\n"
@@ -674,7 +509,7 @@
SsrcSwitchDetector ssrc_switch_detector(delay_analyzer.get());
callbacks.post_insert_packet = &ssrc_switch_detector;
- StatsGetter stats_getter(delay_analyzer.get());
+ NetEqStatsGetter stats_getter(std::move(delay_analyzer));
callbacks.get_audio_callback = &stats_getter;
NetEq::Config config;
config.sample_rate_hz = *sample_rate_hz;
@@ -720,11 +555,10 @@
printf(" current_buffer_size_ms: %f ms\n", stats.current_buffer_size_ms);
printf(" preferred_buffer_size_ms: %f ms\n", stats.preferred_buffer_size_ms);
if (FLAG_concealment_events) {
- std::cout << " concealment_events_ms:"
- << "\n";
+ std::cout << " concealment_events_ms:" << std::endl;
for (auto concealment_event : stats_getter.concealment_events())
- std::cout << concealment_event;
- std::cout << " end of concealment_events_ms\n";
+ std::cout << concealment_event.ToString() << std::endl;
+ std::cout << " end of concealment_events_ms" << std::endl;
}
return 0;
}
diff --git a/modules/audio_coding/neteq/tools/neteq_stats_getter.cc b/modules/audio_coding/neteq/tools/neteq_stats_getter.cc
new file mode 100644
index 0000000..6474e21
--- /dev/null
+++ b/modules/audio_coding/neteq/tools/neteq_stats_getter.cc
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_coding/neteq/tools/neteq_stats_getter.h"
+
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/strings/string_builder.h"
+#include "rtc_base/timeutils.h"
+
+namespace webrtc {
+namespace test {
+
+std::string NetEqStatsGetter::ConcealmentEvent::ToString() const {
+ char ss_buf[256];
+ rtc::SimpleStringBuilder ss(ss_buf);
+ ss << "ConcealmentEvent duration_ms:" << duration_ms
+ << " event_number:" << concealment_event_number
+ << " time_from_previous_event_end_ms:"
+ << time_from_previous_event_end_ms;
+ return ss.str();
+}
+
+NetEqStatsGetter::NetEqStatsGetter(
+ std::unique_ptr<NetEqDelayAnalyzer> delay_analyzer)
+ : delay_analyzer_(std::move(delay_analyzer)) {}
+
+void NetEqStatsGetter::BeforeGetAudio(NetEq* neteq) {
+ if (delay_analyzer_) {
+ delay_analyzer_->BeforeGetAudio(neteq);
+ }
+}
+
+void NetEqStatsGetter::AfterGetAudio(int64_t time_now_ms,
+ const AudioFrame& audio_frame,
+ bool muted,
+ NetEq* neteq) {
+ // TODO(minyue): Get stats should better not be called as a call back after
+ // get audio. It is called independently from get audio in practice.
+ if (last_stats_query_time_ms_ == 0 ||
+ rtc::TimeDiff(time_now_ms, last_stats_query_time_ms_) >=
+ stats_query_interval_ms_) {
+ NetEqNetworkStatistics stats;
+ RTC_CHECK_EQ(neteq->NetworkStatistics(&stats), 0);
+ stats_.push_back(std::make_pair(time_now_ms, stats));
+ last_stats_query_time_ms_ = time_now_ms;
+ }
+ const auto lifetime_stat = neteq->GetLifetimeStatistics();
+ if (current_concealment_event_ != lifetime_stat.concealment_events &&
+ voice_concealed_samples_until_last_event_ <
+ lifetime_stat.voice_concealed_samples) {
+ if (last_event_end_time_ms_ > 0) {
+ // Do not account for the first event to avoid start of the call
+ // skewing.
+ ConcealmentEvent concealment_event;
+ uint64_t last_event_voice_concealed_samples =
+ lifetime_stat.voice_concealed_samples -
+ voice_concealed_samples_until_last_event_;
+ RTC_CHECK_GT(last_event_voice_concealed_samples, 0);
+ concealment_event.duration_ms = last_event_voice_concealed_samples /
+ (audio_frame.sample_rate_hz_ / 1000);
+ concealment_event.concealment_event_number = current_concealment_event_;
+ concealment_event.time_from_previous_event_end_ms =
+ time_now_ms - last_event_end_time_ms_;
+ concealment_events_.emplace_back(concealment_event);
+ voice_concealed_samples_until_last_event_ =
+ lifetime_stat.voice_concealed_samples;
+ }
+ last_event_end_time_ms_ = time_now_ms;
+ voice_concealed_samples_until_last_event_ =
+ lifetime_stat.voice_concealed_samples;
+ current_concealment_event_ = lifetime_stat.concealment_events;
+ }
+
+ if (delay_analyzer_) {
+ delay_analyzer_->AfterGetAudio(time_now_ms, audio_frame, muted, neteq);
+ }
+}
+
+double NetEqStatsGetter::AverageSpeechExpandRate() const {
+ double sum_speech_expand = std::accumulate(
+ stats_.begin(), stats_.end(), double{0.0},
+ [](double a, std::pair<int64_t, NetEqNetworkStatistics> b) {
+ return a + static_cast<double>(b.second.speech_expand_rate);
+ });
+ return sum_speech_expand / 16384.0 / stats_.size();
+}
+
+NetEqStatsGetter::Stats NetEqStatsGetter::AverageStats() const {
+ Stats sum_stats = std::accumulate(
+ stats_.begin(), stats_.end(), Stats(),
+ [](Stats a, std::pair<int64_t, NetEqNetworkStatistics> bb) {
+ const auto& b = bb.second;
+ a.current_buffer_size_ms += b.current_buffer_size_ms;
+ a.preferred_buffer_size_ms += b.preferred_buffer_size_ms;
+ a.jitter_peaks_found += b.jitter_peaks_found;
+ a.packet_loss_rate += b.packet_loss_rate / 16384.0;
+ a.expand_rate += b.expand_rate / 16384.0;
+ a.speech_expand_rate += b.speech_expand_rate / 16384.0;
+ a.preemptive_rate += b.preemptive_rate / 16384.0;
+ a.accelerate_rate += b.accelerate_rate / 16384.0;
+ a.secondary_decoded_rate += b.secondary_decoded_rate / 16384.0;
+ a.secondary_discarded_rate += b.secondary_discarded_rate / 16384.0;
+ a.clockdrift_ppm += b.clockdrift_ppm;
+ a.added_zero_samples += b.added_zero_samples;
+ a.mean_waiting_time_ms += b.mean_waiting_time_ms;
+ a.median_waiting_time_ms += b.median_waiting_time_ms;
+ a.min_waiting_time_ms =
+ std::min(a.min_waiting_time_ms,
+ static_cast<double>(b.min_waiting_time_ms));
+ a.max_waiting_time_ms =
+ std::max(a.max_waiting_time_ms,
+ static_cast<double>(b.max_waiting_time_ms));
+ return a;
+ });
+
+ sum_stats.current_buffer_size_ms /= stats_.size();
+ sum_stats.preferred_buffer_size_ms /= stats_.size();
+ sum_stats.jitter_peaks_found /= stats_.size();
+ sum_stats.packet_loss_rate /= stats_.size();
+ sum_stats.expand_rate /= stats_.size();
+ sum_stats.speech_expand_rate /= stats_.size();
+ sum_stats.preemptive_rate /= stats_.size();
+ sum_stats.accelerate_rate /= stats_.size();
+ sum_stats.secondary_decoded_rate /= stats_.size();
+ sum_stats.secondary_discarded_rate /= stats_.size();
+ sum_stats.clockdrift_ppm /= stats_.size();
+ sum_stats.added_zero_samples /= stats_.size();
+ sum_stats.mean_waiting_time_ms /= stats_.size();
+ sum_stats.median_waiting_time_ms /= stats_.size();
+
+ return sum_stats;
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/modules/audio_coding/neteq/tools/neteq_stats_getter.h b/modules/audio_coding/neteq/tools/neteq_stats_getter.h
new file mode 100644
index 0000000..dbb396a
--- /dev/null
+++ b/modules/audio_coding/neteq/tools/neteq_stats_getter.h
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_GETTER_H_
+#define MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_GETTER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "modules/audio_coding/neteq/tools/neteq_delay_analyzer.h"
+#include "modules/audio_coding/neteq/tools/neteq_test.h"
+
+namespace webrtc {
+namespace test {
+
+class NetEqStatsGetter : public NetEqGetAudioCallback {
+ public:
+ // This struct is a replica of webrtc::NetEqNetworkStatistics, but with all
+ // values stored in double precision.
+ struct Stats {
+ double current_buffer_size_ms = 0.0;
+ double preferred_buffer_size_ms = 0.0;
+ double jitter_peaks_found = 0.0;
+ double packet_loss_rate = 0.0;
+ double expand_rate = 0.0;
+ double speech_expand_rate = 0.0;
+ double preemptive_rate = 0.0;
+ double accelerate_rate = 0.0;
+ double secondary_decoded_rate = 0.0;
+ double secondary_discarded_rate = 0.0;
+ double clockdrift_ppm = 0.0;
+ double added_zero_samples = 0.0;
+ double mean_waiting_time_ms = 0.0;
+ double median_waiting_time_ms = 0.0;
+ double min_waiting_time_ms = 0.0;
+ double max_waiting_time_ms = 0.0;
+ };
+
+ struct ConcealmentEvent {
+ uint64_t duration_ms;
+ size_t concealment_event_number;
+ int64_t time_from_previous_event_end_ms;
+ std::string ToString() const;
+ };
+
+ // Takes a pointer to another callback object, which will be invoked after
+ // this object finishes. This does not transfer ownership, and null is a
+ // valid value.
+ explicit NetEqStatsGetter(std::unique_ptr<NetEqDelayAnalyzer> delay_analyzer);
+
+ void set_stats_query_interval_ms(int64_t stats_query_interval_ms) {
+ stats_query_interval_ms_ = stats_query_interval_ms;
+ }
+
+ void BeforeGetAudio(NetEq* neteq) override;
+
+ void AfterGetAudio(int64_t time_now_ms,
+ const AudioFrame& audio_frame,
+ bool muted,
+ NetEq* neteq) override;
+
+ double AverageSpeechExpandRate() const;
+
+ NetEqDelayAnalyzer* delay_analyzer() const {
+ return delay_analyzer_.get();
+ }
+
+ const std::vector<ConcealmentEvent>& concealment_events() const {
+ // Do not account for the last concealment event to avoid potential end
+ // call skewing.
+ return concealment_events_;
+ }
+
+ const std::vector<std::pair<int64_t, NetEqNetworkStatistics>>& stats() const {
+ return stats_;
+ }
+
+ Stats AverageStats() const;
+
+ private:
+ std::unique_ptr<NetEqDelayAnalyzer> delay_analyzer_;
+ int64_t stats_query_interval_ms_ = 1000;
+ int64_t last_stats_query_time_ms_ = 0;
+ std::vector<std::pair<int64_t, NetEqNetworkStatistics>> stats_;
+ size_t current_concealment_event_ = 1;
+ uint64_t voice_concealed_samples_until_last_event_ = 0;
+ std::vector<ConcealmentEvent> concealment_events_;
+ int64_t last_event_end_time_ms_ = 0;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_CODING_NETEQ_TOOLS_NETEQ_STATS_GETTER_H_
diff --git a/modules/audio_coding/neteq/tools/packet.cc b/modules/audio_coding/neteq/tools/packet.cc
index 71337b6..9505a29 100644
--- a/modules/audio_coding/neteq/tools/packet.cc
+++ b/modules/audio_coding/neteq/tools/packet.cc
@@ -14,7 +14,6 @@
#include <memory>
-#include "modules/include/module_common_types.h"
#include "modules/rtp_rtcp/include/rtp_header_parser.h"
#include "rtc_base/checks.h"
diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc
index d6224ff..b853248 100644
--- a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc
+++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc
@@ -40,7 +40,7 @@
for (; rtp_packet_index_ < parsed_stream_.GetNumberOfEvents();
rtp_packet_index_++) {
if (parsed_stream_.GetEventType(rtp_packet_index_) ==
- ParsedRtcEventLog::RTP_EVENT) {
+ ParsedRtcEventLogNew::RTP_EVENT) {
PacketDirection direction;
size_t header_length;
size_t packet_length;
@@ -66,7 +66,7 @@
}
if (parsed_stream_.GetMediaType(packet->header().ssrc, direction) !=
- webrtc::ParsedRtcEventLog::MediaType::AUDIO) {
+ ParsedRtcEventLogNew::MediaType::AUDIO) {
continue;
}
@@ -84,13 +84,11 @@
int64_t RtcEventLogSource::NextAudioOutputEventMs() {
while (audio_output_index_ < parsed_stream_.GetNumberOfEvents()) {
if (parsed_stream_.GetEventType(audio_output_index_) ==
- ParsedRtcEventLog::AUDIO_PLAYOUT_EVENT) {
- uint64_t timestamp_us = parsed_stream_.GetTimestamp(audio_output_index_);
- // We call GetAudioPlayout only to check that the protobuf event is
- // well-formed.
- parsed_stream_.GetAudioPlayout(audio_output_index_, nullptr);
+ ParsedRtcEventLogNew::AUDIO_PLAYOUT_EVENT) {
+ LoggedAudioPlayoutEvent playout_event =
+ parsed_stream_.GetAudioPlayout(audio_output_index_);
audio_output_index_++;
- return timestamp_us / 1000;
+ return playout_event.timestamp_us / 1000;
}
audio_output_index_++;
}
diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.h b/modules/audio_coding/neteq/tools/rtc_event_log_source.h
index df01e06..db4eb19 100644
--- a/modules/audio_coding/neteq/tools/rtc_event_log_source.h
+++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.h
@@ -14,7 +14,7 @@
#include <memory>
#include <string>
-#include "logging/rtc_event_log/rtc_event_log_parser.h"
+#include "logging/rtc_event_log/rtc_event_log_parser_new.h"
#include "modules/audio_coding/neteq/tools/packet_source.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "rtc_base/constructormagic.h"
@@ -53,7 +53,7 @@
size_t rtp_packet_index_ = 0;
size_t audio_output_index_ = 0;
- ParsedRtcEventLog parsed_stream_;
+ ParsedRtcEventLogNew parsed_stream_;
std::unique_ptr<RtpHeaderParser> parser_;
RTC_DISALLOW_COPY_AND_ASSIGN(RtcEventLogSource);
diff --git a/modules/audio_coding/neteq/tools/rtp_encode.cc b/modules/audio_coding/neteq/tools/rtp_encode.cc
index ce07199..66e7a28 100644
--- a/modules/audio_coding/neteq/tools/rtp_encode.cc
+++ b/modules/audio_coding/neteq/tools/rtp_encode.cc
@@ -21,6 +21,7 @@
#include <map>
#include <string>
+#include "api/audio/audio_frame.h"
#include "api/audio_codecs/L16/audio_encoder_L16.h"
#include "api/audio_codecs/g711/audio_encoder_g711.h"
#include "api/audio_codecs/g722/audio_encoder_g722.h"
diff --git a/modules/audio_coding/neteq/tools/rtp_file_source.cc b/modules/audio_coding/neteq/tools/rtp_file_source.cc
index c9ae5f2..0945667 100644
--- a/modules/audio_coding/neteq/tools/rtp_file_source.cc
+++ b/modules/audio_coding/neteq/tools/rtp_file_source.cc
@@ -12,9 +12,7 @@
#include <assert.h>
#include <string.h>
-#ifdef WIN32
-#include <winsock2.h>
-#else
+#ifndef WIN32
#include <netinet/in.h>
#endif
diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn
index 09b0bd4..a64fc6f 100644
--- a/modules/audio_processing/BUILD.gn
+++ b/modules/audio_processing/BUILD.gn
@@ -29,6 +29,7 @@
rtc_static_library("audio_processing") {
visibility = [ "*" ]
+ allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove.
configs += [ ":apm_debug_dump" ]
sources = [
"aec/aec_resampler.cc",
@@ -122,25 +123,26 @@
":audio_generator_interface",
":audio_processing_c",
":audio_processing_statistics",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../api:array_view",
"../../api:optional",
"../../api/audio:aec3_config",
+ "../../api/audio:audio_frame_api",
"../../api/audio:echo_control",
"../../audio/utility:audio_frame_operations",
"../../common_audio:common_audio_c",
"../../rtc_base:checks",
"../../rtc_base:deprecation",
"../../rtc_base:gtest_prod",
- "../../rtc_base:protobuf_utils",
"../../rtc_base:safe_minmax",
"../../rtc_base:sanitizer",
"../../system_wrappers:cpu_features_api",
"../../system_wrappers:field_trial_api",
"../../system_wrappers:metrics_api",
- "agc2",
+ "agc2:adaptive_digital",
+ "agc2:fixed_digital",
+ "agc2:gain_applier",
"vad",
]
@@ -148,11 +150,6 @@
defines += [ "WEBRTC_UNTRUSTED_DELAY" ]
}
- if (rtc_enable_protobuf) {
- defines += [ "WEBRTC_AUDIOPROC_DEBUG_DUMP" ]
- deps += [ ":audioproc_debug_proto" ]
- }
-
if (rtc_enable_intelligibility_enhancer) {
defines += [ "WEBRTC_INTELLIGIBILITY_ENHANCER=1" ]
sources += [
@@ -212,6 +209,7 @@
deps = [
":audio_frame_view",
"../../api:array_view",
+ "../../api/audio:audio_frame_api",
"../../rtc_base:rtc_base_approved",
]
}
@@ -392,7 +390,7 @@
"utility/ooura_fft_sse2.cc",
"utility/ooura_fft_tables_neon_sse2.h",
]
- if (is_posix) {
+ if (is_posix || is_fuchsia) {
cflags += [ "-msse2" ]
}
}
@@ -524,7 +522,6 @@
":audioproc_test_utils",
":file_audio_generator_unittests",
":mocks",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../api:array_view",
@@ -539,13 +536,17 @@
"../../rtc_base:rtc_base",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:safe_minmax",
+ "../../rtc_base/system:file_wrapper",
"../../system_wrappers",
"../../system_wrappers:cpu_features_api",
"../../test:fileutils",
"../../test:test_support",
"../audio_coding:neteq_input_audio_tools",
"aec_dump:mock_aec_dump_unittests",
+ "agc2:adaptive_digital_unittests",
"agc2:fixed_digital_unittests",
+ "agc2:noise_estimator_unittests",
+ "agc2/rnn_vad:unittests",
"test/conversational_speech:unittest",
"vad:vad_unittests",
"//testing/gtest",
@@ -576,6 +577,7 @@
":audioproc_protobuf_utils",
":audioproc_test_utils",
":audioproc_unittest_proto",
+ "../../api/audio:audio_frame_api",
"../../rtc_base:rtc_task_queue",
"aec_dump",
"aec_dump:aec_dump_unittests",
@@ -624,7 +626,6 @@
":audio_processing",
":audioproc_test_utils",
"../../api:array_view",
- "../../modules:module_api",
"../../rtc_base:protobuf_utils",
"../../rtc_base:rtc_base_approved",
"../../system_wrappers",
@@ -663,8 +664,9 @@
]
deps = [
"../../api:array_view",
+ "../../api:optional",
+ "../../api/audio:audio_frame_api",
"../../common_audio:common_audio",
- "../../modules:module_api",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
]
@@ -738,9 +740,9 @@
deps = [
":audio_processing",
- "..:module_api",
"../../api:array_view",
"../../api:optional",
+ "../../api/audio:audio_frame_api",
"../../common_audio",
"../../rtc_base:checks",
"../../rtc_base:rtc_base_approved",
@@ -761,11 +763,11 @@
]
deps = [
":audio_processing",
- "..:module_api",
"../..:typedefs",
"../..:webrtc_common",
"../../common_audio:common_audio",
"../../rtc_base:rtc_base_approved",
+ "../../rtc_base/system:file_wrapper",
"../../system_wrappers",
"../../system_wrappers:metrics_default",
"../../test:fileutils",
@@ -785,6 +787,7 @@
":audio_processing",
"../..:typedefs",
"../..:webrtc_common",
+ "../../rtc_base/system:file_wrapper",
"../../system_wrappers",
"../../system_wrappers:metrics_default",
]
diff --git a/modules/audio_processing/aec/aec_core.h b/modules/audio_processing/aec/aec_core.h
index 78596ec..9e25292 100644
--- a/modules/audio_processing/aec/aec_core.h
+++ b/modules/audio_processing/aec/aec_core.h
@@ -22,7 +22,6 @@
extern "C" {
#include "common_audio/ring_buffer.h"
}
-#include "common_audio/wav_file.h"
#include "modules/audio_processing/aec/aec_common.h"
#include "modules/audio_processing/utility/block_mean_calculator.h"
#include "modules/audio_processing/utility/ooura_fft.h"
diff --git a/modules/audio_processing/aec/echo_cancellation.cc b/modules/audio_processing/aec/echo_cancellation.cc
index 864db53..1633068 100644
--- a/modules/audio_processing/aec/echo_cancellation.cc
+++ b/modules/audio_processing/aec/echo_cancellation.cc
@@ -677,7 +677,7 @@
}
static void ProcessExtended(Aec* self,
- const float* const* near,
+ const float* const* nearend,
size_t num_bands,
float* const* out,
size_t num_samples,
@@ -709,8 +709,8 @@
if (!self->farend_started) {
for (i = 0; i < num_bands; ++i) {
// Only needed if they don't already point to the same place.
- if (near[i] != out[i]) {
- memcpy(out[i], near[i], sizeof(near[i][0]) * num_samples);
+ if (nearend[i] != out[i]) {
+ memcpy(out[i], nearend[i], sizeof(nearend[i][0]) * num_samples);
}
}
return;
@@ -746,7 +746,7 @@
const int adjusted_known_delay =
WEBRTC_SPL_MAX(0, self->knownDelay + delay_diff_offset);
- WebRtcAec_ProcessFrames(self->aec, near, num_bands, num_samples,
+ WebRtcAec_ProcessFrames(self->aec, nearend, num_bands, num_samples,
adjusted_known_delay, out);
}
}
diff --git a/modules/audio_processing/aec3/BUILD.gn b/modules/audio_processing/aec3/BUILD.gn
index 372b30f..9658f6b 100644
--- a/modules/audio_processing/aec3/BUILD.gn
+++ b/modules/audio_processing/aec3/BUILD.gn
@@ -11,6 +11,7 @@
rtc_static_library("aec3") {
visibility = [ "*" ]
+ allow_poison = [ "audio_codecs" ] # TODO(bugs.webrtc.org/8396): Remove.
configs += [ "..:apm_debug_dump" ]
sources = [
"adaptive_fir_filter.cc",
@@ -29,6 +30,8 @@
"block_processor_metrics.h",
"cascaded_biquad_filter.cc",
"cascaded_biquad_filter.h",
+ "coherence_gain.cc",
+ "coherence_gain.h",
"comfort_noise_generator.cc",
"comfort_noise_generator.h",
"decimator.cc",
@@ -36,6 +39,8 @@
"delay_estimate.h",
"downsampled_render_buffer.cc",
"downsampled_render_buffer.h",
+ "echo_audibility.cc",
+ "echo_audibility.h",
"echo_canceller3.cc",
"echo_canceller3.h",
"echo_path_delay_estimator.cc",
@@ -53,6 +58,8 @@
"fft_buffer.cc",
"fft_buffer.h",
"fft_data.h",
+ "filter_analyzer.cc",
+ "filter_analyzer.h",
"frame_blocker.cc",
"frame_blocker.h",
"main_filter_update_gain.cc",
@@ -63,8 +70,6 @@
"matched_filter_lag_aggregator.h",
"matrix_buffer.cc",
"matrix_buffer.h",
- "output_selector.cc",
- "output_selector.h",
"render_buffer.cc",
"render_buffer.h",
"render_delay_buffer.cc",
@@ -81,6 +86,8 @@
"shadow_filter_update_gain.h",
"skew_estimator.cc",
"skew_estimator.h",
+ "stationarity_estimator.cc",
+ "stationarity_estimator.h",
"subtractor.cc",
"subtractor.h",
"subtractor_output.h",
@@ -169,7 +176,6 @@
"main_filter_update_gain_unittest.cc",
"matched_filter_lag_aggregator_unittest.cc",
"matched_filter_unittest.cc",
- "output_selector_unittest.cc",
"render_buffer_unittest.cc",
"render_delay_buffer_unittest.cc",
"render_delay_controller_metrics_unittest.cc",
diff --git a/modules/audio_processing/aec3/adaptive_fir_filter.cc b/modules/audio_processing/aec3/adaptive_fir_filter.cc
index 9bea40b..f44fe3d 100644
--- a/modules/audio_processing/aec3/adaptive_fir_filter.cc
+++ b/modules/audio_processing/aec3/adaptive_fir_filter.cc
@@ -504,6 +504,9 @@
H_.resize(current_size_partitions_);
H2_.resize(current_size_partitions_);
h_.resize(GetTimeDomainLength(current_size_partitions_));
+ RTC_DCHECK_LT(0, current_size_partitions_);
+ partition_to_constrain_ =
+ std::min(partition_to_constrain_, current_size_partitions_ - 1);
}
void AdaptiveFirFilter::UpdateSize() {
diff --git a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc
index 9fb11cd..9561dff 100644
--- a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc
+++ b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc
@@ -351,7 +351,7 @@
CascadedBiQuadFilter y_hp_filter(kHighPassFilterCoefficients, 1);
SCOPED_TRACE(ProduceDebugText(delay_samples));
- for (size_t k = 0; k < kNumBlocksToProcess; ++k) {
+ for (size_t j = 0; j < kNumBlocksToProcess; ++j) {
RandomizeSampleVector(&random_generator, x[0]);
delay_buffer.Delay(x[0], y);
@@ -365,13 +365,14 @@
y_hp_filter.Process(y);
render_delay_buffer->Insert(x);
- if (k == 0) {
+ if (j == 0) {
render_delay_buffer->Reset();
}
render_delay_buffer->PrepareCaptureProcessing();
const auto& render_buffer = render_delay_buffer->GetRenderBuffer();
- render_signal_analyzer.Update(*render_buffer, aec_state.FilterDelay());
+ render_signal_analyzer.Update(*render_buffer,
+ aec_state.FilterDelayBlocks());
filter.Filter(*render_buffer, &S);
fft.Ifft(S, &s_scratch);
@@ -392,15 +393,14 @@
filter.Adapt(*render_buffer, G);
aec_state.HandleEchoPathChange(EchoPathVariability(
false, EchoPathVariability::DelayAdjustment::kNone, false));
+
aec_state.Update(delay_estimate, filter.FilterFrequencyResponse(),
- filter.FilterImpulseResponse(), true, *render_buffer,
- E2_main, Y2, s, false);
+ filter.FilterImpulseResponse(), true, false,
+ *render_buffer, E2_main, Y2, s);
}
// Verify that the filter is able to perform well.
EXPECT_LT(1000 * std::inner_product(e.begin(), e.end(), e.begin(), 0.f),
std::inner_product(y.begin(), y.end(), y.begin(), 0.f));
- EXPECT_EQ(delay_samples / kBlockSize,
- static_cast<size_t>(aec_state.FilterDelay()));
}
}
} // namespace aec3
diff --git a/modules/audio_processing/aec3/aec3_fft.cc b/modules/audio_processing/aec3/aec3_fft.cc
index d669036..b02f342 100644
--- a/modules/audio_processing/aec3/aec3_fft.cc
+++ b/modules/audio_processing/aec3/aec3_fft.cc
@@ -33,6 +33,41 @@
0.15088159f, 0.11697778f, 0.08688061f, 0.06088921f, 0.03926189f,
0.0222136f, 0.00991376f, 0.00248461f, 0.f};
+// Hanning window from Matlab command win = sqrt(hanning(128)).
+const float kSqrtHanning128[kFftLength] = {
+ 0.00000000000000f, 0.02454122852291f, 0.04906767432742f, 0.07356456359967f,
+ 0.09801714032956f, 0.12241067519922f, 0.14673047445536f, 0.17096188876030f,
+ 0.19509032201613f, 0.21910124015687f, 0.24298017990326f, 0.26671275747490f,
+ 0.29028467725446f, 0.31368174039889f, 0.33688985339222f, 0.35989503653499f,
+ 0.38268343236509f, 0.40524131400499f, 0.42755509343028f, 0.44961132965461f,
+ 0.47139673682600f, 0.49289819222978f, 0.51410274419322f, 0.53499761988710f,
+ 0.55557023301960f, 0.57580819141785f, 0.59569930449243f, 0.61523159058063f,
+ 0.63439328416365f, 0.65317284295378f, 0.67155895484702f, 0.68954054473707f,
+ 0.70710678118655f, 0.72424708295147f, 0.74095112535496f, 0.75720884650648f,
+ 0.77301045336274f, 0.78834642762661f, 0.80320753148064f, 0.81758481315158f,
+ 0.83146961230255f, 0.84485356524971f, 0.85772861000027f, 0.87008699110871f,
+ 0.88192126434835f, 0.89322430119552f, 0.90398929312344f, 0.91420975570353f,
+ 0.92387953251129f, 0.93299279883474f, 0.94154406518302f, 0.94952818059304f,
+ 0.95694033573221f, 0.96377606579544f, 0.97003125319454f, 0.97570213003853f,
+ 0.98078528040323f, 0.98527764238894f, 0.98917650996478f, 0.99247953459871f,
+ 0.99518472667220f, 0.99729045667869f, 0.99879545620517f, 0.99969881869620f,
+ 1.00000000000000f, 0.99969881869620f, 0.99879545620517f, 0.99729045667869f,
+ 0.99518472667220f, 0.99247953459871f, 0.98917650996478f, 0.98527764238894f,
+ 0.98078528040323f, 0.97570213003853f, 0.97003125319454f, 0.96377606579544f,
+ 0.95694033573221f, 0.94952818059304f, 0.94154406518302f, 0.93299279883474f,
+ 0.92387953251129f, 0.91420975570353f, 0.90398929312344f, 0.89322430119552f,
+ 0.88192126434835f, 0.87008699110871f, 0.85772861000027f, 0.84485356524971f,
+ 0.83146961230255f, 0.81758481315158f, 0.80320753148064f, 0.78834642762661f,
+ 0.77301045336274f, 0.75720884650648f, 0.74095112535496f, 0.72424708295147f,
+ 0.70710678118655f, 0.68954054473707f, 0.67155895484702f, 0.65317284295378f,
+ 0.63439328416365f, 0.61523159058063f, 0.59569930449243f, 0.57580819141785f,
+ 0.55557023301960f, 0.53499761988710f, 0.51410274419322f, 0.49289819222978f,
+ 0.47139673682600f, 0.44961132965461f, 0.42755509343028f, 0.40524131400499f,
+ 0.38268343236509f, 0.35989503653499f, 0.33688985339222f, 0.31368174039889f,
+ 0.29028467725446f, 0.26671275747490f, 0.24298017990326f, 0.21910124015687f,
+ 0.19509032201613f, 0.17096188876030f, 0.14673047445536f, 0.12241067519922f,
+ 0.09801714032956f, 0.07356456359967f, 0.04906767432742f, 0.02454122852291f};
+
} // namespace
// TODO(peah): Change x to be std::array once the rest of the code allows this.
@@ -52,6 +87,9 @@
fft.begin() + kFftLengthBy2,
[](float a, float b) { return a * b; });
break;
+ case Window::kSqrtHanning:
+ RTC_NOTREACHED();
+ break;
default:
RTC_NOTREACHED();
}
@@ -60,15 +98,33 @@
}
void Aec3Fft::PaddedFft(rtc::ArrayView<const float> x,
- rtc::ArrayView<float> x_old,
+ rtc::ArrayView<const float> x_old,
+ Window window,
FftData* X) const {
RTC_DCHECK(X);
RTC_DCHECK_EQ(kFftLengthBy2, x.size());
RTC_DCHECK_EQ(kFftLengthBy2, x_old.size());
std::array<float, kFftLength> fft;
- std::copy(x_old.begin(), x_old.end(), fft.begin());
- std::copy(x.begin(), x.end(), fft.begin() + x_old.size());
- std::copy(x.begin(), x.end(), x_old.begin());
+
+ switch (window) {
+ case Window::kRectangular:
+ std::copy(x_old.begin(), x_old.end(), fft.begin());
+ std::copy(x.begin(), x.end(), fft.begin() + x_old.size());
+ break;
+ case Window::kHanning:
+ RTC_NOTREACHED();
+ break;
+ case Window::kSqrtHanning:
+ std::transform(x_old.begin(), x_old.end(), std::begin(kSqrtHanning128),
+ fft.begin(), std::multiplies<float>());
+ std::transform(x.begin(), x.end(),
+ std::begin(kSqrtHanning128) + x_old.size(),
+ fft.begin() + x_old.size(), std::multiplies<float>());
+ break;
+ default:
+ RTC_NOTREACHED();
+ }
+
Fft(&fft, X);
}
diff --git a/modules/audio_processing/aec3/aec3_fft.h b/modules/audio_processing/aec3/aec3_fft.h
index f3dddb3..b700222 100644
--- a/modules/audio_processing/aec3/aec3_fft.h
+++ b/modules/audio_processing/aec3/aec3_fft.h
@@ -25,7 +25,7 @@
// FftData type.
class Aec3Fft {
public:
- enum class Window { kRectangular, kHanning };
+ enum class Window { kRectangular, kHanning, kSqrtHanning };
Aec3Fft() = default;
// Computes the FFT. Note that both the input and output are modified.
@@ -51,7 +51,15 @@
// Concatenates the kFftLengthBy2 values long x and x_old before computing the
// Fft. After that, x is copied to x_old.
void PaddedFft(rtc::ArrayView<const float> x,
- rtc::ArrayView<float> x_old,
+ rtc::ArrayView<const float> x_old,
+ FftData* X) const {
+ PaddedFft(x, x_old, Window::kRectangular, X);
+ }
+
+ // Padded Fft using a time-domain window.
+ void PaddedFft(rtc::ArrayView<const float> x,
+ rtc::ArrayView<const float> x_old,
+ Window window,
FftData* X) const;
private:
diff --git a/modules/audio_processing/aec3/aec3_fft_unittest.cc b/modules/audio_processing/aec3/aec3_fft_unittest.cc
index 87fe7a8..82d6e76 100644
--- a/modules/audio_processing/aec3/aec3_fft_unittest.cc
+++ b/modules/audio_processing/aec3/aec3_fft_unittest.cc
@@ -199,6 +199,7 @@
std::for_each(x_ref.begin(), x_ref.end(), [](float& a) { a *= 64.f; });
fft.PaddedFft(x_in, x_old, &X);
+ std::copy(x_in.begin(), x_in.end(), x_old.begin());
fft.Ifft(X, &x_out);
for (size_t j = 0; j < x_out.size(); ++j) {
diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc
index 533290c..5f36e65 100644
--- a/modules/audio_processing/aec3/aec_state.cc
+++ b/modules/audio_processing/aec3/aec_state.cc
@@ -23,35 +23,14 @@
namespace webrtc {
namespace {
-// Computes delay of the adaptive filter.
-int EstimateFilterDelay(
- const std::vector<std::array<float, kFftLengthBy2Plus1>>&
- adaptive_filter_frequency_response) {
- const auto& H2 = adaptive_filter_frequency_response;
- constexpr size_t kUpperBin = kFftLengthBy2 - 5;
- RTC_DCHECK_GE(kMaxAdaptiveFilterLength, H2.size());
- std::array<int, kMaxAdaptiveFilterLength> delays;
- delays.fill(0);
- for (size_t k = 1; k < kUpperBin; ++k) {
- // Find the maximum of H2[j].
- size_t peak = 0;
- for (size_t j = 0; j < H2.size(); ++j) {
- if (H2[j][k] > H2[peak][k]) {
- peak = j;
- }
- }
- ++delays[peak];
- }
-
- return std::distance(delays.begin(),
- std::max_element(delays.begin(), delays.end()));
-}
-
float ComputeGainRampupIncrease(const EchoCanceller3Config& config) {
const auto& c = config.echo_removal_control.gain_rampup;
return powf(1.f / c.first_non_zero_gain, 1.f / c.non_zero_gain_blocks);
}
+constexpr size_t kBlocksSinceConvergencedFilterInit = 10000;
+constexpr size_t kBlocksSinceConsistentEstimateInit = 10000;
+
} // namespace
int AecState::instance_count_ = 0;
@@ -64,27 +43,32 @@
max_render_(config_.filter.main.length_blocks, 0.f),
reverb_decay_(fabsf(config_.ep_strength.default_len)),
gain_rampup_increase_(ComputeGainRampupIncrease(config_)),
- suppression_gain_limiter_(config_) {}
+ suppression_gain_limiter_(config_),
+ filter_analyzer_(config_),
+ blocks_since_converged_filter_(kBlocksSinceConvergencedFilterInit),
+ active_blocks_since_consistent_filter_estimate_(
+ kBlocksSinceConsistentEstimateInit) {}
AecState::~AecState() = default;
void AecState::HandleEchoPathChange(
const EchoPathVariability& echo_path_variability) {
const auto full_reset = [&]() {
+ filter_analyzer_.Reset();
blocks_since_last_saturation_ = 0;
usable_linear_estimate_ = false;
- echo_leakage_detected_ = false;
capture_signal_saturation_ = false;
echo_saturation_ = false;
- previous_max_sample_ = 0.f;
std::fill(max_render_.begin(), max_render_.end(), 0.f);
blocks_with_proper_filter_adaptation_ = 0;
- capture_block_counter_ = 0;
+ blocks_since_reset_ = 0;
filter_has_had_time_to_converge_ = false;
render_received_ = false;
blocks_with_active_render_ = 0;
initial_state_ = true;
suppression_gain_limiter_.Reset();
+ blocks_since_converged_filter_ = kBlocksSinceConvergencedFilterInit;
+ diverged_blocks_ = 0;
};
// TODO(peah): Refine the reset scheme according to the type of gain and
@@ -106,30 +90,38 @@
EchoPathVariability::DelayAdjustment::kNewDetectedDelay) {
full_reset();
} else if (echo_path_variability.gain_change) {
- capture_block_counter_ = kNumBlocksPerSecond;
+ blocks_since_reset_ = kNumBlocksPerSecond;
}
}
void AecState::Update(
- const rtc::Optional<DelayEstimate>& delay_estimate,
+ const rtc::Optional<DelayEstimate>& external_delay,
const std::vector<std::array<float, kFftLengthBy2Plus1>>&
adaptive_filter_frequency_response,
const std::vector<float>& adaptive_filter_impulse_response,
bool converged_filter,
+ bool diverged_filter,
const RenderBuffer& render_buffer,
const std::array<float, kFftLengthBy2Plus1>& E2_main,
const std::array<float, kFftLengthBy2Plus1>& Y2,
- const std::array<float, kBlockSize>& s,
- bool echo_leakage_detected) {
- // Store input parameters.
- echo_leakage_detected_ = echo_leakage_detected;
+ const std::array<float, kBlockSize>& s) {
+ // Analyze the filter and compute the delays.
+ filter_analyzer_.Update(adaptive_filter_impulse_response, render_buffer);
+ filter_delay_blocks_ = filter_analyzer_.DelayBlocks();
- // Estimate the filter delay.
- filter_delay_ = EstimateFilterDelay(adaptive_filter_frequency_response);
- const std::vector<float>& x = render_buffer.Block(-filter_delay_)[0];
+ if (filter_analyzer_.Consistent()) {
+ internal_delay_ = filter_analyzer_.DelayBlocks();
+ } else {
+ internal_delay_ = rtc::nullopt;
+ }
+
+ external_delay_seen_ = external_delay_seen_ || external_delay;
+
+ const std::vector<float>& x = render_buffer.Block(-filter_delay_blocks_)[0];
// Update counters.
++capture_block_counter_;
+ ++blocks_since_reset_;
const bool active_render_block = DetectActiveRender(x);
blocks_with_active_render_ += active_render_block ? 1 : 0;
blocks_with_proper_filter_adaptation_ +=
@@ -137,45 +129,145 @@
// Update the limit on the echo suppression after an echo path change to avoid
// an initial echo burst.
- suppression_gain_limiter_.Update(render_buffer.GetRenderActivity());
+ suppression_gain_limiter_.Update(render_buffer.GetRenderActivity(),
+ transparent_mode_);
+
+ if (UseStationaryProperties()) {
+ // Update the echo audibility evaluator.
+ echo_audibility_.Update(render_buffer, FilterDelayBlocks(),
+ external_delay_seen_);
+ }
// Update the ERL and ERLE measures.
- if (converged_filter && capture_block_counter_ >= 2 * kNumBlocksPerSecond) {
- const auto& X2 = render_buffer.Spectrum(filter_delay_);
+ if (converged_filter && blocks_since_reset_ >= 2 * kNumBlocksPerSecond) {
+ const auto& X2 = render_buffer.Spectrum(filter_delay_blocks_);
erle_estimator_.Update(X2, Y2, E2_main);
erl_estimator_.Update(X2, Y2);
}
- // Update the echo audibility evaluator.
- echo_audibility_.Update(x, s, converged_filter);
-
// Detect and flag echo saturation.
// TODO(peah): Add the delay in this computation to ensure that the render and
// capture signals are properly aligned.
if (config_.ep_strength.echo_can_saturate) {
- echo_saturation_ = DetectEchoSaturation(x);
+ echo_saturation_ = DetectEchoSaturation(x, EchoPathGain());
}
- // TODO(peah): Move?
- filter_has_had_time_to_converge_ =
+ bool filter_has_had_time_to_converge =
blocks_with_proper_filter_adaptation_ >= 1.5f * kNumBlocksPerSecond;
+ if (!filter_should_have_converged_) {
+ filter_should_have_converged_ =
+ blocks_with_proper_filter_adaptation_ > 6 * kNumBlocksPerSecond;
+ }
+
+ // Flag whether the initial state is still active.
initial_state_ =
blocks_with_proper_filter_adaptation_ < 5 * kNumBlocksPerSecond;
- // Flag whether the linear filter estimate is usable.
- usable_linear_estimate_ =
- !echo_saturation_ &&
- (converged_filter && filter_has_had_time_to_converge_) &&
- capture_block_counter_ >= 1.f * kNumBlocksPerSecond && !TransparentMode();
+ // Update counters for the filter divergence and convergence.
+ diverged_blocks_ = diverged_filter ? diverged_blocks_ + 1 : 0;
+ if (diverged_blocks_ >= 60) {
+ blocks_since_converged_filter_ = kBlocksSinceConvergencedFilterInit;
+ } else {
+ blocks_since_converged_filter_ =
+ converged_filter ? 0 : blocks_since_converged_filter_ + 1;
+ }
+ if (converged_filter) {
+ active_blocks_since_converged_filter_ = 0;
+ } else if (active_render_block) {
+ ++active_blocks_since_converged_filter_;
+ }
+
+ bool recently_converged_filter =
+ blocks_since_converged_filter_ < 60 * kNumBlocksPerSecond;
+
+ if (blocks_since_converged_filter_ > 20 * kNumBlocksPerSecond) {
+ converged_filter_count_ = 0;
+ } else if (converged_filter) {
+ ++converged_filter_count_;
+ }
+ if (converged_filter_count_ > 50) {
+ finite_erl_ = true;
+ }
+
+ if (filter_analyzer_.Consistent() && filter_delay_blocks_ < 5) {
+ consistent_filter_seen_ = true;
+ active_blocks_since_consistent_filter_estimate_ = 0;
+ } else if (active_render_block) {
+ ++active_blocks_since_consistent_filter_estimate_;
+ }
+
+ bool consistent_filter_estimate_not_seen;
+ if (!consistent_filter_seen_) {
+ consistent_filter_estimate_not_seen =
+ capture_block_counter_ > 5 * kNumBlocksPerSecond;
+ } else {
+ consistent_filter_estimate_not_seen =
+ active_blocks_since_consistent_filter_estimate_ >
+ 30 * kNumBlocksPerSecond;
+ }
+
+ converged_filter_seen_ = converged_filter_seen_ || converged_filter;
+
+ // If no filter convergence is seen for a long time, reset the estimated
+ // properties of the echo path.
+ if (active_blocks_since_converged_filter_ > 60 * kNumBlocksPerSecond) {
+ converged_filter_seen_ = false;
+ finite_erl_ = false;
+ }
// After an amount of active render samples for which an echo should have been
// detected in the capture signal if the ERL was not infinite, flag that a
// transparent mode should be entered.
+ transparent_mode_ = !config_.ep_strength.bounded_erl && !finite_erl_;
transparent_mode_ =
- !converged_filter &&
- (blocks_with_active_render_ == 0 ||
- blocks_with_proper_filter_adaptation_ >= 5 * kNumBlocksPerSecond);
+ transparent_mode_ &&
+ (consistent_filter_estimate_not_seen || !converged_filter_seen_);
+ transparent_mode_ = transparent_mode_ &&
+ (filter_should_have_converged_ ||
+ (!external_delay_seen_ &&
+ capture_block_counter_ > 10 * kNumBlocksPerSecond));
+
+ usable_linear_estimate_ = !echo_saturation_;
+ usable_linear_estimate_ =
+ usable_linear_estimate_ && filter_has_had_time_to_converge;
+ usable_linear_estimate_ =
+ usable_linear_estimate_ && recently_converged_filter;
+ usable_linear_estimate_ = usable_linear_estimate_ && !diverged_filter;
+ usable_linear_estimate_ = usable_linear_estimate_ && external_delay;
+
+ use_linear_filter_output_ = usable_linear_estimate_ && !TransparentMode();
+
+ data_dumper_->DumpRaw("aec3_erle", Erle());
+ data_dumper_->DumpRaw("aec3_erle_onset", erle_estimator_.ErleOnsets());
+ data_dumper_->DumpRaw("aec3_erl", Erl());
+ data_dumper_->DumpRaw("aec3_erle_time_domain", ErleTimeDomain());
+ data_dumper_->DumpRaw("aec3_erl_time_domain", ErlTimeDomain());
+ data_dumper_->DumpRaw("aec3_usable_linear_estimate", UsableLinearEstimate());
+ data_dumper_->DumpRaw("aec3_transparent_mode", transparent_mode_);
+ data_dumper_->DumpRaw("aec3_state_internal_delay",
+ internal_delay_ ? *internal_delay_ : -1);
+ data_dumper_->DumpRaw("aec3_filter_delay", filter_analyzer_.DelayBlocks());
+
+ data_dumper_->DumpRaw("aec3_consistent_filter",
+ filter_analyzer_.Consistent());
+ data_dumper_->DumpRaw("aec3_suppression_gain_limit", SuppressionGainLimit());
+ data_dumper_->DumpRaw("aec3_initial_state", InitialState());
+ data_dumper_->DumpRaw("aec3_capture_saturation", SaturatedCapture());
+ data_dumper_->DumpRaw("aec3_echo_saturation", echo_saturation_);
+ data_dumper_->DumpRaw("aec3_converged_filter", converged_filter);
+ data_dumper_->DumpRaw("aec3_diverged_filter", diverged_filter);
+
+ data_dumper_->DumpRaw("aec3_external_delay_avaliable",
+ external_delay ? 1 : 0);
+ data_dumper_->DumpRaw("aec3_consistent_filter_estimate_not_seen",
+ consistent_filter_estimate_not_seen);
+ data_dumper_->DumpRaw("aec3_filter_should_have_converged",
+ filter_should_have_converged_);
+ data_dumper_->DumpRaw("aec3_filter_has_had_time_to_converge",
+ filter_has_had_time_to_converge);
+ data_dumper_->DumpRaw("aec3_recently_converged_filter",
+ recently_converged_filter);
}
void AecState::UpdateReverb(const std::vector<float>& impulse_response) {
@@ -184,8 +276,8 @@
return;
}
- if ((!(filter_delay_ && usable_linear_estimate_)) ||
- (filter_delay_ >
+ if ((!(filter_delay_blocks_ && usable_linear_estimate_)) ||
+ (filter_delay_blocks_ >
static_cast<int>(config_.filter.main.length_blocks) - 4)) {
return;
}
@@ -371,67 +463,22 @@
kFftLengthBy2;
}
-bool AecState::DetectEchoSaturation(rtc::ArrayView<const float> x) {
+bool AecState::DetectEchoSaturation(rtc::ArrayView<const float> x,
+ float echo_path_gain) {
RTC_DCHECK_LT(0, x.size());
const float max_sample = fabs(*std::max_element(
x.begin(), x.end(), [](float a, float b) { return a * a < b * b; }));
- previous_max_sample_ = max_sample;
// Set flag for potential presence of saturated echo
- blocks_since_last_saturation_ =
- previous_max_sample_ > 200.f && SaturatedCapture()
- ? 0
- : blocks_since_last_saturation_ + 1;
-
- return blocks_since_last_saturation_ < 20;
-}
-
-void AecState::EchoAudibility::Update(rtc::ArrayView<const float> x,
- const std::array<float, kBlockSize>& s,
- bool converged_filter) {
- auto result_x = std::minmax_element(x.begin(), x.end());
- auto result_s = std::minmax_element(s.begin(), s.end());
- const float x_abs = std::max(fabsf(*result_x.first), fabsf(*result_x.second));
- const float s_abs = std::max(fabsf(*result_s.first), fabsf(*result_s.second));
-
- if (converged_filter) {
- if (x_abs < 20.f) {
- ++low_farend_counter_;
- } else {
- low_farend_counter_ = 0;
- }
+ const float kMargin = 10.f;
+ float peak_echo_amplitude = max_sample * echo_path_gain * kMargin;
+ if (SaturatedCapture() && peak_echo_amplitude > 32000) {
+ blocks_since_last_saturation_ = 0;
} else {
- if (x_abs < 100.f) {
- ++low_farend_counter_;
- } else {
- low_farend_counter_ = 0;
- }
+ ++blocks_since_last_saturation_;
}
- // The echo is deemed as not audible if the echo estimate is on the level of
- // the quantization noise in the FFTs and the nearend level is sufficiently
- // strong to mask that by ensuring that the playout and AGC gains do not boost
- // any residual echo that is below the quantization noise level. Furthermore,
- // cases where the render signal is very close to zero are also identified as
- // not producing audible echo.
- inaudible_echo_ = (max_nearend_ > 500 && s_abs < 30.f) ||
- (!converged_filter && x_abs < 500);
- inaudible_echo_ = inaudible_echo_ || low_farend_counter_ > 20;
-}
-
-void AecState::EchoAudibility::UpdateWithOutput(rtc::ArrayView<const float> e) {
- const float e_max = *std::max_element(e.begin(), e.end());
- const float e_min = *std::min_element(e.begin(), e.end());
- const float e_abs = std::max(fabsf(e_max), fabsf(e_min));
-
- if (max_nearend_ < e_abs) {
- max_nearend_ = e_abs;
- max_nearend_counter_ = 0;
- } else {
- if (++max_nearend_counter_ > 5 * kNumBlocksPerSecond) {
- max_nearend_ *= 0.995f;
- }
- }
+ return blocks_since_last_saturation_ < 5;
}
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/aec_state.h b/modules/audio_processing/aec3/aec_state.h
index 6dcd43d..ebfa62f 100644
--- a/modules/audio_processing/aec3/aec_state.h
+++ b/modules/audio_processing/aec3/aec_state.h
@@ -22,9 +22,11 @@
#include "api/optional.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
+#include "modules/audio_processing/aec3/echo_audibility.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/erl_estimator.h"
#include "modules/audio_processing/aec3/erle_estimator.h"
+#include "modules/audio_processing/aec3/filter_analyzer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/suppression_gain_limiter.h"
#include "rtc_base/constructormagic.h"
@@ -43,12 +45,27 @@
// echo.
bool UsableLinearEstimate() const { return usable_linear_estimate_; }
- // Returns whether there has been echo leakage detected.
- bool EchoLeakageDetected() const { return echo_leakage_detected_; }
+ // Returns whether the echo subtractor output should be used as output.
+ bool UseLinearFilterOutput() const { return use_linear_filter_output_; }
+
+ // Returns the estimated echo path gain.
+ bool EchoPathGain() const { return filter_analyzer_.Gain(); }
// Returns whether the render signal is currently active.
bool ActiveRender() const { return blocks_with_active_render_ > 200; }
+ // Returns the appropriate scaling of the residual echo to match the
+ // audibility.
+ void GetResidualEchoScaling(rtc::ArrayView<float> residual_scaling) const {
+ echo_audibility_.GetResidualEchoScaling(residual_scaling);
+ }
+
+ // Returns whether the stationary properties of the signals are used in the
+ // aec.
+ bool UseStationaryProperties() const {
+ return config_.echo_audibility.use_stationary_properties;
+ }
+
// Returns the ERLE.
const std::array<float, kFftLengthBy2Plus1>& Erle() const {
return erle_estimator_.Erle();
@@ -66,7 +83,10 @@
float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); }
// Returns the delay estimate based on the linear filter.
- int FilterDelay() const { return filter_delay_; }
+ int FilterDelayBlocks() const { return filter_delay_blocks_; }
+
+ // Returns the internal delay estimate based on the linear filter.
+ rtc::Optional<int> InternalDelay() const { return internal_delay_; }
// Returns whether the capture signal is saturated.
bool SaturatedCapture() const { return capture_signal_saturation_; }
@@ -74,9 +94,6 @@
// Returns whether the echo signal is saturated.
bool SaturatedEcho() const { return echo_saturation_; }
- // Returns whether the echo path can saturate.
- bool SaturatingEchoPath() const { return saturating_echo_path_; }
-
// Updates the capture signal saturation.
void UpdateCaptureSaturation(bool capture_signal_saturation) {
capture_signal_saturation_ = capture_signal_saturation;
@@ -96,14 +113,6 @@
return suppression_gain_limiter_.Limit();
}
- // Returns whether the echo in the capture signal is audible.
- bool InaudibleEcho() const { return echo_audibility_.InaudibleEcho(); }
-
- // Updates the aec state with the AEC output signal.
- void UpdateWithOutput(rtc::ArrayView<const float> e) {
- echo_audibility_.UpdateWithOutput(e);
- }
-
// Returns whether the linear filter should have been able to properly adapt.
bool FilterHasHadTimeToConverge() const {
return filter_has_had_time_to_converge_;
@@ -113,53 +122,38 @@
bool InitialState() const { return initial_state_; }
// Updates the aec state.
- void Update(const rtc::Optional<DelayEstimate>& delay_estimate,
+ void Update(const rtc::Optional<DelayEstimate>& external_delay,
const std::vector<std::array<float, kFftLengthBy2Plus1>>&
adaptive_filter_frequency_response,
const std::vector<float>& adaptive_filter_impulse_response,
bool converged_filter,
+ bool diverged_filter,
const RenderBuffer& render_buffer,
const std::array<float, kFftLengthBy2Plus1>& E2_main,
const std::array<float, kFftLengthBy2Plus1>& Y2,
- const std::array<float, kBlockSize>& s_main,
- bool echo_leakage_detected);
+ const std::array<float, kBlockSize>& s);
private:
- class EchoAudibility {
- public:
- void Update(rtc::ArrayView<const float> x,
- const std::array<float, kBlockSize>& s,
- bool converged_filter);
- void UpdateWithOutput(rtc::ArrayView<const float> e);
- bool InaudibleEcho() const { return inaudible_echo_; }
-
- private:
- float max_nearend_ = 0.f;
- size_t max_nearend_counter_ = 0;
- size_t low_farend_counter_ = 0;
- bool inaudible_echo_ = false;
- };
-
void UpdateReverb(const std::vector<float>& impulse_response);
bool DetectActiveRender(rtc::ArrayView<const float> x) const;
void UpdateSuppressorGainLimit(bool render_activity);
- bool DetectEchoSaturation(rtc::ArrayView<const float> x);
+ bool DetectEchoSaturation(rtc::ArrayView<const float> x,
+ float echo_path_gain);
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
ErlEstimator erl_estimator_;
ErleEstimator erle_estimator_;
size_t capture_block_counter_ = 0;
+ size_t blocks_since_reset_ = 0;
size_t blocks_with_proper_filter_adaptation_ = 0;
size_t blocks_with_active_render_ = 0;
bool usable_linear_estimate_ = false;
- bool echo_leakage_detected_ = false;
bool capture_signal_saturation_ = false;
bool echo_saturation_ = false;
bool transparent_mode_ = false;
- float previous_max_sample_ = 0.f;
bool render_received_ = false;
- int filter_delay_ = 0;
+ int filter_delay_blocks_ = 0;
size_t blocks_since_last_saturation_ = 1000;
float tail_energy_ = 0.f;
float accumulated_nz_ = 0.f;
@@ -171,15 +165,27 @@
bool found_end_of_reverb_decay_ = false;
bool main_filter_is_adapting_ = true;
std::array<float, kMaxAdaptiveFilterLength> block_energies_;
- EchoAudibility echo_audibility_;
const EchoCanceller3Config config_;
std::vector<float> max_render_;
float reverb_decay_ = fabsf(config_.ep_strength.default_len);
- bool saturating_echo_path_ = false;
bool filter_has_had_time_to_converge_ = false;
bool initial_state_ = true;
const float gain_rampup_increase_;
SuppressionGainUpperLimiter suppression_gain_limiter_;
+ FilterAnalyzer filter_analyzer_;
+ bool use_linear_filter_output_ = false;
+ rtc::Optional<int> internal_delay_;
+ size_t diverged_blocks_ = 0;
+ bool filter_should_have_converged_ = false;
+ size_t blocks_since_converged_filter_;
+ size_t active_blocks_since_consistent_filter_estimate_;
+ bool converged_filter_seen_ = false;
+ bool consistent_filter_seen_ = false;
+ bool external_delay_seen_ = false;
+ size_t converged_filter_count_ = 0;
+ bool finite_erl_ = false;
+ size_t active_blocks_since_converged_filter_ = 0;
+ EchoAudibility echo_audibility_;
RTC_DISALLOW_COPY_AND_ASSIGN(AecState);
};
diff --git a/modules/audio_processing/aec3/aec_state_unittest.cc b/modules/audio_processing/aec3/aec_state_unittest.cc
index 9008232..83213b5 100644
--- a/modules/audio_processing/aec3/aec_state_unittest.cc
+++ b/modules/audio_processing/aec3/aec_state_unittest.cc
@@ -22,7 +22,8 @@
ApmDataDumper data_dumper(42);
EchoCanceller3Config config;
AecState state(config);
- rtc::Optional<DelayEstimate> delay_estimate;
+ rtc::Optional<DelayEstimate> delay_estimate =
+ DelayEstimate(DelayEstimate::Quality::kRefined, 10);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 3));
std::array<float, kFftLengthBy2Plus1> E2_main = {};
@@ -49,17 +50,17 @@
// Verify that linear AEC usability is false when the filter is diverged.
state.Update(delay_estimate, diverged_filter_frequency_response,
- impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
- E2_main, Y2, s, false);
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
EXPECT_FALSE(state.UsableLinearEstimate());
// Verify that linear AEC usability is true when the filter is converged
std::fill(x[0].begin(), x[0].end(), 101.f);
for (int k = 0; k < 3000; ++k) {
render_delay_buffer->Insert(x);
- state.Update(
- delay_estimate, converged_filter_frequency_response, impulse_response,
- true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
+ state.Update(delay_estimate, converged_filter_frequency_response,
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
}
EXPECT_TRUE(state.UsableLinearEstimate());
@@ -68,8 +69,8 @@
state.HandleEchoPathChange(EchoPathVariability(
true, EchoPathVariability::DelayAdjustment::kNone, false));
state.Update(delay_estimate, converged_filter_frequency_response,
- impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
- E2_main, Y2, s, false);
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
EXPECT_FALSE(state.UsableLinearEstimate());
// Verify that the active render detection works as intended.
@@ -78,29 +79,18 @@
state.HandleEchoPathChange(EchoPathVariability(
true, EchoPathVariability::DelayAdjustment::kNewDetectedDelay, false));
state.Update(delay_estimate, converged_filter_frequency_response,
- impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
- E2_main, Y2, s, false);
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
EXPECT_FALSE(state.ActiveRender());
for (int k = 0; k < 1000; ++k) {
render_delay_buffer->Insert(x);
- state.Update(
- delay_estimate, converged_filter_frequency_response, impulse_response,
- true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
+ state.Update(delay_estimate, converged_filter_frequency_response,
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
}
EXPECT_TRUE(state.ActiveRender());
- // Verify that echo leakage is properly reported.
- state.Update(delay_estimate, converged_filter_frequency_response,
- impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
- E2_main, Y2, s, false);
- EXPECT_FALSE(state.EchoLeakageDetected());
-
- state.Update(delay_estimate, converged_filter_frequency_response,
- impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
- E2_main, Y2, s, true);
- EXPECT_TRUE(state.EchoLeakageDetected());
-
// Verify that the ERL is properly estimated
for (auto& x_k : x) {
x_k = std::vector<float>(kBlockSize, 0.f);
@@ -118,9 +108,9 @@
Y2.fill(10.f * 10000.f * 10000.f);
for (size_t k = 0; k < 1000; ++k) {
- state.Update(
- delay_estimate, converged_filter_frequency_response, impulse_response,
- true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
+ state.Update(delay_estimate, converged_filter_frequency_response,
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
}
ASSERT_TRUE(state.UsableLinearEstimate());
@@ -135,9 +125,9 @@
E2_main.fill(1.f * 10000.f * 10000.f);
Y2.fill(10.f * E2_main[0]);
for (size_t k = 0; k < 1000; ++k) {
- state.Update(
- delay_estimate, converged_filter_frequency_response, impulse_response,
- true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
+ state.Update(delay_estimate, converged_filter_frequency_response,
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
}
ASSERT_TRUE(state.UsableLinearEstimate());
{
@@ -145,7 +135,7 @@
EXPECT_EQ(erle[0], erle[1]);
constexpr size_t kLowFrequencyLimit = 32;
for (size_t k = 1; k < kLowFrequencyLimit; ++k) {
- EXPECT_NEAR(k % 2 == 0 ? 8.f : 1.f, erle[k], 0.1);
+ EXPECT_NEAR(k % 2 == 0 ? 4.f : 1.f, erle[k], 0.1);
}
for (size_t k = kLowFrequencyLimit; k < erle.size() - 1; ++k) {
EXPECT_NEAR(k % 2 == 0 ? 1.5f : 1.f, erle[k], 0.1);
@@ -156,9 +146,9 @@
E2_main.fill(1.f * 10000.f * 10000.f);
Y2.fill(5.f * E2_main[0]);
for (size_t k = 0; k < 1000; ++k) {
- state.Update(
- delay_estimate, converged_filter_frequency_response, impulse_response,
- true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
+ state.Update(delay_estimate, converged_filter_frequency_response,
+ impulse_response, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
}
ASSERT_TRUE(state.UsableLinearEstimate());
@@ -167,7 +157,7 @@
EXPECT_EQ(erle[0], erle[1]);
constexpr size_t kLowFrequencyLimit = 32;
for (size_t k = 1; k < kLowFrequencyLimit; ++k) {
- EXPECT_NEAR(k % 2 == 0 ? 5.f : 1.f, erle[k], 0.1);
+ EXPECT_NEAR(k % 2 == 0 ? 4.f : 1.f, erle[k], 0.1);
}
for (size_t k = kLowFrequencyLimit; k < erle.size() - 1; ++k) {
EXPECT_NEAR(k % 2 == 0 ? 1.5f : 1.f, erle[k], 0.1);
@@ -178,7 +168,7 @@
// Verifies the delay for a converged filter is correctly identified.
TEST(AecState, ConvergedFilterDelay) {
- constexpr int kFilterLength = 10;
+ constexpr int kFilterLengthBlocks = 10;
EchoCanceller3Config config;
AecState state(config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
@@ -194,25 +184,23 @@
x.fill(0.f);
std::vector<std::array<float, kFftLengthBy2Plus1>> frequency_response(
- kFilterLength);
+ kFilterLengthBlocks);
+ for (auto& v : frequency_response) {
+ v.fill(0.01f);
+ }
std::vector<float> impulse_response(
GetTimeDomainLength(config.filter.main.length_blocks), 0.f);
// Verify that the filter delay for a converged filter is properly identified.
- for (int k = 0; k < kFilterLength; ++k) {
- for (auto& v : frequency_response) {
- v.fill(0.01f);
- }
- frequency_response[k].fill(100.f);
- frequency_response[k][0] = 0.f;
+ for (int k = 0; k < kFilterLengthBlocks; ++k) {
+ std::fill(impulse_response.begin(), impulse_response.end(), 0.f);
+ impulse_response[k * kBlockSize + 1] = 1.f;
+
state.HandleEchoPathChange(echo_path_variability);
state.Update(delay_estimate, frequency_response, impulse_response, true,
- *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
- false);
- if (k != (kFilterLength - 1)) {
- EXPECT_EQ(k, state.FilterDelay());
- }
+ false, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2,
+ s);
}
}
diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc
index 7f702ff..f954be3 100644
--- a/modules/audio_processing/aec3/block_processor.cc
+++ b/modules/audio_processing/aec3/block_processor.cc
@@ -43,6 +43,8 @@
void GetMetrics(EchoControl::Metrics* metrics) const override;
+ void SetAudioBufferDelay(size_t delay_ms) override;
+
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
@@ -57,6 +59,7 @@
RenderDelayBuffer::BufferingEvent render_event_;
size_t capture_call_counter_ = 0;
rtc::Optional<DelayEstimate> estimated_delay_;
+ rtc::Optional<int> echo_remover_delay_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl);
};
@@ -158,7 +161,8 @@
// Compute and and apply the render delay required to achieve proper signal
// alignment.
estimated_delay_ = delay_controller_->GetDelay(
- render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]);
+ render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(),
+ echo_remover_delay_, (*capture_block)[0]);
if (estimated_delay_) {
if (render_buffer_->CausalDelay(estimated_delay_->delay)) {
@@ -191,6 +195,10 @@
echo_path_variability, capture_signal_saturation, estimated_delay_,
render_buffer_->GetRenderBuffer(), capture_block);
+ // Check to see if a refined delay estimate has been obtained from the echo
+ // remover.
+ echo_remover_delay_ = echo_remover_->Delay();
+
// Update the metrics.
metrics_.UpdateCapture(false);
@@ -228,6 +236,10 @@
metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
}
+void BlockProcessorImpl::SetAudioBufferDelay(size_t delay_ms) {
+ render_buffer_->SetAudioBufferDelay(delay_ms);
+}
+
} // namespace
BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
diff --git a/modules/audio_processing/aec3/block_processor.h b/modules/audio_processing/aec3/block_processor.h
index 8687bc2..a3967ea 100644
--- a/modules/audio_processing/aec3/block_processor.h
+++ b/modules/audio_processing/aec3/block_processor.h
@@ -42,6 +42,9 @@
// Get current metrics.
virtual void GetMetrics(EchoControl::Metrics* metrics) const = 0;
+ // Provides an optional external estimate of the audio buffer delay.
+ virtual void SetAudioBufferDelay(size_t delay_ms) = 0;
+
// Processes a block of capture data.
virtual void ProcessCapture(
bool echo_path_gain_change,
diff --git a/modules/audio_processing/aec3/block_processor_unittest.cc b/modules/audio_processing/aec3/block_processor_unittest.cc
index 87b5da9..5906018 100644
--- a/modules/audio_processing/aec3/block_processor_unittest.cc
+++ b/modules/audio_processing/aec3/block_processor_unittest.cc
@@ -166,7 +166,7 @@
EXPECT_CALL(*render_delay_buffer_mock, Delay())
.Times(kNumBlocks)
.WillRepeatedly(Return(0));
- EXPECT_CALL(*render_delay_controller_mock, GetDelay(_, _))
+ EXPECT_CALL(*render_delay_controller_mock, GetDelay(_, _, _, _))
.Times(kNumBlocks);
EXPECT_CALL(*echo_remover_mock, ProcessCapture(_, _, _, _, _))
.Times(kNumBlocks);
diff --git a/modules/audio_processing/aec3/coherence_gain.cc b/modules/audio_processing/aec3/coherence_gain.cc
new file mode 100644
index 0000000..ad33382
--- /dev/null
+++ b/modules/audio_processing/aec3/coherence_gain.cc
@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/aec3/coherence_gain.h"
+
+#include <math.h>
+
+#include <algorithm>
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+namespace {
+
+// Matlab code to produce table:
+// overDriveCurve = [sqrt(linspace(0,1,65))' + 1];
+// fprintf(1, '\t%.4f, %.4f, %.4f, %.4f, %.4f, %.4f,\n', overDriveCurve);
+const float kOverDriveCurve[kFftLengthBy2Plus1] = {
+ 1.0000f, 1.1250f, 1.1768f, 1.2165f, 1.2500f, 1.2795f, 1.3062f, 1.3307f,
+ 1.3536f, 1.3750f, 1.3953f, 1.4146f, 1.4330f, 1.4507f, 1.4677f, 1.4841f,
+ 1.5000f, 1.5154f, 1.5303f, 1.5449f, 1.5590f, 1.5728f, 1.5863f, 1.5995f,
+ 1.6124f, 1.6250f, 1.6374f, 1.6495f, 1.6614f, 1.6731f, 1.6847f, 1.6960f,
+ 1.7071f, 1.7181f, 1.7289f, 1.7395f, 1.7500f, 1.7603f, 1.7706f, 1.7806f,
+ 1.7906f, 1.8004f, 1.8101f, 1.8197f, 1.8292f, 1.8385f, 1.8478f, 1.8570f,
+ 1.8660f, 1.8750f, 1.8839f, 1.8927f, 1.9014f, 1.9100f, 1.9186f, 1.9270f,
+ 1.9354f, 1.9437f, 1.9520f, 1.9601f, 1.9682f, 1.9763f, 1.9843f, 1.9922f,
+ 2.0000f};
+
+// Matlab code to produce table:
+// weightCurve = [0 ; 0.3 * sqrt(linspace(0,1,64))' + 0.1];
+// fprintf(1, '\t%.4f, %.4f, %.4f, %.4f, %.4f, %.4f,\n', weightCurve);
+const float kWeightCurve[kFftLengthBy2Plus1] = {
+ 0.0000f, 0.1000f, 0.1378f, 0.1535f, 0.1655f, 0.1756f, 0.1845f, 0.1926f,
+ 0.2000f, 0.2069f, 0.2134f, 0.2195f, 0.2254f, 0.2309f, 0.2363f, 0.2414f,
+ 0.2464f, 0.2512f, 0.2558f, 0.2604f, 0.2648f, 0.2690f, 0.2732f, 0.2773f,
+ 0.2813f, 0.2852f, 0.2890f, 0.2927f, 0.2964f, 0.3000f, 0.3035f, 0.3070f,
+ 0.3104f, 0.3138f, 0.3171f, 0.3204f, 0.3236f, 0.3268f, 0.3299f, 0.3330f,
+ 0.3360f, 0.3390f, 0.3420f, 0.3449f, 0.3478f, 0.3507f, 0.3535f, 0.3563f,
+ 0.3591f, 0.3619f, 0.3646f, 0.3673f, 0.3699f, 0.3726f, 0.3752f, 0.3777f,
+ 0.3803f, 0.3828f, 0.3854f, 0.3878f, 0.3903f, 0.3928f, 0.3952f, 0.3976f,
+ 0.4000f};
+
+int CmpFloat(const void* a, const void* b) {
+ const float* da = static_cast<const float*>(a);
+ const float* db = static_cast<const float*>(b);
+ return (*da > *db) - (*da < *db);
+}
+
+} // namespace
+
+CoherenceGain::CoherenceGain(int sample_rate_hz, size_t num_bands_to_compute)
+ : num_bands_to_compute_(num_bands_to_compute),
+ sample_rate_scaler_(sample_rate_hz >= 16000 ? 2 : 1) {
+ spectra_.Cye.Clear();
+ spectra_.Cxy.Clear();
+ spectra_.Pe.fill(0.f);
+ // Initialize to 1 in order to prevent numerical instability in the first
+ // block.
+ spectra_.Py.fill(1.f);
+ spectra_.Px.fill(1.f);
+}
+
+CoherenceGain::~CoherenceGain() = default;
+
+void CoherenceGain::ComputeGain(const FftData& E,
+ const FftData& X,
+ const FftData& Y,
+ rtc::ArrayView<float> gain) {
+ std::array<float, kFftLengthBy2Plus1> coherence_ye;
+ std::array<float, kFftLengthBy2Plus1> coherence_xy;
+
+ UpdateCoherenceSpectra(E, X, Y);
+ ComputeCoherence(coherence_ye, coherence_xy);
+ FormSuppressionGain(coherence_ye, coherence_xy, gain);
+}
+
+// Updates the following smoothed Power Spectral Densities (PSD):
+// - sd : near-end
+// - se : residual echo
+// - sx : far-end
+// - sde : cross-PSD of near-end and residual echo
+// - sxd : cross-PSD of near-end and far-end
+//
+void CoherenceGain::UpdateCoherenceSpectra(const FftData& E,
+ const FftData& X,
+ const FftData& Y) {
+ const float s = sample_rate_scaler_ == 1 ? 0.9f : 0.92f;
+ const float one_minus_s = 1.f - s;
+ auto& c = spectra_;
+
+ for (size_t i = 0; i < c.Py.size(); i++) {
+ c.Py[i] =
+ s * c.Py[i] + one_minus_s * (Y.re[i] * Y.re[i] + Y.im[i] * Y.im[i]);
+ c.Pe[i] =
+ s * c.Pe[i] + one_minus_s * (E.re[i] * E.re[i] + E.im[i] * E.im[i]);
+ // We threshold here to protect against the ill-effects of a zero farend.
+ // The threshold is not arbitrarily chosen, but balances protection and
+ // adverse interaction with the algorithm's tuning.
+
+ // Threshold to protect against the ill-effects of a zero far-end.
+ c.Px[i] =
+ s * c.Px[i] +
+ one_minus_s * std::max(X.re[i] * X.re[i] + X.im[i] * X.im[i], 15.f);
+
+ c.Cye.re[i] =
+ s * c.Cye.re[i] + one_minus_s * (Y.re[i] * E.re[i] + Y.im[i] * E.im[i]);
+ c.Cye.im[i] =
+ s * c.Cye.im[i] + one_minus_s * (Y.re[i] * E.im[i] - Y.im[i] * E.re[i]);
+
+ c.Cxy.re[i] =
+ s * c.Cxy.re[i] + one_minus_s * (Y.re[i] * X.re[i] + Y.im[i] * X.im[i]);
+ c.Cxy.im[i] =
+ s * c.Cxy.im[i] + one_minus_s * (Y.re[i] * X.im[i] - Y.im[i] * X.re[i]);
+ }
+}
+
+void CoherenceGain::FormSuppressionGain(
+ rtc::ArrayView<const float> coherence_ye,
+ rtc::ArrayView<const float> coherence_xy,
+ rtc::ArrayView<float> gain) {
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, coherence_ye.size());
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, coherence_xy.size());
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, gain.size());
+ constexpr int kPrefBandSize = 24;
+ auto& gs = gain_state_;
+ std::array<float, kPrefBandSize> h_nl_pref;
+ float h_nl_fb = 0;
+ float h_nl_fb_low = 0;
+ const int pref_band_size = kPrefBandSize / sample_rate_scaler_;
+ const int min_pref_band = 4 / sample_rate_scaler_;
+
+ float h_nl_de_avg = 0.f;
+ float h_nl_xd_avg = 0.f;
+ for (int i = min_pref_band; i < pref_band_size + min_pref_band; ++i) {
+ h_nl_xd_avg += coherence_xy[i];
+ h_nl_de_avg += coherence_ye[i];
+ }
+ h_nl_xd_avg /= pref_band_size;
+ h_nl_xd_avg = 1 - h_nl_xd_avg;
+ h_nl_de_avg /= pref_band_size;
+
+ if (h_nl_xd_avg < 0.75f && h_nl_xd_avg < gs.h_nl_xd_avg_min) {
+ gs.h_nl_xd_avg_min = h_nl_xd_avg;
+ }
+
+ if (h_nl_de_avg > 0.98f && h_nl_xd_avg > 0.9f) {
+ gs.near_state = true;
+ } else if (h_nl_de_avg < 0.95f || h_nl_xd_avg < 0.8f) {
+ gs.near_state = false;
+ }
+
+ std::array<float, kFftLengthBy2Plus1> h_nl;
+ if (gs.h_nl_xd_avg_min == 1) {
+ gs.overdrive = 15.f;
+
+ if (gs.near_state) {
+ std::copy(coherence_ye.begin(), coherence_ye.end(), h_nl.begin());
+ h_nl_fb = h_nl_de_avg;
+ h_nl_fb_low = h_nl_de_avg;
+ } else {
+ for (size_t i = 0; i < h_nl.size(); ++i) {
+ h_nl[i] = 1 - coherence_xy[i];
+ h_nl[i] = std::max(h_nl[i], 0.f);
+ }
+ h_nl_fb = h_nl_xd_avg;
+ h_nl_fb_low = h_nl_xd_avg;
+ }
+ } else {
+ if (gs.near_state) {
+ std::copy(coherence_ye.begin(), coherence_ye.end(), h_nl.begin());
+ h_nl_fb = h_nl_de_avg;
+ h_nl_fb_low = h_nl_de_avg;
+ } else {
+ for (size_t i = 0; i < h_nl.size(); ++i) {
+ h_nl[i] = std::min(coherence_ye[i], 1 - coherence_xy[i]);
+ h_nl[i] = std::max(h_nl[i], 0.f);
+ }
+
+ // Select an order statistic from the preferred bands.
+ // TODO(peah): Using quicksort now, but a selection algorithm may be
+ // preferred.
+ std::copy(h_nl.begin() + min_pref_band,
+ h_nl.begin() + min_pref_band + pref_band_size,
+ h_nl_pref.begin());
+ std::qsort(h_nl_pref.data(), pref_band_size, sizeof(float), CmpFloat);
+
+ constexpr float kPrefBandQuant = 0.75f;
+ h_nl_fb = h_nl_pref[static_cast<int>(
+ floor(kPrefBandQuant * (pref_band_size - 1)))];
+ constexpr float kPrefBandQuantLow = 0.5f;
+ h_nl_fb_low = h_nl_pref[static_cast<int>(
+ floor(kPrefBandQuantLow * (pref_band_size - 1)))];
+ }
+ }
+
+ // Track the local filter minimum to determine suppression overdrive.
+ if (h_nl_fb_low < 0.6f && h_nl_fb_low < gs.h_nl_fb_local_min) {
+ gs.h_nl_fb_local_min = h_nl_fb_low;
+ gs.h_nl_fb_min = h_nl_fb_low;
+ gs.h_nl_new_min = 1;
+ gs.h_nl_min_ctr = 0;
+ }
+ gs.h_nl_fb_local_min =
+ std::min(gs.h_nl_fb_local_min + 0.0008f / sample_rate_scaler_, 1.f);
+ gs.h_nl_xd_avg_min =
+ std::min(gs.h_nl_xd_avg_min + 0.0006f / sample_rate_scaler_, 1.f);
+
+ if (gs.h_nl_new_min == 1) {
+ ++gs.h_nl_min_ctr;
+ }
+ if (gs.h_nl_min_ctr == 2) {
+ gs.h_nl_new_min = 0;
+ gs.h_nl_min_ctr = 0;
+ constexpr float epsilon = 1e-10f;
+ gs.overdrive = std::max(
+ -18.4f / static_cast<float>(log(gs.h_nl_fb_min + epsilon) + epsilon),
+ 15.f);
+ }
+
+ // Smooth the overdrive.
+ if (gs.overdrive < gs.overdrive_scaling) {
+ gs.overdrive_scaling = 0.99f * gs.overdrive_scaling + 0.01f * gs.overdrive;
+ } else {
+ gs.overdrive_scaling = 0.9f * gs.overdrive_scaling + 0.1f * gs.overdrive;
+ }
+
+ // Apply the overdrive.
+ RTC_DCHECK_LE(num_bands_to_compute_, gain.size());
+ for (size_t i = 0; i < num_bands_to_compute_; ++i) {
+ if (h_nl[i] > h_nl_fb) {
+ h_nl[i] = kWeightCurve[i] * h_nl_fb + (1 - kWeightCurve[i]) * h_nl[i];
+ }
+ gain[i] = powf(h_nl[i], gs.overdrive_scaling * kOverDriveCurve[i]);
+ }
+}
+
+void CoherenceGain::ComputeCoherence(rtc::ArrayView<float> coherence_ye,
+ rtc::ArrayView<float> coherence_xy) const {
+ const auto& c = spectra_;
+ constexpr float epsilon = 1e-10f;
+ for (size_t i = 0; i < coherence_ye.size(); ++i) {
+ coherence_ye[i] = (c.Cye.re[i] * c.Cye.re[i] + c.Cye.im[i] * c.Cye.im[i]) /
+ (c.Py[i] * c.Pe[i] + epsilon);
+ coherence_xy[i] = (c.Cxy.re[i] * c.Cxy.re[i] + c.Cxy.im[i] * c.Cxy.im[i]) /
+ (c.Px[i] * c.Py[i] + epsilon);
+ }
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/aec3/coherence_gain.h b/modules/audio_processing/aec3/coherence_gain.h
new file mode 100644
index 0000000..b6e22fd
--- /dev/null
+++ b/modules/audio_processing/aec3/coherence_gain.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_COHERENCE_GAIN_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_COHERENCE_GAIN_H_
+
+#include <array>
+
+#include "modules/audio_processing/aec3/aec3_common.h"
+#include "modules/audio_processing/aec3/aec3_fft.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+// Class for computing an echo suppression gain based on the coherence measure.
+class CoherenceGain {
+ public:
+ CoherenceGain(int sample_rate_hz, size_t num_bands_to_compute);
+ ~CoherenceGain();
+
+ // Computes the gain based on the FFTs of the filter error output signal, the
+ // render signal and the capture signal.
+ void ComputeGain(const FftData& E,
+ const FftData& X,
+ const FftData& Y,
+ rtc::ArrayView<float> gain);
+
+ private:
+ struct {
+ FftData Cye;
+ FftData Cxy;
+ std::array<float, kFftLengthBy2Plus1> Px;
+ std::array<float, kFftLengthBy2Plus1> Py;
+ std::array<float, kFftLengthBy2Plus1> Pe;
+ } spectra_;
+
+ struct {
+ float h_nl_fb_min = 1;
+ float h_nl_fb_local_min = 1;
+ float h_nl_xd_avg_min = 1.f;
+ int h_nl_new_min = 0;
+ float h_nl_min_ctr = 0;
+ float overdrive = 2;
+ float overdrive_scaling = 2;
+ bool near_state = false;
+ } gain_state_;
+
+ const Aec3Fft fft_;
+ const size_t num_bands_to_compute_;
+ const int sample_rate_scaler_;
+
+ // Updates the spectral estimates used for the coherence computation.
+ void UpdateCoherenceSpectra(const FftData& E,
+ const FftData& X,
+ const FftData& Y);
+
+ // Compute the suppression gain based on the coherence.
+ void FormSuppressionGain(rtc::ArrayView<const float> coherence_ye,
+ rtc::ArrayView<const float> coherence_xy,
+ rtc::ArrayView<float> h_nl);
+
+ // Compute the coherence.
+ void ComputeCoherence(rtc::ArrayView<float> coherence_ye,
+ rtc::ArrayView<float> coherence_xy) const;
+
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CoherenceGain);
+};
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AEC3_COHERENCE_GAIN_H_
diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc
new file mode 100644
index 0000000..68d2dd1
--- /dev/null
+++ b/modules/audio_processing/aec3/echo_audibility.cc
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/aec3/echo_audibility.h"
+
+#include <algorithm>
+#include <cmath>
+
+#include "modules/audio_processing/aec3/aec3_common.h"
+#include "modules/audio_processing/aec3/matrix_buffer.h"
+#include "modules/audio_processing/aec3/stationarity_estimator.h"
+#include "modules/audio_processing/aec3/vector_buffer.h"
+
+namespace webrtc {
+
+EchoAudibility::EchoAudibility() {
+ Reset();
+}
+
+EchoAudibility::~EchoAudibility() = default;
+
+void EchoAudibility::Update(const RenderBuffer& render_buffer,
+ int delay_blocks,
+ bool external_delay_seen) {
+ UpdateRenderNoiseEstimator(render_buffer.GetSpectrumBuffer(),
+ render_buffer.GetBlockBuffer(),
+ external_delay_seen);
+
+ if (external_delay_seen) {
+ UpdateRenderStationarityFlags(render_buffer, delay_blocks);
+ }
+}
+
+void EchoAudibility::Reset() {
+ render_stationarity_.Reset();
+ non_zero_render_seen_ = false;
+ render_spectrum_write_prev_ = rtc::nullopt;
+}
+
+void EchoAudibility::UpdateRenderStationarityFlags(
+ const RenderBuffer& render_buffer,
+ int delay_blocks) {
+ const VectorBuffer& spectrum_buffer = render_buffer.GetSpectrumBuffer();
+ int idx_at_delay =
+ spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks);
+
+ int num_lookahead = render_buffer.Headroom() - delay_blocks + 1;
+ num_lookahead = std::max(0, num_lookahead);
+
+ render_stationarity_.UpdateStationarityFlags(spectrum_buffer, idx_at_delay,
+ num_lookahead);
+}
+
+void EchoAudibility::UpdateRenderNoiseEstimator(
+ const VectorBuffer& spectrum_buffer,
+ const MatrixBuffer& block_buffer,
+ bool external_delay_seen) {
+ if (!render_spectrum_write_prev_) {
+ render_spectrum_write_prev_ = spectrum_buffer.write;
+ render_block_write_prev_ = block_buffer.write;
+ return;
+ }
+ int render_spectrum_write_current = spectrum_buffer.write;
+ if (!non_zero_render_seen_ && !external_delay_seen) {
+ non_zero_render_seen_ = !IsRenderTooLow(block_buffer);
+ }
+ if (non_zero_render_seen_) {
+ for (int idx = render_spectrum_write_prev_.value();
+ idx != render_spectrum_write_current;
+ idx = spectrum_buffer.DecIndex(idx)) {
+ render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]);
+ }
+ }
+ render_spectrum_write_prev_ = render_spectrum_write_current;
+}
+
+bool EchoAudibility::IsRenderTooLow(const MatrixBuffer& block_buffer) {
+ bool too_low = false;
+ const int render_block_write_current = block_buffer.write;
+ if (render_block_write_current == render_block_write_prev_) {
+ too_low = true;
+ } else {
+ for (int idx = render_block_write_prev_; idx != render_block_write_current;
+ idx = block_buffer.IncIndex(idx)) {
+ auto block = block_buffer.buffer[idx][0];
+ auto r = std::minmax_element(block.cbegin(), block.cend());
+ float max_abs = std::max(std::fabs(*r.first), std::fabs(*r.second));
+ if (max_abs < 10) {
+ too_low = true; // Discards all blocks if one of them is too low.
+ break;
+ }
+ }
+ }
+ render_block_write_prev_ = render_block_write_current;
+ return too_low;
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/aec3/echo_audibility.h b/modules/audio_processing/aec3/echo_audibility.h
new file mode 100644
index 0000000..038951e
--- /dev/null
+++ b/modules/audio_processing/aec3/echo_audibility.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_AUDIBILITY_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_AUDIBILITY_H_
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/optional.h"
+#include "modules/audio_processing/aec3/matrix_buffer.h"
+#include "modules/audio_processing/aec3/render_buffer.h"
+#include "modules/audio_processing/aec3/stationarity_estimator.h"
+#include "modules/audio_processing/aec3/vector_buffer.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+
+class EchoAudibility {
+ public:
+ EchoAudibility();
+ ~EchoAudibility();
+
+ // Feed new render data to the echo audibility estimator.
+ void Update(const RenderBuffer& render_buffer,
+ int delay_blocks,
+ bool external_delay_seen);
+
+ // Get the residual echo scaling.
+ void GetResidualEchoScaling(rtc::ArrayView<float> residual_scaling) const {
+ for (size_t band = 0; band < residual_scaling.size(); ++band) {
+ if (render_stationarity_.IsBandStationary(band)) {
+ residual_scaling[band] = 0.f;
+ } else {
+ residual_scaling[band] = 1.0f;
+ }
+ }
+ }
+
+ private:
+ // Reset the EchoAudibility class.
+ void Reset();
+
+ // Updates the render stationarity flags for the current frame.
+ void UpdateRenderStationarityFlags(const RenderBuffer& render_buffer,
+ int delay_blocks);
+
+ // Updates the noise estimator with the new render data since the previous
+ // call to this method.
+ void UpdateRenderNoiseEstimator(const VectorBuffer& spectrum_buffer,
+ const MatrixBuffer& block_buffer,
+ bool external_delay_seen);
+
+ // Returns a bool being true if the render signal contains just close to zero
+ // values.
+ bool IsRenderTooLow(const MatrixBuffer& block_buffer);
+
+ rtc::Optional<int> render_spectrum_write_prev_;
+ int render_block_write_prev_;
+ bool non_zero_render_seen_;
+ StationarityEstimator render_stationarity_;
+ RTC_DISALLOW_COPY_AND_ASSIGN(EchoAudibility);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_AUDIBILITY_H_
diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc
index f0cbbc8..765e4de 100644
--- a/modules/audio_processing/aec3/echo_canceller3.cc
+++ b/modules/audio_processing/aec3/echo_canceller3.cc
@@ -9,10 +9,9 @@
*/
#include "modules/audio_processing/aec3/echo_canceller3.h"
-#include <sstream>
-
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomicops.h"
+#include "rtc_base/logging.h"
namespace webrtc {
@@ -29,6 +28,43 @@
return false;
}
+// Method for adjusting config parameter dependencies..
+EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
+ EchoCanceller3Config adjusted_cfg = config;
+
+ // Use customized parameters when the system has clock-drift.
+ if (config.echo_removal_control.has_clock_drift) {
+ RTC_LOG(LS_WARNING)
+ << "Customizing parameters to work well for the clock-drift case.";
+ if (config.ep_strength.bounded_erl) {
+ adjusted_cfg.ep_strength.default_len = 0.85f;
+ adjusted_cfg.ep_strength.lf = 0.01f;
+ adjusted_cfg.ep_strength.mf = 0.01f;
+ adjusted_cfg.ep_strength.hf = 0.01f;
+ adjusted_cfg.echo_model.render_pre_window_size = 1;
+ adjusted_cfg.echo_model.render_post_window_size = 1;
+ adjusted_cfg.echo_model.nonlinear_hold = 3;
+ adjusted_cfg.echo_model.nonlinear_release = 0.001f;
+ } else {
+ adjusted_cfg.ep_strength.bounded_erl = true;
+ adjusted_cfg.delay.down_sampling_factor = 2;
+ adjusted_cfg.ep_strength.default_len = 0.8f;
+ adjusted_cfg.ep_strength.lf = 0.01f;
+ adjusted_cfg.ep_strength.mf = 0.01f;
+ adjusted_cfg.ep_strength.hf = 0.01f;
+ adjusted_cfg.filter.main = {30, 0.1f, 0.8f, 0.001f, 20075344.f};
+ adjusted_cfg.filter.shadow = {30, 0.7f, 20075344.f};
+ adjusted_cfg.filter.main_initial = {30, 0.1f, 1.5f, 0.001f, 20075344.f};
+ adjusted_cfg.filter.shadow_initial = {30, 0.9f, 20075344.f};
+ adjusted_cfg.echo_model.render_pre_window_size = 2;
+ adjusted_cfg.echo_model.render_post_window_size = 2;
+ adjusted_cfg.echo_model.nonlinear_hold = 3;
+ adjusted_cfg.echo_model.nonlinear_release = 0.6f;
+ }
+ }
+ return adjusted_cfg;
+}
+
void FillSubFrameView(AudioBuffer* frame,
size_t sub_frame_index,
std::vector<rtc::ArrayView<float>>* sub_frame_view) {
@@ -209,11 +245,12 @@
EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config,
int sample_rate_hz,
bool use_highpass_filter)
- : EchoCanceller3(config,
- sample_rate_hz,
- use_highpass_filter,
- std::unique_ptr<BlockProcessor>(
- BlockProcessor::Create(config, sample_rate_hz))) {}
+ : EchoCanceller3(
+ AdjustConfig(config),
+ sample_rate_hz,
+ use_highpass_filter,
+ std::unique_ptr<BlockProcessor>(
+ BlockProcessor::Create(AdjustConfig(config), sample_rate_hz))) {}
EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config,
int sample_rate_hz,
bool use_highpass_filter,
@@ -338,6 +375,11 @@
return metrics;
}
+void EchoCanceller3::SetAudioBufferDelay(size_t delay_ms) {
+ RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
+ block_processor_->SetAudioBufferDelay(delay_ms);
+}
+
void EchoCanceller3::EmptyRenderQueue() {
RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
bool frame_to_buffer =
diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h
index 8658814..b66b8b1 100644
--- a/modules/audio_processing/aec3/echo_canceller3.h
+++ b/modules/audio_processing/aec3/echo_canceller3.h
@@ -82,6 +82,8 @@
void ProcessCapture(AudioBuffer* capture, bool level_change) override;
// Collect current metrics from the echo canceller.
Metrics GetMetrics() const override;
+ // Provides an optional external estimate of the audio buffer delay.
+ void SetAudioBufferDelay(size_t delay_ms) override;
// Signals whether an external detector has detected echo leakage from the
// echo canceller.
diff --git a/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/modules/audio_processing/aec3/echo_canceller3_unittest.cc
index d4ad4f6..f652642 100644
--- a/modules/audio_processing/aec3/echo_canceller3_unittest.cc
+++ b/modules/audio_processing/aec3/echo_canceller3_unittest.cc
@@ -104,6 +104,8 @@
void GetMetrics(EchoControl::Metrics* metrics) const override {}
+ void SetAudioBufferDelay(size_t delay_ms) override{};
+
private:
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CaptureTransportVerificationProcessor);
};
@@ -132,6 +134,8 @@
void GetMetrics(EchoControl::Metrics* metrics) const override {}
+ void SetAudioBufferDelay(size_t delay_ms) override{};
+
private:
std::deque<std::vector<std::vector<float>>> received_render_blocks_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderTransportVerificationProcessor);
diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc
index da1fa4b..96887fe 100644
--- a/modules/audio_processing/aec3/echo_remover.cc
+++ b/modules/audio_processing/aec3/echo_remover.cc
@@ -22,7 +22,6 @@
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/echo_remover_metrics.h"
#include "modules/audio_processing/aec3/fft_data.h"
-#include "modules/audio_processing/aec3/output_selector.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/residual_echo_estimator.h"
@@ -46,11 +45,20 @@
}
}
+// Computes a windowed (square root Hanning) padded FFT and updates the related
+// memory.
+void WindowedPaddedFft(const Aec3Fft& fft,
+ rtc::ArrayView<const float> v,
+ rtc::ArrayView<float> v_old,
+ FftData* V) {
+ fft.PaddedFft(v, v_old, Aec3Fft::Window::kSqrtHanning, V);
+ std::copy(v.begin(), v.end(), v_old.begin());
+}
+
// Class for removing the echo from the capture signal.
class EchoRemoverImpl final : public EchoRemover {
public:
- explicit EchoRemoverImpl(const EchoCanceller3Config& config,
- int sample_rate_hz);
+ EchoRemoverImpl(const EchoCanceller3Config& config, int sample_rate_hz);
~EchoRemoverImpl() override;
void GetMetrics(EchoControl::Metrics* metrics) const override;
@@ -60,10 +68,15 @@
// signal.
void ProcessCapture(const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
- const rtc::Optional<DelayEstimate>& delay_estimate,
+ const rtc::Optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture) override;
+ // Returns the internal delay estimate in blocks.
+ rtc::Optional<int> Delay() const override {
+ return aec_state_.InternalDelay();
+ }
+
// Updates the status on whether echo leakage is detected in the output of the
// echo remover.
void UpdateEchoLeakageStatus(bool leakage_detected) override {
@@ -82,12 +95,14 @@
ComfortNoiseGenerator cng_;
SuppressionFilter suppression_filter_;
RenderSignalAnalyzer render_signal_analyzer_;
- OutputSelector output_selector_;
ResidualEchoEstimator residual_echo_estimator_;
bool echo_leakage_detected_ = false;
AecState aec_state_;
EchoRemoverMetrics metrics_;
bool initial_state_ = true;
+ std::array<float, kFftLengthBy2> e_old_;
+ std::array<float, kFftLengthBy2> x_old_;
+ std::array<float, kFftLengthBy2> y_old_;
RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl);
};
@@ -103,13 +118,16 @@
optimization_(DetectOptimization()),
sample_rate_hz_(sample_rate_hz),
subtractor_(config, data_dumper_.get(), optimization_),
- suppression_gain_(config_, optimization_),
+ suppression_gain_(config_, optimization_, sample_rate_hz),
cng_(optimization_),
suppression_filter_(sample_rate_hz_),
render_signal_analyzer_(config_),
residual_echo_estimator_(config_),
aec_state_(config_) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
+ x_old_.fill(0.f);
+ y_old_.fill(0.f);
+ e_old_.fill(0.f);
}
EchoRemoverImpl::~EchoRemoverImpl() = default;
@@ -124,7 +142,7 @@
void EchoRemoverImpl::ProcessCapture(
const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
- const rtc::Optional<DelayEstimate>& delay_estimate,
+ const rtc::Optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture) {
const std::vector<std::vector<float>>& x = render_buffer->Block(0);
@@ -155,21 +173,20 @@
}
std::array<float, kFftLengthBy2Plus1> Y2;
+ std::array<float, kFftLengthBy2Plus1> E2;
std::array<float, kFftLengthBy2Plus1> R2;
std::array<float, kFftLengthBy2Plus1> S2_linear;
std::array<float, kFftLengthBy2Plus1> G;
float high_bands_gain;
FftData Y;
+ FftData E;
FftData comfort_noise;
FftData high_band_comfort_noise;
SubtractorOutput subtractor_output;
- FftData& E_main_nonwindowed = subtractor_output.E_main_nonwindowed;
- auto& E2_main = subtractor_output.E2_main_nonwindowed;
- auto& E2_shadow = subtractor_output.E2_shadow;
- auto& e_main = subtractor_output.e_main;
// Analyze the render signal.
- render_signal_analyzer_.Update(*render_buffer, aec_state_.FilterDelay());
+ render_signal_analyzer_.Update(*render_buffer,
+ aec_state_.FilterDelayBlocks());
// Perform linear echo cancellation.
if (initial_state_ && !aec_state_.InitialState()) {
@@ -177,27 +194,46 @@
suppression_gain_.SetInitialState(false);
initial_state_ = false;
}
+
+ // If the delay is known, use the echo subtractor.
subtractor_.Process(*render_buffer, y0, render_signal_analyzer_, aec_state_,
&subtractor_output);
+ const auto& e = subtractor_output.e_main;
// Compute spectra.
- // fft_.ZeroPaddedFft(y0, Aec3Fft::Window::kHanning, &Y);
- fft_.ZeroPaddedFft(y0, Aec3Fft::Window::kRectangular, &Y);
- LinearEchoPower(E_main_nonwindowed, Y, &S2_linear);
+ WindowedPaddedFft(fft_, y0, y_old_, &Y);
+ WindowedPaddedFft(fft_, e, e_old_, &E);
+ LinearEchoPower(E, Y, &S2_linear);
Y.Spectrum(optimization_, Y2);
+ E.Spectrum(optimization_, E2);
// Update the AEC state information.
- aec_state_.Update(delay_estimate, subtractor_.FilterFrequencyResponse(),
+ aec_state_.Update(external_delay, subtractor_.FilterFrequencyResponse(),
subtractor_.FilterImpulseResponse(),
- subtractor_.ConvergedFilter(), *render_buffer, E2_main, Y2,
- subtractor_output.s_main, echo_leakage_detected_);
+ subtractor_.ConvergedFilter(), subtractor_.DivergedFilter(),
+ *render_buffer, E2, Y2, subtractor_output.s_main);
+
+ // Compute spectra.
+ const bool suppression_gain_uses_ffts =
+ config_.suppressor.bands_with_reliable_coherence > 0;
+ FftData X;
+ if (suppression_gain_uses_ffts) {
+ auto& x_aligned = render_buffer->Block(-aec_state_.FilterDelayBlocks())[0];
+ WindowedPaddedFft(fft_, x_aligned, x_old_, &X);
+ } else {
+ X.Clear();
+ }
// Choose the linear output.
- output_selector_.FormLinearOutput(!aec_state_.TransparentMode(), e_main, y0);
+ data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0],
+ LowestBandRate(sample_rate_hz_), 1);
+ if (aec_state_.UseLinearFilterOutput()) {
+ std::copy(e.begin(), e.end(), y0.begin());
+ }
+ const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y;
+
data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0],
LowestBandRate(sample_rate_hz_), 1);
- data_dumper_->DumpRaw("aec3_output_linear", y0);
- const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2;
// Estimate the residual echo power.
residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2,
@@ -206,19 +242,19 @@
// Estimate the comfort noise.
cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise);
- // A choose and apply echo suppression gain.
- suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(),
+
+
+ // Compute and apply the suppression gain.
+ suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(), E, X, Y,
render_signal_analyzer_, aec_state_, x,
&high_bands_gain, &G);
+
suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
- high_bands_gain, y);
+ high_bands_gain, Y_fft, y);
// Update the metrics.
metrics_.Update(aec_state_, cng_.NoiseSpectrum(), G);
- // Update the aec state with the aec output characteristics.
- aec_state_.UpdateWithOutput(y0);
-
// Debug outputs for the purpose of development and analysis.
data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize,
&subtractor_output.s_main[0],
@@ -232,19 +268,14 @@
rtc::ArrayView<const float>(&y0[0], kBlockSize),
LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpRaw("aec3_using_subtractor_output",
- output_selector_.UseSubtractorOutput() ? 1 : 0);
+ aec_state_.UseLinearFilterOutput() ? 1 : 0);
data_dumper_->DumpRaw("aec3_E2", E2);
- data_dumper_->DumpRaw("aec3_E2_main", E2_main);
- data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow);
data_dumper_->DumpRaw("aec3_S2_linear", S2_linear);
data_dumper_->DumpRaw("aec3_Y2", Y2);
- data_dumper_->DumpRaw("aec3_X2", render_buffer->Spectrum(0));
+ data_dumper_->DumpRaw(
+ "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks()));
data_dumper_->DumpRaw("aec3_R2", R2);
- data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle());
- data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl());
- data_dumper_->DumpRaw("aec3_usable_linear_estimate",
- aec_state_.UsableLinearEstimate());
- data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelay());
+ data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks());
data_dumper_->DumpRaw("aec3_capture_saturation",
aec_state_.SaturatedCapture() ? 1 : 0);
}
diff --git a/modules/audio_processing/aec3/echo_remover.h b/modules/audio_processing/aec3/echo_remover.h
index 08fc3db..61d2999 100644
--- a/modules/audio_processing/aec3/echo_remover.h
+++ b/modules/audio_processing/aec3/echo_remover.h
@@ -38,10 +38,13 @@
virtual void ProcessCapture(
const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
- const rtc::Optional<DelayEstimate>& delay_estimate,
+ const rtc::Optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture) = 0;
+ // Returns the internal delay estimate in blocks.
+ virtual rtc::Optional<int> Delay() const = 0;
+
// Updates the status on whether echo leakage is detected in the output of the
// echo remover.
virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0;
diff --git a/modules/audio_processing/aec3/echo_remover_metrics.cc b/modules/audio_processing/aec3/echo_remover_metrics.cc
index bc815eb..c970649 100644
--- a/modules/audio_processing/aec3/echo_remover_metrics.cc
+++ b/modules/audio_processing/aec3/echo_remover_metrics.cc
@@ -237,7 +237,7 @@
static_cast<int>(
active_render_count_ > kMetricsCollectionBlocksBy2 ? 1 : 0));
RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.EchoCanceller.FilterDelay",
- aec_state.FilterDelay(), 0, 30, 31);
+ aec_state.FilterDelayBlocks(), 0, 30, 31);
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.EchoCanceller.CaptureSaturation",
static_cast<int>(saturated_capture_ ? 1 : 0));
break;
diff --git a/modules/audio_processing/aec3/erle_estimator.cc b/modules/audio_processing/aec3/erle_estimator.cc
index 0e4cbe1..18763cb 100644
--- a/modules/audio_processing/aec3/erle_estimator.cc
+++ b/modules/audio_processing/aec3/erle_estimator.cc
@@ -24,7 +24,9 @@
max_erle_lf_(max_erle_lf),
max_erle_hf_(max_erle_hf) {
erle_.fill(min_erle_);
+ erle_onsets_.fill(min_erle_);
hold_counters_.fill(0);
+ coming_onset_.fill(true);
erle_time_domain_ = min_erle_;
hold_counter_time_domain_ = 0;
}
@@ -43,29 +45,55 @@
// Corresponds of WGN of power -46 dBFS.
constexpr float kX2Min = 44015068.0f;
+ constexpr int kOnsetSizeBlocks = 4;
+ constexpr int kErleHold = 100;
+ constexpr int kErleOnsetHold = kErleHold + kOnsetSizeBlocks;
+
+ auto erle_band_update = [](float erle_band, float new_erle, float alpha_inc,
+ float alpha_dec, float min_erle, float max_erle) {
+ float alpha = new_erle > erle_band ? alpha_inc : alpha_dec;
+ float erle_band_out = erle_band;
+ erle_band_out = erle_band + alpha * (new_erle - erle_band);
+ erle_band_out = rtc::SafeClamp(erle_band_out, min_erle, max_erle);
+ return erle_band_out;
+ };
// Update the estimates in a clamped minimum statistics manner.
auto erle_update = [&](size_t start, size_t stop, float max_erle) {
for (size_t k = start; k < stop; ++k) {
if (X2[k] > kX2Min && E2[k] > 0.f) {
const float new_erle = Y2[k] / E2[k];
- if (new_erle > erle_[k]) {
- hold_counters_[k - 1] = 100;
- erle_[k] += 0.1f * (new_erle - erle_[k]);
- erle_[k] = rtc::SafeClamp(erle_[k], min_erle_, max_erle);
+
+ if (coming_onset_[k - 1]) {
+ hold_counters_[k - 1] = kErleOnsetHold;
+ coming_onset_[k - 1] = false;
}
+ if (hold_counters_[k - 1] > kErleHold) {
+ erle_onsets_[k] = erle_band_update(erle_onsets_[k], new_erle, 0.05f,
+ 0.1f, min_erle_, max_erle);
+ } else {
+ hold_counters_[k - 1] = kErleHold;
+ }
+ erle_[k] = erle_band_update(erle_[k], new_erle, 0.01f, 0.02f, min_erle_,
+ max_erle);
}
}
};
- erle_update(1, kFftLengthBy2 / 2, max_erle_lf_);
- erle_update(kFftLengthBy2 / 2, kFftLengthBy2, max_erle_hf_);
- std::for_each(hold_counters_.begin(), hold_counters_.end(),
- [](int& a) { --a; });
- std::transform(hold_counters_.begin(), hold_counters_.end(),
- erle_.begin() + 1, erle_.begin() + 1, [&](int a, float b) {
- return a > 0 ? b : std::max(min_erle_, 0.97f * b);
- });
+ constexpr size_t kFftLengthBy4 = kFftLengthBy2 / 2;
+ erle_update(1, kFftLengthBy4, max_erle_lf_);
+ erle_update(kFftLengthBy4, kFftLengthBy2, max_erle_hf_);
+
+ for (size_t k = 0; k < hold_counters_.size(); ++k) {
+ hold_counters_[k]--;
+ if (hold_counters_[k] <= 0) {
+ coming_onset_[k] = true;
+ if (erle_[k + 1] > erle_onsets_[k + 1]) {
+ erle_[k + 1] = std::max(erle_onsets_[k + 1], 0.97f * erle_[k + 1]);
+ RTC_DCHECK_LE(min_erle_, erle_[k + 1]);
+ }
+ }
+ }
erle_[0] = erle_[1];
erle_[kFftLengthBy2] = erle_[kFftLengthBy2 - 1];
@@ -77,7 +105,7 @@
const float Y2_sum = std::accumulate(Y2.begin(), Y2.end(), 0.0f);
const float new_erle = Y2_sum / E2_sum;
if (new_erle > erle_time_domain_) {
- hold_counter_time_domain_ = 100;
+ hold_counter_time_domain_ = kErleHold;
erle_time_domain_ += 0.1f * (new_erle - erle_time_domain_);
erle_time_domain_ =
rtc::SafeClamp(erle_time_domain_, min_erle_, max_erle_lf_);
diff --git a/modules/audio_processing/aec3/erle_estimator.h b/modules/audio_processing/aec3/erle_estimator.h
index cb9fce6..809466c 100644
--- a/modules/audio_processing/aec3/erle_estimator.h
+++ b/modules/audio_processing/aec3/erle_estimator.h
@@ -32,10 +32,16 @@
// Returns the most recent ERLE estimate.
const std::array<float, kFftLengthBy2Plus1>& Erle() const { return erle_; }
+ // Returns the ERLE that is estimated during onsets. Use for logging/testing.
+ const std::array<float, kFftLengthBy2Plus1>& ErleOnsets() const {
+ return erle_onsets_;
+ }
float ErleTimeDomain() const { return erle_time_domain_; }
private:
std::array<float, kFftLengthBy2Plus1> erle_;
+ std::array<float, kFftLengthBy2Plus1> erle_onsets_;
+ std::array<bool, kFftLengthBy2Minus1> coming_onset_;
std::array<int, kFftLengthBy2Minus1> hold_counters_;
float erle_time_domain_;
int hold_counter_time_domain_;
diff --git a/modules/audio_processing/aec3/erle_estimator_unittest.cc b/modules/audio_processing/aec3/erle_estimator_unittest.cc
index f3dd7d9..9ccdb20 100644
--- a/modules/audio_processing/aec3/erle_estimator_unittest.cc
+++ b/modules/audio_processing/aec3/erle_estimator_unittest.cc
@@ -16,52 +16,109 @@
namespace {
constexpr int kLowFrequencyLimit = kFftLengthBy2 / 2;
+constexpr float kMaxErleLf = 8.f;
+constexpr float kMaxErleHf = 1.5f;
+constexpr float kMinErle = 1.0f;
+constexpr float kTrueErle = 10.f;
+constexpr float kTrueErleOnsets = 1.0f;
-void VerifyErle(const std::array<float, kFftLengthBy2Plus1>& erle,
- float erle_time_domain,
- float reference_lf,
- float reference_hf) {
+void VerifyErleBands(const std::array<float, kFftLengthBy2Plus1>& erle,
+ float reference_lf,
+ float reference_hf) {
std::for_each(
erle.begin(), erle.begin() + kLowFrequencyLimit,
[reference_lf](float a) { EXPECT_NEAR(reference_lf, a, 0.001); });
std::for_each(
erle.begin() + kLowFrequencyLimit, erle.end(),
[reference_hf](float a) { EXPECT_NEAR(reference_hf, a, 0.001); });
+}
+
+void VerifyErle(const std::array<float, kFftLengthBy2Plus1>& erle,
+ float erle_time_domain,
+ float reference_lf,
+ float reference_hf) {
+ VerifyErleBands(erle, reference_lf, reference_hf);
EXPECT_NEAR(reference_lf, erle_time_domain, 0.001);
}
+void FormFarendFrame(std::array<float, kFftLengthBy2Plus1>* X2,
+ std::array<float, kFftLengthBy2Plus1>* E2,
+ std::array<float, kFftLengthBy2Plus1>* Y2,
+ float erle) {
+ X2->fill(500 * 1000.f * 1000.f);
+ E2->fill(1000.f * 1000.f);
+ Y2->fill(erle * (*E2)[0]);
+}
+
+void FormNearendFrame(std::array<float, kFftLengthBy2Plus1>* X2,
+ std::array<float, kFftLengthBy2Plus1>* E2,
+ std::array<float, kFftLengthBy2Plus1>* Y2) {
+ X2->fill(0.f);
+ Y2->fill(500.f * 1000.f * 1000.f);
+ E2->fill((*Y2)[0]);
+}
+
} // namespace
-// Verifies that the correct ERLE estimates are achieved.
-TEST(ErleEstimator, Estimates) {
+TEST(ErleEstimator, VerifyErleIncreaseAndHold) {
std::array<float, kFftLengthBy2Plus1> X2;
std::array<float, kFftLengthBy2Plus1> E2;
std::array<float, kFftLengthBy2Plus1> Y2;
- ErleEstimator estimator(1.f, 8.f, 1.5f);
+ ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf);
- // Verifies that the ERLE estimate is properley increased to higher values.
- X2.fill(500 * 1000.f * 1000.f);
- E2.fill(1000.f * 1000.f);
- Y2.fill(10 * E2[0]);
+ // Verifies that the ERLE estimate is properly increased to higher values.
+ FormFarendFrame(&X2, &E2, &Y2, kTrueErle);
+
for (size_t k = 0; k < 200; ++k) {
estimator.Update(X2, Y2, E2);
}
VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f);
- // Verifies that the ERLE is not immediately decreased when the ERLE in the
- // data decreases.
- Y2.fill(0.1f * E2[0]);
+ FormNearendFrame(&X2, &E2, &Y2);
+ // Verifies that the ERLE is not immediately decreased during nearend
+ // activity.
for (size_t k = 0; k < 98; ++k) {
estimator.Update(X2, Y2, E2);
}
VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f);
+}
- // Verifies that the minimum ERLE is eventually achieved.
- for (size_t k = 0; k < 1000; ++k) {
+TEST(ErleEstimator, VerifyErleTrackingOnOnsets) {
+ std::array<float, kFftLengthBy2Plus1> X2;
+ std::array<float, kFftLengthBy2Plus1> E2;
+ std::array<float, kFftLengthBy2Plus1> Y2;
+
+ ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf);
+
+ for (size_t burst = 0; burst < 20; ++burst) {
+ FormFarendFrame(&X2, &E2, &Y2, kTrueErleOnsets);
+ for (size_t k = 0; k < 10; ++k) {
+ estimator.Update(X2, Y2, E2);
+ }
+ FormFarendFrame(&X2, &E2, &Y2, kTrueErle);
+ for (size_t k = 0; k < 200; ++k) {
+ estimator.Update(X2, Y2, E2);
+ }
+ FormNearendFrame(&X2, &E2, &Y2);
+ for (size_t k = 0; k < 100; ++k) {
+ estimator.Update(X2, Y2, E2);
+ }
+ }
+ VerifyErleBands(estimator.ErleOnsets(), kMinErle, kMinErle);
+ FormNearendFrame(&X2, &E2, &Y2);
+ for (size_t k = 0; k < 1000; k++) {
estimator.Update(X2, Y2, E2);
}
- VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f);
+ // Verifies that during ne activity, Erle converges to the Erle for onsets.
+ VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), kMinErle, kMinErle);
+}
+
+TEST(ErleEstimator, VerifyNoErleUpdateDuringLowActivity) {
+ std::array<float, kFftLengthBy2Plus1> X2;
+ std::array<float, kFftLengthBy2Plus1> E2;
+ std::array<float, kFftLengthBy2Plus1> Y2;
+ ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf);
// Verifies that the ERLE estimate is is not updated for low-level render
// signals.
@@ -70,6 +127,7 @@
for (size_t k = 0; k < 200; ++k) {
estimator.Update(X2, Y2, E2);
}
- VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f);
+ VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), kMinErle, kMinErle);
}
+
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/filter_analyzer.cc b/modules/audio_processing/aec3/filter_analyzer.cc
new file mode 100644
index 0000000..363373c
--- /dev/null
+++ b/modules/audio_processing/aec3/filter_analyzer.cc
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/aec3/filter_analyzer.h"
+#include <math.h>
+
+#include <algorithm>
+#include <array>
+#include <numeric>
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+
+size_t FindPeakIndex(rtc::ArrayView<const float> filter_time_domain) {
+ size_t peak_index = 0;
+ float max_h2 = filter_time_domain[0] * filter_time_domain[0];
+ for (size_t k = 1; k < filter_time_domain.size(); ++k) {
+ float tmp = filter_time_domain[k] * filter_time_domain[k];
+ if (tmp > max_h2) {
+ peak_index = k;
+ max_h2 = tmp;
+ }
+ }
+
+ return peak_index;
+}
+
+} // namespace
+
+FilterAnalyzer::FilterAnalyzer(const EchoCanceller3Config& config)
+ : bounded_erl_(config.ep_strength.bounded_erl),
+ default_gain_(config.ep_strength.lf),
+ active_render_threshold_(config.render_levels.active_render_limit *
+ config.render_levels.active_render_limit *
+ kFftLengthBy2) {
+ Reset();
+}
+
+FilterAnalyzer::~FilterAnalyzer() = default;
+
+void FilterAnalyzer::Reset() {
+ delay_blocks_ = 0;
+ consistent_estimate_ = false;
+ blocks_since_reset_ = 0;
+ consistent_estimate_ = false;
+ consistent_estimate_counter_ = 0;
+ consistent_delay_reference_ = -10;
+ gain_ = default_gain_;
+}
+
+void FilterAnalyzer::Update(rtc::ArrayView<const float> filter_time_domain,
+ const RenderBuffer& render_buffer) {
+ size_t peak_index = FindPeakIndex(filter_time_domain);
+ delay_blocks_ = peak_index / kBlockSize;
+
+ UpdateFilterGain(filter_time_domain, peak_index);
+
+ float filter_floor = 0;
+ float filter_secondary_peak = 0;
+ size_t limit1 = peak_index < 64 ? 0 : peak_index - 64;
+ size_t limit2 =
+ peak_index > filter_time_domain.size() - 129 ? 0 : peak_index + 128;
+
+ for (size_t k = 0; k < limit1; ++k) {
+ float abs_h = fabsf(filter_time_domain[k]);
+ filter_floor += abs_h;
+ filter_secondary_peak = std::max(filter_secondary_peak, abs_h);
+ }
+ for (size_t k = limit2; k < filter_time_domain.size(); ++k) {
+ float abs_h = fabsf(filter_time_domain[k]);
+ filter_floor += abs_h;
+ filter_secondary_peak = std::max(filter_secondary_peak, abs_h);
+ }
+
+ filter_floor /= (limit1 + filter_time_domain.size() - limit2);
+
+ float abs_peak = fabsf(filter_time_domain[peak_index]);
+ bool significant_peak_index =
+ abs_peak > 10.f * filter_floor && abs_peak > 2.f * filter_secondary_peak;
+
+ if (consistent_delay_reference_ != delay_blocks_ || !significant_peak_index) {
+ consistent_estimate_counter_ = 0;
+ consistent_delay_reference_ = delay_blocks_;
+ } else {
+ const auto& x = render_buffer.Block(-delay_blocks_)[0];
+ const float x_energy =
+ std::inner_product(x.begin(), x.end(), x.begin(), 0.f);
+ const bool active_render_block = x_energy > active_render_threshold_;
+
+ if (active_render_block) {
+ ++consistent_estimate_counter_;
+ }
+ }
+
+ consistent_estimate_ =
+ consistent_estimate_counter_ > 1.5f * kNumBlocksPerSecond;
+}
+
+void FilterAnalyzer::UpdateFilterGain(
+ rtc::ArrayView<const float> filter_time_domain,
+ size_t peak_index) {
+ bool sufficient_time_to_converge =
+ ++blocks_since_reset_ > 5 * kNumBlocksPerSecond;
+
+ if (sufficient_time_to_converge && consistent_estimate_) {
+ gain_ = fabsf(filter_time_domain[peak_index]);
+ } else {
+ if (gain_) {
+ gain_ = std::max(gain_, fabsf(filter_time_domain[peak_index]));
+ }
+ }
+
+ if (bounded_erl_ && gain_) {
+ gain_ = std::max(gain_, 0.01f);
+ }
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/aec3/filter_analyzer.h b/modules/audio_processing/aec3/filter_analyzer.h
new file mode 100644
index 0000000..f02a210
--- /dev/null
+++ b/modules/audio_processing/aec3/filter_analyzer.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_FILTER_ANALYZER_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_FILTER_ANALYZER_H_
+
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/audio/echo_canceller3_config.h"
+#include "api/optional.h"
+#include "modules/audio_processing/aec3/aec3_common.h"
+#include "modules/audio_processing/aec3/render_buffer.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+// Class for analyzing the properties of an adaptive filter.
+class FilterAnalyzer {
+ public:
+ explicit FilterAnalyzer(const EchoCanceller3Config& config);
+ ~FilterAnalyzer();
+
+ // Resets the analysis.
+ void Reset();
+
+ // Updates the estimates with new input data.
+ void Update(rtc::ArrayView<const float> filter_time_domain,
+ const RenderBuffer& render_buffer);
+
+ // Returns the delay of the filter in terms of blocks.
+ int DelayBlocks() const { return delay_blocks_; }
+
+ // Returns whether the filter is consistent in the sense that it does not
+ // change much over time.
+ bool Consistent() const { return consistent_estimate_; }
+
+ // Returns the estimated filter gain.
+ float Gain() const { return gain_; }
+
+ private:
+ void UpdateFilterGain(rtc::ArrayView<const float> filter_time_domain,
+ size_t max_index);
+
+ const bool bounded_erl_;
+ const float default_gain_;
+ const float active_render_threshold_;
+
+ int delay_blocks_ = 0;
+ size_t blocks_since_reset_ = 0;
+ bool consistent_estimate_ = false;
+ size_t consistent_estimate_counter_ = 0;
+ int consistent_delay_reference_ = -10;
+ float gain_;
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(FilterAnalyzer);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AEC3_FILTER_ANALYZER_H_
diff --git a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc
index 13747d4..3d0a8c3 100644
--- a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc
+++ b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc
@@ -114,7 +114,7 @@
render_delay_buffer->PrepareCaptureProcessing();
render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(),
- aec_state.FilterDelay());
+ aec_state.FilterDelayBlocks());
// Apply the main filter.
main_filter.Filter(*render_delay_buffer->GetRenderBuffer(), &S);
@@ -162,9 +162,8 @@
aec_state.HandleEchoPathChange(EchoPathVariability(
false, EchoPathVariability::DelayAdjustment::kNone, false));
aec_state.Update(delay_estimate, main_filter.FilterFrequencyResponse(),
- main_filter.FilterImpulseResponse(), true,
- *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
- false);
+ main_filter.FilterImpulseResponse(), true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
}
std::copy(e_main.begin(), e_main.end(), e_last_block->begin());
diff --git a/modules/audio_processing/aec3/matched_filter.h b/modules/audio_processing/aec3/matched_filter.h
index c9bdc46..36c9cad 100644
--- a/modules/audio_processing/aec3/matched_filter.h
+++ b/modules/audio_processing/aec3/matched_filter.h
@@ -15,6 +15,7 @@
#include <memory>
#include <vector>
+#include "api/array_view.h"
#include "api/optional.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
diff --git a/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc b/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc
index 9041924..23cd71a 100644
--- a/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc
+++ b/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc
@@ -70,8 +70,6 @@
if (histogram_[candidate] > 25) {
significant_candidate_found_ = true;
return DelayEstimate(DelayEstimate::Quality::kRefined, candidate);
- } else if (!significant_candidate_found_) {
- return DelayEstimate(DelayEstimate::Quality::kCoarse, candidate);
}
}
return rtc::nullopt;
diff --git a/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc b/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc
index ce303d4..18b8829 100644
--- a/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc
+++ b/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc
@@ -22,7 +22,7 @@
namespace webrtc {
namespace {
-constexpr size_t kNumLagsBeforeDetection = 25;
+constexpr size_t kNumLagsBeforeDetection = 26;
} // namespace
@@ -37,7 +37,7 @@
lag_estimates[1] = MatchedFilter::LagEstimate(0.5f, true, kLag2, true);
for (size_t k = 0; k < kNumLagsBeforeDetection; ++k) {
- EXPECT_TRUE(aggregator.Aggregate(lag_estimates));
+ aggregator.Aggregate(lag_estimates);
}
rtc::Optional<DelayEstimate> aggregated_lag =
diff --git a/modules/audio_processing/aec3/mock/mock_block_processor.h b/modules/audio_processing/aec3/mock/mock_block_processor.h
index 5fff456..803bf78 100644
--- a/modules/audio_processing/aec3/mock/mock_block_processor.h
+++ b/modules/audio_processing/aec3/mock/mock_block_processor.h
@@ -31,6 +31,7 @@
void(const std::vector<std::vector<float>>& block));
MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected));
MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics));
+ MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms));
};
} // namespace test
diff --git a/modules/audio_processing/aec3/mock/mock_echo_remover.h b/modules/audio_processing/aec3/mock/mock_echo_remover.h
index 638e3f0..0acf139 100644
--- a/modules/audio_processing/aec3/mock/mock_echo_remover.h
+++ b/modules/audio_processing/aec3/mock/mock_echo_remover.h
@@ -32,7 +32,7 @@
const rtc::Optional<DelayEstimate>& delay_estimate,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture));
-
+ MOCK_CONST_METHOD0(Delay, rtc::Optional<int>());
MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected));
MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics));
};
diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h
index 1ed2b40..00c13f4 100644
--- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h
+++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h
@@ -47,12 +47,13 @@
const std::vector<std::vector<float>>& block));
MOCK_METHOD0(PrepareCaptureProcessing, RenderDelayBuffer::BufferingEvent());
MOCK_METHOD1(SetDelay, bool(size_t delay));
- MOCK_CONST_METHOD0(Delay, rtc::Optional<size_t>());
+ MOCK_CONST_METHOD0(Delay, size_t());
MOCK_CONST_METHOD0(MaxDelay, size_t());
MOCK_METHOD0(GetRenderBuffer, RenderBuffer*());
MOCK_CONST_METHOD0(GetDownsampledRenderBuffer,
const DownsampledRenderBuffer&());
MOCK_CONST_METHOD1(CausalDelay, bool(size_t delay));
+ MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms));
private:
RenderBuffer* FakeGetRenderBuffer() { return &render_buffer_; }
diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h
index 8fb7a8e..fab2b65 100644
--- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h
+++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h
@@ -26,9 +26,11 @@
MOCK_METHOD0(Reset, void());
MOCK_METHOD0(LogRenderCall, void());
- MOCK_METHOD2(
+ MOCK_METHOD4(
GetDelay,
rtc::Optional<DelayEstimate>(const DownsampledRenderBuffer& render_buffer,
+ size_t render_delay_buffer_delay,
+ const rtc::Optional<int>& echo_remover_delay,
rtc::ArrayView<const float> capture));
};
diff --git a/modules/audio_processing/aec3/output_selector.cc b/modules/audio_processing/aec3/output_selector.cc
deleted file mode 100644
index 4f547d9..0000000
--- a/modules/audio_processing/aec3/output_selector.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "modules/audio_processing/aec3/output_selector.h"
-
-#include <algorithm>
-#include <numeric>
-
-#include "rtc_base/checks.h"
-
-namespace webrtc {
-namespace {
-
-// Performs the transition between the signals in a smooth manner.
-void SmoothFrameTransition(bool from_y_to_e,
- rtc::ArrayView<const float> e,
- rtc::ArrayView<float> y) {
- RTC_DCHECK_LT(0u, e.size());
- RTC_DCHECK_EQ(y.size(), e.size());
-
- const float change_factor = (from_y_to_e ? 1.f : -1.f) / e.size();
- float averaging = from_y_to_e ? 0.f : 1.f;
- for (size_t k = 0; k < e.size(); ++k) {
- y[k] += averaging * (e[k] - y[k]);
- averaging += change_factor;
- }
- RTC_DCHECK_EQ(from_y_to_e ? 1.f : 0.f, averaging);
-}
-
-} // namespace
-
-OutputSelector::OutputSelector() = default;
-
-OutputSelector::~OutputSelector() = default;
-
-void OutputSelector::FormLinearOutput(
- bool use_subtractor_output,
- rtc::ArrayView<const float> subtractor_output,
- rtc::ArrayView<float> capture) {
- RTC_DCHECK_EQ(subtractor_output.size(), capture.size());
- rtc::ArrayView<const float>& e_main = subtractor_output;
- rtc::ArrayView<float> y = capture;
-
- if (use_subtractor_output != use_subtractor_output_) {
- use_subtractor_output_ = use_subtractor_output;
- SmoothFrameTransition(use_subtractor_output_, e_main, y);
- } else if (use_subtractor_output_) {
- std::copy(e_main.begin(), e_main.end(), y.begin());
- }
-}
-
-} // namespace webrtc
diff --git a/modules/audio_processing/aec3/output_selector.h b/modules/audio_processing/aec3/output_selector.h
deleted file mode 100644
index a406c61..0000000
--- a/modules/audio_processing/aec3/output_selector.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef MODULES_AUDIO_PROCESSING_AEC3_OUTPUT_SELECTOR_H_
-#define MODULES_AUDIO_PROCESSING_AEC3_OUTPUT_SELECTOR_H_
-
-#include "api/array_view.h"
-#include "rtc_base/constructormagic.h"
-
-namespace webrtc {
-
-// Performs the selection between which of the linear aec output and the
-// microphone signal should be used as the echo suppressor output.
-class OutputSelector {
- public:
- OutputSelector();
- ~OutputSelector();
-
- // Forms the most appropriate output signal.
- void FormLinearOutput(bool use_subtractor_output,
- rtc::ArrayView<const float> subtractor_output,
- rtc::ArrayView<float> capture);
-
- // Returns true if the linear aec output is the one used.
- bool UseSubtractorOutput() const { return use_subtractor_output_; }
-
- private:
- bool use_subtractor_output_ = false;
- RTC_DISALLOW_COPY_AND_ASSIGN(OutputSelector);
-};
-
-} // namespace webrtc
-
-#endif // MODULES_AUDIO_PROCESSING_AEC3_OUTPUT_SELECTOR_H_
diff --git a/modules/audio_processing/aec3/output_selector_unittest.cc b/modules/audio_processing/aec3/output_selector_unittest.cc
deleted file mode 100644
index c7add1c..0000000
--- a/modules/audio_processing/aec3/output_selector_unittest.cc
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "modules/audio_processing/aec3/output_selector.h"
-
-#include <algorithm>
-#include <array>
-
-#include "modules/audio_processing/aec3/aec3_common.h"
-#include "test/gtest.h"
-
-namespace webrtc {
-
-// Verifies that the switching between the signals in the output works as
-// intended.
-TEST(OutputSelector, ProperSwitching) {
- OutputSelector selector;
-
- std::array<float, kBlockSize> y;
- std::array<float, kBlockSize> e;
- std::array<float, kBlockSize> e_ref;
- std::array<float, kBlockSize> y_ref;
- auto init_blocks = [](std::array<float, kBlockSize>* e,
- std::array<float, kBlockSize>* y) {
- e->fill(10.f);
- y->fill(20.f);
- };
-
- init_blocks(&e_ref, &y_ref);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(false, e, y);
- EXPECT_EQ(y_ref, y);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(true, e, y);
- EXPECT_NE(e_ref, y);
- EXPECT_NE(y_ref, y);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(true, e, y);
- EXPECT_EQ(e_ref, y);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(true, e, y);
- EXPECT_EQ(e_ref, y);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(false, e, y);
- EXPECT_NE(e_ref, y);
- EXPECT_NE(y_ref, y);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(false, e, y);
- EXPECT_EQ(y_ref, y);
-
- init_blocks(&e, &y);
- selector.FormLinearOutput(false, e, y);
- EXPECT_EQ(y_ref, y);
-}
-
-} // namespace webrtc
diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h
index 7789ffd..9419b30 100644
--- a/modules/audio_processing/aec3/render_buffer.h
+++ b/modules/audio_processing/aec3/render_buffer.h
@@ -67,6 +67,28 @@
// Specifies the recent activity seen in the render signal.
void SetRenderActivity(bool activity) { render_activity_ = activity; }
+ // Returns the headroom between the write and the read positions in the
+ // buffer.
+ int Headroom() const {
+ // The write and read indices are decreased over time.
+ int headroom =
+ fft_buffer_->write < fft_buffer_->read
+ ? fft_buffer_->read - fft_buffer_->write
+ : fft_buffer_->size - fft_buffer_->write + fft_buffer_->read;
+
+ RTC_DCHECK_LE(0, headroom);
+ RTC_DCHECK_GE(fft_buffer_->size, headroom);
+
+ return headroom;
+ }
+
+
+ // Returns a reference to the spectrum buffer.
+ const VectorBuffer& GetSpectrumBuffer() const { return *spectrum_buffer_; }
+
+ // Returns a reference to the block buffer.
+ const MatrixBuffer& GetBlockBuffer() const { return *block_buffer_; }
+
private:
const MatrixBuffer* const block_buffer_;
const VectorBuffer* const spectrum_buffer_;
diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc
index 60606bf..5aa432d 100644
--- a/modules/audio_processing/aec3/render_delay_buffer.cc
+++ b/modules/audio_processing/aec3/render_delay_buffer.cc
@@ -38,7 +38,7 @@
BufferingEvent Insert(const std::vector<std::vector<float>>& block) override;
BufferingEvent PrepareCaptureProcessing() override;
bool SetDelay(size_t delay) override;
- rtc::Optional<size_t> Delay() const override { return delay_; }
+ size_t Delay() const override { return MapInternalDelayToExternalDelay(); }
size_t MaxDelay() const override {
return blocks_.buffer.size() - 1 - buffer_headroom_;
}
@@ -50,6 +50,8 @@
bool CausalDelay(size_t delay) const override;
+ void SetAudioBufferDelay(size_t delay_ms) override;
+
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
@@ -75,9 +77,12 @@
size_t render_call_counter_ = 0;
bool render_activity_ = false;
size_t render_activity_counter_ = 0;
+ rtc::Optional<size_t> external_audio_buffer_delay_ms_;
+ bool external_delay_verified_after_reset_ = false;
int LowRateBufferOffset() const { return DelayEstimatorOffset(config_) >> 1; }
- int MaxExternalDelayToInternalDelay(size_t delay) const;
+ int MapExternalDelayToInternalDelay(size_t external_delay_blocks) const;
+ int MapInternalDelayToExternalDelay() const;
void ApplyDelay(int delay);
void InsertBlock(const std::vector<std::vector<float>>& block,
int previous_write);
@@ -167,7 +172,7 @@
kBlockSize),
spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1),
ffts_(blocks_.buffer.size()),
- delay_(config_.delay.min_echo_path_delay_blocks),
+ delay_(config_.delay.default_delay),
echo_remover_buffer_(&blocks_, &spectra_, &ffts_),
low_rate_(GetDownSampledBufferSize(config.delay.down_sampling_factor,
config.delay.num_filters)),
@@ -196,12 +201,33 @@
low_rate_.read = low_rate_.OffsetIndex(
low_rate_.write, LowRateBufferOffset() * sub_block_size_);
- // Set the render buffer delays to the default delay.
- ApplyDelay(config_.delay.default_delay);
+ // Check for any external audio buffer delay and whether it is feasible.
+ if (external_audio_buffer_delay_ms_) {
+ constexpr size_t kHeadroom = 5;
+ size_t external_delay_to_set = 0;
+ if (*external_audio_buffer_delay_ms_ < kHeadroom) {
+ external_delay_to_set = 0;
+ } else {
+ external_delay_to_set = *external_audio_buffer_delay_ms_ - kHeadroom;
+ }
- // Unset the delays which are set by ApplyConfig.
- delay_ = rtc::nullopt;
- internal_delay_ = rtc::nullopt;
+ constexpr size_t kMaxExternalDelay = 170;
+ external_delay_to_set = std::min(external_delay_to_set, kMaxExternalDelay);
+
+ // When an external delay estimate is available, use that delay as the
+ // initial render buffer delay. Avoid verifying the set delay.
+ external_delay_verified_after_reset_ = true;
+ SetDelay(external_delay_to_set);
+ external_delay_verified_after_reset_ = false;
+ } else {
+ // If an external delay estimate is not available, use that delay as the
+ // initial delay. Set the render buffer delays to the default delay.
+ ApplyDelay(config_.delay.default_delay);
+
+ // Unset the delays which are set by SetDelay.
+ delay_ = rtc::nullopt;
+ internal_delay_ = rtc::nullopt;
+ }
}
// Inserts a new block into the render buffers.
@@ -215,7 +241,7 @@
} else {
if (++num_api_calls_in_a_row_ > max_observed_jitter_) {
max_observed_jitter_ = num_api_calls_in_a_row_;
- RTC_LOG(LS_INFO)
+ RTC_LOG(LS_WARNING)
<< "New max number api jitter observed at render block "
<< render_call_counter_ << ": " << num_api_calls_in_a_row_
<< " blocks";
@@ -263,7 +289,7 @@
} else {
if (++num_api_calls_in_a_row_ > max_observed_jitter_) {
max_observed_jitter_ = num_api_calls_in_a_row_;
- RTC_LOG(LS_INFO)
+ RTC_LOG(LS_WARNING)
<< "New max number api jitter observed at capture block "
<< capture_call_counter_ << ": " << num_api_calls_in_a_row_
<< " blocks";
@@ -304,13 +330,22 @@
// Sets the delay and returns a bool indicating whether the delay was changed.
bool RenderDelayBufferImpl::SetDelay(size_t delay) {
+ if (!external_delay_verified_after_reset_ &&
+ external_audio_buffer_delay_ms_) {
+ int delay_difference = static_cast<int>(*external_audio_buffer_delay_ms_) -
+ static_cast<int>(delay);
+ RTC_LOG(LS_WARNING) << "Difference between the externally reported delay "
+ "and the first delay estimate: "
+ << delay_difference << " ms.";
+ external_delay_verified_after_reset_ = true;
+ }
if (delay_ && *delay_ == delay) {
return false;
}
delay_ = delay;
// Compute the internal delay and limit the delay to the allowed range.
- int internal_delay = MaxExternalDelayToInternalDelay(*delay_);
+ int internal_delay = MapExternalDelayToInternalDelay(*delay_);
internal_delay_ =
std::min(MaxDelay(), static_cast<size_t>(std::max(internal_delay, 0)));
@@ -322,7 +357,7 @@
// Returns whether the specified delay is causal.
bool RenderDelayBufferImpl::CausalDelay(size_t delay) const {
// Compute the internal delay and limit the delay to the allowed range.
- int internal_delay = MaxExternalDelayToInternalDelay(delay);
+ int internal_delay = MapExternalDelayToInternalDelay(delay);
internal_delay =
std::min(MaxDelay(), static_cast<size_t>(std::max(internal_delay, 0)));
@@ -330,8 +365,17 @@
static_cast<int>(config_.delay.min_echo_path_delay_blocks);
}
+void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) {
+ if (!external_audio_buffer_delay_ms_) {
+ RTC_LOG(LS_WARNING)
+ << "Receiving a first reported externally buffer delay of " << delay_ms
+ << " ms.";
+ }
+ external_audio_buffer_delay_ms_ = delay_ms;
+}
+
// Maps the externally computed delay to the delay used internally.
-int RenderDelayBufferImpl::MaxExternalDelayToInternalDelay(
+int RenderDelayBufferImpl::MapExternalDelayToInternalDelay(
size_t external_delay_blocks) const {
const int latency = BufferLatency(low_rate_);
RTC_DCHECK_LT(0, sub_block_size_);
@@ -341,8 +385,20 @@
DelayEstimatorOffset(config_);
}
+// Maps the internally used delay to the delay used externally.
+int RenderDelayBufferImpl::MapInternalDelayToExternalDelay() const {
+ const int latency = BufferLatency(low_rate_);
+ int latency_blocks = latency / sub_block_size_;
+ int internal_delay = spectra_.read >= spectra_.write
+ ? spectra_.read - spectra_.write
+ : spectra_.size + spectra_.read - spectra_.write;
+
+ return internal_delay - latency_blocks + DelayEstimatorOffset(config_);
+}
+
// Set the read indices according to the delay.
void RenderDelayBufferImpl::ApplyDelay(int delay) {
+ RTC_LOG(LS_WARNING) << "Applying internal delay of " << delay << " blocks.";
blocks_.read = blocks_.OffsetIndex(blocks_.write, -delay);
spectra_.read = spectra_.OffsetIndex(spectra_.write, delay);
ffts_.read = ffts_.OffsetIndex(ffts_.write, delay);
diff --git a/modules/audio_processing/aec3/render_delay_buffer.h b/modules/audio_processing/aec3/render_delay_buffer.h
index 22b0c7f..e361628 100644
--- a/modules/audio_processing/aec3/render_delay_buffer.h
+++ b/modules/audio_processing/aec3/render_delay_buffer.h
@@ -57,7 +57,7 @@
virtual bool SetDelay(size_t delay) = 0;
// Gets the buffer delay.
- virtual rtc::Optional<size_t> Delay() const = 0;
+ virtual size_t Delay() const = 0;
// Gets the buffer delay.
virtual size_t MaxDelay() const = 0;
@@ -73,6 +73,9 @@
// Returns the maximum non calusal offset that can occur in the delay buffer.
static int DelayEstimatorOffset(const EchoCanceller3Config& config);
+
+ // Provides an optional external estimate of the audio buffer delay.
+ virtual void SetAudioBufferDelay(size_t delay_ms) = 0;
};
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc
index fb9c48d..78f0b5a 100644
--- a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc
+++ b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc
@@ -75,12 +75,14 @@
EchoCanceller3Config config;
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create(config, 1));
- ASSERT_FALSE(delay_buffer->Delay());
- for (size_t delay = config.delay.min_echo_path_delay_blocks + 1; delay < 20;
- ++delay) {
- delay_buffer->SetDelay(delay);
- ASSERT_TRUE(delay_buffer->Delay());
- EXPECT_EQ(delay, *delay_buffer->Delay());
+ ASSERT_TRUE(delay_buffer->Delay());
+ delay_buffer->Reset();
+ size_t initial_internal_delay = config.delay.min_echo_path_delay_blocks +
+ config.delay.api_call_jitter_blocks;
+ for (size_t delay = initial_internal_delay;
+ delay < initial_internal_delay + 20; ++delay) {
+ ASSERT_TRUE(delay_buffer->SetDelay(delay));
+ EXPECT_EQ(delay, delay_buffer->Delay());
}
}
diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc
index db00b9b..fc91108 100644
--- a/modules/audio_processing/aec3/render_delay_controller.cc
+++ b/modules/audio_processing/aec3/render_delay_controller.cc
@@ -40,6 +40,8 @@
void LogRenderCall() override;
rtc::Optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
+ size_t render_delay_buffer_delay,
+ const rtc::Optional<int>& echo_remover_delay,
rtc::ArrayView<const float> capture) override;
private:
@@ -146,6 +148,8 @@
rtc::Optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
const DownsampledRenderBuffer& render_buffer,
+ size_t render_delay_buffer_delay,
+ const rtc::Optional<int>& echo_remover_delay,
rtc::ArrayView<const float> capture) {
RTC_DCHECK_EQ(kBlockSize, capture.size());
++capture_call_counter_;
@@ -157,6 +161,14 @@
auto delay_samples =
delay_estimator_.EstimateDelay(render_buffer, capture_delayed);
+ // Overrule the delay estimator delay if the echo remover reports a delay.
+ if (echo_remover_delay) {
+ int total_echo_remover_delay_samples =
+ (render_delay_buffer_delay + *echo_remover_delay) * kBlockSize;
+ delay_samples = DelayEstimate(DelayEstimate::Quality::kRefined,
+ total_echo_remover_delay_samples);
+ }
+
std::copy(capture.begin(), capture.end(),
delay_buf_.begin() + delay_buf_index_);
delay_buf_index_ = (delay_buf_index_ + kBlockSize) % delay_buf_.size();
@@ -165,6 +177,9 @@
rtc::Optional<int> skew = skew_estimator_.GetSkewFromCapture();
if (delay_samples) {
+ // TODO(peah): Refactor the rest of the code to assume a kRefined estimate
+ // quality.
+ RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality);
if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
delay_change_counter_ = 0;
}
diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h
index 24d7590..1e1df0d 100644
--- a/modules/audio_processing/aec3/render_delay_controller.h
+++ b/modules/audio_processing/aec3/render_delay_controller.h
@@ -38,6 +38,8 @@
// Aligns the render buffer content with the capture signal.
virtual rtc::Optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
+ size_t render_delay_buffer_delay,
+ const rtc::Optional<int>& echo_remover_delay,
rtc::ArrayView<const float> capture) = 0;
};
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/modules/audio_processing/aec3/render_delay_controller_unittest.cc
index 656c5e8..2c9bbef 100644
--- a/modules/audio_processing/aec3/render_delay_controller_unittest.cc
+++ b/modules/audio_processing/aec3/render_delay_controller_unittest.cc
@@ -48,6 +48,7 @@
TEST(RenderDelayController, NoRenderSignal) {
std::vector<float> block(kBlockSize, 0.f);
EchoCanceller3Config config;
+ rtc::Optional<int> echo_remover_delay_;
for (size_t num_matched_filters = 4; num_matched_filters == 10;
num_matched_filters++) {
for (auto down_sampling_factor : kDownSamplingFactors) {
@@ -62,7 +63,8 @@
config, RenderDelayBuffer::DelayEstimatorOffset(config), rate));
for (size_t k = 0; k < 100; ++k) {
auto delay = delay_controller->GetDelay(
- delay_buffer->GetDownsampledRenderBuffer(), block);
+ delay_buffer->GetDownsampledRenderBuffer(), delay_buffer->Delay(),
+ echo_remover_delay_, block);
EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay->delay);
}
}
@@ -74,6 +76,7 @@
TEST(RenderDelayController, BasicApiCalls) {
std::vector<float> capture_block(kBlockSize, 0.f);
rtc::Optional<DelayEstimate> delay_blocks;
+ rtc::Optional<int> echo_remover_delay;
for (size_t num_matched_filters = 4; num_matched_filters == 10;
num_matched_filters++) {
for (auto down_sampling_factor : kDownSamplingFactors) {
@@ -94,7 +97,8 @@
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
- render_delay_buffer->GetDownsampledRenderBuffer(), capture_block);
+ render_delay_buffer->GetDownsampledRenderBuffer(),
+ render_delay_buffer->Delay(), echo_remover_delay, capture_block);
}
EXPECT_TRUE(delay_blocks);
EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay_blocks->delay);
@@ -107,6 +111,7 @@
// simple timeshifts between the signals.
TEST(RenderDelayController, Alignment) {
Random random_generator(42U);
+ rtc::Optional<int> echo_remover_delay;
std::vector<float> capture_block(kBlockSize, 0.f);
for (size_t num_matched_filters = 4; num_matched_filters == 10;
num_matched_filters++) {
@@ -136,6 +141,7 @@
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
render_delay_buffer->GetDownsampledRenderBuffer(),
+ render_delay_buffer->Delay(), echo_remover_delay,
capture_block);
}
ASSERT_TRUE(!!delay_blocks);
@@ -156,6 +162,7 @@
// delays.
TEST(RenderDelayController, NonCausalAlignment) {
Random random_generator(42U);
+ rtc::Optional<int> echo_remover_delay;
for (size_t num_matched_filters = 4; num_matched_filters == 10;
num_matched_filters++) {
for (auto down_sampling_factor : kDownSamplingFactors) {
@@ -186,6 +193,7 @@
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
render_delay_buffer->GetDownsampledRenderBuffer(),
+ render_delay_buffer->Delay(), echo_remover_delay,
capture_block[0]);
}
@@ -200,6 +208,7 @@
// simple timeshifts between the signals when there is jitter in the API calls.
TEST(RenderDelayController, AlignmentWithJitter) {
Random random_generator(42U);
+ rtc::Optional<int> echo_remover_delay;
std::vector<float> capture_block(kBlockSize, 0.f);
for (size_t num_matched_filters = 4; num_matched_filters == 10;
num_matched_filters++) {
@@ -237,6 +246,7 @@
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
render_delay_buffer->GetDownsampledRenderBuffer(),
+ render_delay_buffer->Delay(), echo_remover_delay,
capture_block_buffer[k]);
}
}
@@ -286,6 +296,7 @@
TEST(RenderDelayController, WrongCaptureSize) {
std::vector<float> block(kBlockSize - 1, 0.f);
EchoCanceller3Config config;
+ rtc::Optional<int> echo_remover_delay;
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
@@ -296,7 +307,7 @@
EchoCanceller3Config(),
RenderDelayBuffer::DelayEstimatorOffset(config), rate))
->GetDelay(render_delay_buffer->GetDownsampledRenderBuffer(),
- block),
+ render_delay_buffer->Delay(), echo_remover_delay, block),
"");
}
}
diff --git a/modules/audio_processing/aec3/residual_echo_estimator.cc b/modules/audio_processing/aec3/residual_echo_estimator.cc
index f0c971d..4b6c959 100644
--- a/modules/audio_processing/aec3/residual_echo_estimator.cc
+++ b/modules/audio_processing/aec3/residual_echo_estimator.cc
@@ -1,3 +1,4 @@
+
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
@@ -16,65 +17,6 @@
#include "rtc_base/checks.h"
namespace webrtc {
-namespace {
-
-// Estimates the echo generating signal power as gated maximal power over a time
-// window.
-void EchoGeneratingPower(const RenderBuffer& render_buffer,
- size_t min_delay,
- size_t max_delay,
- std::array<float, kFftLengthBy2Plus1>* X2) {
- X2->fill(0.f);
- for (size_t k = min_delay; k <= max_delay; ++k) {
- std::transform(X2->begin(), X2->end(), render_buffer.Spectrum(k).begin(),
- X2->begin(),
- [](float a, float b) { return std::max(a, b); });
- }
-
- // Apply soft noise gate of -78 dBFS.
- static constexpr float kNoiseGatePower = 27509.42f;
- std::for_each(X2->begin(), X2->end(), [](float& a) {
- if (kNoiseGatePower > a) {
- a = std::max(0.f, a - 0.3f * (kNoiseGatePower - a));
- }
- });
-}
-
-constexpr int kNoiseFloorCounterMax = 50;
-constexpr float kNoiseFloorMin = 10.f * 10.f * 128.f * 128.f;
-
-// Updates estimate for the power of the stationary noise component in the
-// render signal.
-void RenderNoisePower(
- const RenderBuffer& render_buffer,
- std::array<float, kFftLengthBy2Plus1>* X2_noise_floor,
- std::array<int, kFftLengthBy2Plus1>* X2_noise_floor_counter) {
- RTC_DCHECK(X2_noise_floor);
- RTC_DCHECK(X2_noise_floor_counter);
-
- const auto render_power = render_buffer.Spectrum(0);
- RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size());
- RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size());
-
- // Estimate the stationary noise power in a minimum statistics manner.
- for (size_t k = 0; k < render_power.size(); ++k) {
- // Decrease rapidly.
- if (render_power[k] < (*X2_noise_floor)[k]) {
- (*X2_noise_floor)[k] = render_power[k];
- (*X2_noise_floor_counter)[k] = 0;
- } else {
- // Increase in a delayed, leaky manner.
- if ((*X2_noise_floor_counter)[k] >= kNoiseFloorCounterMax) {
- (*X2_noise_floor)[k] =
- std::max((*X2_noise_floor)[k] * 1.1f, kNoiseFloorMin);
- } else {
- ++(*X2_noise_floor_counter)[k];
- }
- }
- }
-}
-
-} // namespace
ResidualEchoEstimator::ResidualEchoEstimator(const EchoCanceller3Config& config)
: config_(config), S2_old_(config_.filter.main.length_blocks) {
@@ -96,44 +38,59 @@
// Estimate the residual echo power.
if (aec_state.UsableLinearEstimate()) {
- LinearEstimate(S2_linear, aec_state.Erle(), aec_state.FilterDelay(), R2);
- AddEchoReverb(S2_linear, aec_state.SaturatedEcho(), aec_state.FilterDelay(),
+ RTC_DCHECK(!aec_state.SaturatedEcho());
+ LinearEstimate(S2_linear, aec_state.Erle(), R2);
+ AddEchoReverb(S2_linear, aec_state.FilterDelayBlocks(),
aec_state.ReverbDecay(), R2);
+ } else {
+ // Estimate the echo generating signal power.
+ std::array<float, kFftLengthBy2Plus1> X2;
+
+ // Computes the spectral power over the blocks surrounding the delay.
+ size_t window_start = std::max(
+ 0, aec_state.FilterDelayBlocks() -
+ static_cast<int>(config_.echo_model.render_pre_window_size));
+ size_t window_end =
+ aec_state.FilterDelayBlocks() +
+ static_cast<int>(config_.echo_model.render_post_window_size);
+ EchoGeneratingPower(render_buffer, window_start, window_end, &X2);
+
+ // TODO(devicentepena): look if this is competing/completing
+ // with the stationarity estimator
+ // Subtract the stationary noise power to avoid stationary noise causing
+ // excessive echo suppression.
+ std::transform(X2.begin(), X2.end(), X2_noise_floor_.begin(), X2.begin(),
+ [&](float a, float b) {
+ return std::max(
+ 0.f, a - config_.echo_model.stationary_gate_slope * b);
+ });
+
+ NonLinearEstimate(aec_state.EchoPathGain(), X2, Y2, R2);
// If the echo is saturated, estimate the echo power as the maximum echo
// power with a leakage factor.
if (aec_state.SaturatedEcho()) {
R2->fill((*std::max_element(R2->begin(), R2->end())) * 100.f);
}
- } else {
- // Estimate the echo generating signal power.
- std::array<float, kFftLengthBy2Plus1> X2;
- // Computes the spectral power over the blocks surrounding the delay.
- EchoGeneratingPower(render_buffer, std::max(0, aec_state.FilterDelay() - 1),
- aec_state.FilterDelay() + 10, &X2);
+ AddEchoReverb(*R2, config_.filter.main.length_blocks,
+ aec_state.ReverbDecay(), R2);
+ }
- // Subtract the stationary noise power to avoid stationary noise causing
- // excessive echo suppression.
- std::transform(
- X2.begin(), X2.end(), X2_noise_floor_.begin(), X2.begin(),
- [](float a, float b) { return std::max(0.f, a - 10.f * b); });
-
- NonLinearEstimate(aec_state.FilterHasHadTimeToConverge(),
- aec_state.SaturatedEcho(),
- config_.ep_strength.bounded_erl,
- aec_state.TransparentMode(), X2, Y2, R2);
-
- if (aec_state.SaturatedEcho()) {
- // TODO(peah): Modify to make sense theoretically.
- AddEchoReverb(*R2, aec_state.SaturatedEcho(),
- config_.filter.main.length_blocks, aec_state.ReverbDecay(),
- R2);
+ if (aec_state.UseStationaryProperties()) {
+ // Scale the echo according to echo audibility.
+ std::array<float, kFftLengthBy2Plus1> residual_scaling;
+ aec_state.GetResidualEchoScaling(residual_scaling);
+ for (size_t k = 0; k < R2->size(); ++k) {
+ (*R2)[k] *= residual_scaling[k];
+ if (residual_scaling[k] == 0.f) {
+ R2_hold_counter_[k] = 0;
+ }
}
}
// If the echo is deemed inaudible, set the residual echo to zero.
- if (aec_state.InaudibleEcho()) {
+ if (aec_state.TransparentMode()) {
R2->fill(0.f);
R2_old_.fill(0.f);
R2_hold_counter_.fill(0.f);
@@ -143,8 +100,8 @@
}
void ResidualEchoEstimator::Reset() {
- X2_noise_floor_counter_.fill(kNoiseFloorCounterMax);
- X2_noise_floor_.fill(kNoiseFloorMin);
+ X2_noise_floor_counter_.fill(config_.echo_model.noise_floor_hold);
+ X2_noise_floor_.fill(config_.echo_model.min_noise_floor_power);
R2_reverb_.fill(0.f);
R2_old_.fill(0.f);
R2_hold_counter_.fill(0.f);
@@ -156,7 +113,6 @@
void ResidualEchoEstimator::LinearEstimate(
const std::array<float, kFftLengthBy2Plus1>& S2_linear,
const std::array<float, kFftLengthBy2Plus1>& erle,
- size_t delay,
std::array<float, kFftLengthBy2Plus1>* R2) {
std::fill(R2_hold_counter_.begin(), R2_hold_counter_.end(), 10.f);
std::transform(erle.begin(), erle.end(), S2_linear.begin(), R2->begin(),
@@ -167,46 +123,15 @@
}
void ResidualEchoEstimator::NonLinearEstimate(
- bool sufficient_filter_updates,
- bool saturated_echo,
- bool bounded_erl,
- bool transparent_mode,
+ float echo_path_gain,
const std::array<float, kFftLengthBy2Plus1>& X2,
const std::array<float, kFftLengthBy2Plus1>& Y2,
std::array<float, kFftLengthBy2Plus1>* R2) {
- float echo_path_gain_lf;
- float echo_path_gain_mf;
- float echo_path_gain_hf;
-
- // Set echo path gains.
- if (saturated_echo) {
- // If the echo could be saturated, use a very conservative gain.
- echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 10000.f;
- } else if (sufficient_filter_updates && !bounded_erl) {
- // If the filter should have been able to converge, and no assumption is
- // possible on the ERL, use a low gain.
- echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.01f;
- } else if ((sufficient_filter_updates && bounded_erl) || transparent_mode) {
- // If the filter should have been able to converge, and and it is known that
- // the ERL is bounded, use a very low gain.
- echo_path_gain_lf = echo_path_gain_mf = echo_path_gain_hf = 0.001f;
- } else {
- // In the initial state, use conservative gains.
- echo_path_gain_lf = config_.ep_strength.lf;
- echo_path_gain_mf = config_.ep_strength.mf;
- echo_path_gain_hf = config_.ep_strength.hf;
- }
// Compute preliminary residual echo.
- std::transform(
- X2.begin(), X2.begin() + 12, R2->begin(),
- [echo_path_gain_lf](float a) { return a * echo_path_gain_lf; });
- std::transform(
- X2.begin() + 12, X2.begin() + 25, R2->begin() + 12,
- [echo_path_gain_mf](float a) { return a * echo_path_gain_mf; });
- std::transform(
- X2.begin() + 25, X2.end(), R2->begin() + 25,
- [echo_path_gain_hf](float a) { return a * echo_path_gain_hf; });
+ std::transform(X2.begin(), X2.end(), R2->begin(), [echo_path_gain](float a) {
+ return a * echo_path_gain * echo_path_gain;
+ });
for (size_t k = 0; k < R2->size(); ++k) {
// Update hold counter.
@@ -214,15 +139,17 @@
// Compute the residual echo by holding a maximum echo powers and an echo
// fading corresponding to a room with an RT60 value of about 50 ms.
- (*R2)[k] = R2_hold_counter_[k] < 2
- ? std::max((*R2)[k], R2_old_[k])
- : std::min((*R2)[k] + R2_old_[k] * 0.1f, Y2[k]);
+ (*R2)[k] =
+ R2_hold_counter_[k] < config_.echo_model.nonlinear_hold
+ ? std::max((*R2)[k], R2_old_[k])
+ : std::min(
+ (*R2)[k] + R2_old_[k] * config_.echo_model.nonlinear_release,
+ Y2[k]);
}
}
void ResidualEchoEstimator::AddEchoReverb(
const std::array<float, kFftLengthBy2Plus1>& S2,
- bool saturated_echo,
size_t delay,
float reverb_decay_factor,
std::array<float, kFftLengthBy2Plus1>* R2) {
@@ -249,16 +176,63 @@
});
// Update the buffer of old echo powers.
- if (saturated_echo) {
- S2_old_[S2_old_index_].fill((*std::max_element(S2.begin(), S2.end())) *
- 100.f);
- } else {
- std::copy(S2.begin(), S2.end(), S2_old_[S2_old_index_].begin());
- }
+ std::copy(S2.begin(), S2.end(), S2_old_[S2_old_index_].begin());
// Add the power of the echo reverb to the residual echo power.
std::transform(R2->begin(), R2->end(), R2_reverb_.begin(), R2->begin(),
std::plus<float>());
}
+void ResidualEchoEstimator::EchoGeneratingPower(
+ const RenderBuffer& render_buffer,
+ size_t min_delay,
+ size_t max_delay,
+ std::array<float, kFftLengthBy2Plus1>* X2) const {
+ X2->fill(0.f);
+ for (size_t k = min_delay; k <= max_delay; ++k) {
+ std::transform(X2->begin(), X2->end(), render_buffer.Spectrum(k).begin(),
+ X2->begin(),
+ [](float a, float b) { return std::max(a, b); });
+ }
+
+ // Apply soft noise gate.
+ std::for_each(X2->begin(), X2->end(), [&](float& a) {
+ if (config_.echo_model.noise_gate_power > a) {
+ a = std::max(0.f, a - config_.echo_model.noise_gate_slope *
+ (config_.echo_model.noise_gate_power - a));
+ }
+ });
+}
+
+void ResidualEchoEstimator::RenderNoisePower(
+ const RenderBuffer& render_buffer,
+ std::array<float, kFftLengthBy2Plus1>* X2_noise_floor,
+ std::array<int, kFftLengthBy2Plus1>* X2_noise_floor_counter) const {
+ RTC_DCHECK(X2_noise_floor);
+ RTC_DCHECK(X2_noise_floor_counter);
+
+ const auto render_power = render_buffer.Spectrum(0);
+ RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size());
+ RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size());
+
+ // Estimate the stationary noise power in a minimum statistics manner.
+ for (size_t k = 0; k < render_power.size(); ++k) {
+ // Decrease rapidly.
+ if (render_power[k] < (*X2_noise_floor)[k]) {
+ (*X2_noise_floor)[k] = render_power[k];
+ (*X2_noise_floor_counter)[k] = 0;
+ } else {
+ // Increase in a delayed, leaky manner.
+ if ((*X2_noise_floor_counter)[k] >=
+ static_cast<int>(config_.echo_model.noise_floor_hold)) {
+ (*X2_noise_floor)[k] =
+ std::max((*X2_noise_floor)[k] * 1.1f,
+ config_.echo_model.min_noise_floor_power);
+ } else {
+ ++(*X2_noise_floor_counter)[k];
+ }
+ }
+ }
+}
+
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/residual_echo_estimator.h b/modules/audio_processing/aec3/residual_echo_estimator.h
index f7e2d1d..7b8a9b1 100644
--- a/modules/audio_processing/aec3/residual_echo_estimator.h
+++ b/modules/audio_processing/aec3/residual_echo_estimator.h
@@ -43,15 +43,11 @@
// (ERLE) and the linear power estimate.
void LinearEstimate(const std::array<float, kFftLengthBy2Plus1>& S2_linear,
const std::array<float, kFftLengthBy2Plus1>& erle,
- size_t delay,
std::array<float, kFftLengthBy2Plus1>* R2);
// Estimates the residual echo power based on the estimate of the echo path
// gain.
- void NonLinearEstimate(bool sufficient_filter_updates,
- bool saturated_echo,
- bool bounded_erl,
- bool transparent_mode,
+ void NonLinearEstimate(float echo_path_gain,
const std::array<float, kFftLengthBy2Plus1>& X2,
const std::array<float, kFftLengthBy2Plus1>& Y2,
std::array<float, kFftLengthBy2Plus1>* R2);
@@ -59,10 +55,24 @@
// Adds the estimated unmodelled echo power to the residual echo power
// estimate.
void AddEchoReverb(const std::array<float, kFftLengthBy2Plus1>& S2,
- bool saturated_echo,
size_t delay,
float reverb_decay_factor,
std::array<float, kFftLengthBy2Plus1>* R2);
+
+ // Estimates the echo generating signal power as gated maximal power over a
+ // time window.
+ void EchoGeneratingPower(const RenderBuffer& render_buffer,
+ size_t min_delay,
+ size_t max_delay,
+ std::array<float, kFftLengthBy2Plus1>* X2) const;
+
+ // Updates estimate for the power of the stationary noise component in the
+ // render signal.
+ void RenderNoisePower(
+ const RenderBuffer& render_buffer,
+ std::array<float, kFftLengthBy2Plus1>* X2_noise_floor,
+ std::array<int, kFftLengthBy2Plus1>* X2_noise_floor_counter) const;
+
const EchoCanceller3Config config_;
std::array<float, kFftLengthBy2Plus1> R2_old_;
std::array<int, kFftLengthBy2Plus1> R2_hold_counter_;
diff --git a/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc b/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc
index d46d518..7f9ad8d 100644
--- a/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc
+++ b/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc
@@ -93,9 +93,8 @@
render_delay_buffer->PrepareCaptureProcessing();
aec_state.HandleEchoPathChange(echo_path_variability);
- aec_state.Update(delay_estimate, H2, h, true,
- *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
- false);
+ aec_state.Update(delay_estimate, H2, h, true, false,
+ *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s);
estimator.Estimate(aec_state, *render_delay_buffer->GetRenderBuffer(),
S2_linear, Y2, &R2);
diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc
new file mode 100644
index 0000000..8e065d2
--- /dev/null
+++ b/modules/audio_processing/aec3/stationarity_estimator.cc
@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/aec3/stationarity_estimator.h"
+
+#include <algorithm>
+#include <array>
+#include <vector>
+
+#include "modules/audio_processing/aec3/aec3_common.h"
+#include "modules/audio_processing/aec3/vector_buffer.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/atomicops.h"
+
+namespace webrtc {
+
+namespace {
+constexpr float kMinNoisePower = 10.f;
+constexpr int kHangoverBlocks = kNumBlocksPerSecond / 20;
+constexpr int kNBlocksAverageInitPhase = 20;
+constexpr int kNBlocksInitialPhase = kNumBlocksPerSecond * 2.;
+} // namespace
+
+StationarityEstimator::StationarityEstimator()
+ : data_dumper_(
+ new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))) {
+ Reset();
+}
+
+StationarityEstimator::~StationarityEstimator() = default;
+
+void StationarityEstimator::Reset() {
+ noise_.Reset();
+ hangovers_.fill(0);
+ stationarity_flags_.fill(false);
+}
+
+// Update just the noise estimator. Usefull until the delay is known
+void StationarityEstimator::UpdateNoiseEstimator(
+ rtc::ArrayView<const float> spectrum) {
+ noise_.Update(spectrum);
+ data_dumper_->DumpRaw("aec3_stationarity_noise_spectrum", noise_.Spectrum());
+}
+
+void StationarityEstimator::UpdateStationarityFlags(
+ const VectorBuffer& spectrum_buffer,
+ int idx_current,
+ int num_lookahead) {
+ std::array<int, kWindowLength> indexes;
+ int num_lookahead_bounded = std::min(num_lookahead, kWindowLength - 1);
+ int idx = idx_current;
+
+ if (num_lookahead_bounded < kWindowLength - 1) {
+ int num_lookback = (kWindowLength - 1) - num_lookahead_bounded;
+ idx = spectrum_buffer.OffsetIndex(idx_current, num_lookback);
+ }
+ // For estimating the stationarity properties of the current frame, the
+ // power for each band is accumulated for several consecutive spectra in the
+ // method EstimateBandStationarity.
+ // In order to avoid getting the indexes of the spectra for every band with
+ // its associated overhead, those indexes are stored in an array and then use
+ // when the estimation is done.
+ indexes[0] = idx;
+ for (size_t k = 1; k < indexes.size(); ++k) {
+ indexes[k] = spectrum_buffer.DecIndex(indexes[k - 1]);
+ }
+ RTC_DCHECK_EQ(
+ spectrum_buffer.DecIndex(indexes[kWindowLength - 1]),
+ spectrum_buffer.OffsetIndex(idx_current, -(num_lookahead_bounded + 1)));
+
+ for (size_t k = 0; k < stationarity_flags_.size(); ++k) {
+ stationarity_flags_[k] =
+ EstimateBandStationarity(spectrum_buffer, indexes, k);
+ }
+ UpdateHangover();
+ SmoothStationaryPerFreq();
+
+}
+
+bool StationarityEstimator::EstimateBandStationarity(
+ const VectorBuffer& spectrum_buffer,
+ const std::array<int, kWindowLength>& indexes,
+ size_t band) const {
+ constexpr float kThrStationarity = 10.f;
+ float acum_power = 0.f;
+ for (auto idx : indexes) {
+ acum_power += spectrum_buffer.buffer[idx][band];
+ }
+ float noise = kWindowLength * GetStationarityPowerBand(band);
+ RTC_CHECK_LT(0.f, noise);
+ bool stationary = acum_power < kThrStationarity * noise;
+ data_dumper_->DumpRaw("aec3_stationarity_long_ratio", acum_power / noise);
+ return stationary;
+}
+
+bool StationarityEstimator::AreAllBandsStationary() {
+ for (auto b : stationarity_flags_) {
+ if (!b)
+ return false;
+ }
+ return true;
+}
+
+void StationarityEstimator::UpdateHangover() {
+ bool reduce_hangover = AreAllBandsStationary();
+ for (size_t k = 0; k < stationarity_flags_.size(); ++k) {
+ if (!stationarity_flags_[k]) {
+ hangovers_[k] = kHangoverBlocks;
+ } else if (reduce_hangover) {
+ hangovers_[k] = std::max(hangovers_[k] - 1, 0);
+ }
+ }
+}
+
+void StationarityEstimator::SmoothStationaryPerFreq() {
+ std::array<bool, kFftLengthBy2Plus1> all_ahead_stationary_smooth;
+ for (size_t k = 1; k < kFftLengthBy2Plus1 - 1; ++k) {
+ all_ahead_stationary_smooth[k] = stationarity_flags_[k - 1] &&
+ stationarity_flags_[k] &&
+ stationarity_flags_[k + 1];
+ }
+
+ all_ahead_stationary_smooth[0] = all_ahead_stationary_smooth[1];
+ all_ahead_stationary_smooth[kFftLengthBy2Plus1 - 1] =
+ all_ahead_stationary_smooth[kFftLengthBy2Plus1 - 2];
+
+ stationarity_flags_ = all_ahead_stationary_smooth;
+}
+
+int StationarityEstimator::instance_count_ = 0;
+
+StationarityEstimator::NoiseSpectrum::NoiseSpectrum() {
+ Reset();
+}
+
+StationarityEstimator::NoiseSpectrum::~NoiseSpectrum() = default;
+
+void StationarityEstimator::NoiseSpectrum::Reset() {
+ block_counter_ = 0;
+ noise_spectrum_.fill(kMinNoisePower);
+}
+
+void StationarityEstimator::NoiseSpectrum::Update(
+ rtc::ArrayView<const float> spectrum) {
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, spectrum.size());
+ ++block_counter_;
+ float alpha = GetAlpha();
+ for (size_t k = 0; k < spectrum.size(); ++k) {
+ if (block_counter_ <= kNBlocksAverageInitPhase) {
+ noise_spectrum_[k] += (1.f / kNBlocksAverageInitPhase) * spectrum[k];
+ } else {
+ noise_spectrum_[k] =
+ UpdateBandBySmoothing(spectrum[k], noise_spectrum_[k], alpha);
+ }
+ }
+}
+
+float StationarityEstimator::NoiseSpectrum::GetAlpha() const {
+ constexpr float kAlpha = 0.004f;
+ constexpr float kAlphaInit = 0.04f;
+ constexpr float kTiltAlpha = (kAlphaInit - kAlpha) / kNBlocksInitialPhase;
+
+ if (block_counter_ > (kNBlocksInitialPhase + kNBlocksAverageInitPhase)) {
+ return kAlpha;
+ } else {
+ return kAlphaInit -
+ kTiltAlpha * (block_counter_ - kNBlocksAverageInitPhase);
+ }
+}
+
+float StationarityEstimator::NoiseSpectrum::UpdateBandBySmoothing(
+ float power_band,
+ float power_band_noise,
+ float alpha) const {
+ float power_band_noise_updated = power_band_noise;
+ if (power_band_noise < power_band) {
+ RTC_DCHECK_GT(power_band, 0.f);
+ float alpha_inc = alpha * (power_band_noise / power_band);
+ if (block_counter_ > kNBlocksInitialPhase) {
+ if (10.f * power_band_noise < power_band) {
+ alpha_inc *= 0.1f;
+ }
+ }
+ power_band_noise_updated += alpha_inc * (power_band - power_band_noise);
+ } else {
+ power_band_noise_updated += alpha * (power_band - power_band_noise);
+ power_band_noise_updated =
+ std::max(power_band_noise_updated, kMinNoisePower);
+ }
+ return power_band_noise_updated;
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/aec3/stationarity_estimator.h b/modules/audio_processing/aec3/stationarity_estimator.h
new file mode 100644
index 0000000..8f01790
--- /dev/null
+++ b/modules/audio_processing/aec3/stationarity_estimator.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AEC3_STATIONARITY_ESTIMATOR_H_
+#define MODULES_AUDIO_PROCESSING_AEC3_STATIONARITY_ESTIMATOR_H_
+
+#include <array>
+#include <memory>
+#include <vector>
+
+#include "api/array_view.h"
+#include "modules/audio_processing/aec3/aec3_common.h"
+#include "modules/audio_processing/aec3/vector_buffer.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+
+class StationarityEstimator {
+ public:
+ StationarityEstimator();
+ ~StationarityEstimator();
+
+ // Reset the stationarity estimator.
+ void Reset();
+
+ // Update just the noise estimator. Usefull until the delay is known
+ void UpdateNoiseEstimator(rtc::ArrayView<const float> spectrum);
+
+ // Update the flag indicating whether this current frame is stationary. For
+ // getting a more robust estimation, it looks at future and/or past frames.
+ void UpdateStationarityFlags(const VectorBuffer& spectrum_buffer,
+ int idx_current,
+ int num_lookahead);
+
+ // Returns true if the current band is stationary.
+ bool IsBandStationary(size_t band) const {
+ return stationarity_flags_[band] && (hangovers_[band] == 0);
+ }
+
+ private:
+ static constexpr int kWindowLength = 13;
+ // Returns the power of the stationary noise spectrum at a band.
+ float GetStationarityPowerBand(size_t k) const { return noise_.Power(k); }
+
+ // Get an estimation of the stationarity for the current band by looking
+ // at the past/present/future available data.
+ bool EstimateBandStationarity(const VectorBuffer& spectrum_buffer,
+ const std::array<int, kWindowLength>& indexes,
+ size_t band) const;
+
+ // True if all bands at the current point are stationary.
+ bool AreAllBandsStationary();
+
+ // Update the hangover depending on the stationary status of the current
+ // frame.
+ void UpdateHangover();
+
+ // Smooth the stationarity detection by looking at neighbouring frequency
+ // bands.
+ void SmoothStationaryPerFreq();
+
+ class NoiseSpectrum {
+ public:
+ NoiseSpectrum();
+ ~NoiseSpectrum();
+
+ // Reset the noise power spectrum estimate state.
+ void Reset();
+
+ // Update the noise power spectrum with a new frame.
+ void Update(rtc::ArrayView<const float> spectrum);
+
+ // Get the noise estimation power spectrum.
+ rtc::ArrayView<const float> Spectrum() const { return noise_spectrum_; }
+
+ // Get the noise power spectrum at a certain band.
+ float Power(size_t band) const {
+ RTC_DCHECK_LT(band, noise_spectrum_.size());
+ return noise_spectrum_[band];
+ }
+
+ private:
+ // Get the update coefficient to be used for the current frame.
+ float GetAlpha() const;
+
+ // Update the noise power spectrum at a certain band with a new frame.
+ float UpdateBandBySmoothing(float power_band,
+ float power_band_noise,
+ float alpha) const;
+ std::array<float, kFftLengthBy2Plus1> noise_spectrum_;
+ size_t block_counter_;
+ };
+
+ static int instance_count_;
+ std::unique_ptr<ApmDataDumper> data_dumper_;
+ NoiseSpectrum noise_;
+ std::array<int, kFftLengthBy2Plus1> hangovers_;
+ std::array<bool, kFftLengthBy2Plus1> stationarity_flags_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AEC3_STATIONARITY_ESTIMATOR_H_
diff --git a/modules/audio_processing/aec3/subtractor.cc b/modules/audio_processing/aec3/subtractor.cc
index b6a68af..a72a667 100644
--- a/modules/audio_processing/aec3/subtractor.cc
+++ b/modules/audio_processing/aec3/subtractor.cc
@@ -99,6 +99,7 @@
shadow_filter_converged_ = false;
main_filter_.SetSizePartitions(config_.filter.main_initial.length_blocks,
true);
+ main_filter_once_converged_ = false;
shadow_filter_.SetSizePartitions(
config_.filter.shadow_initial.length_blocks, true);
};
@@ -133,7 +134,6 @@
RTC_DCHECK_EQ(kBlockSize, capture.size());
rtc::ArrayView<const float> y = capture;
FftData& E_main = output->E_main;
- FftData& E_main_nonwindowed = output->E_main_nonwindowed;
FftData E_shadow;
std::array<float, kBlockSize>& e_main = output->e_main;
std::array<float, kBlockSize>& e_shadow = output->e_shadow;
@@ -153,37 +153,26 @@
PredictionError(fft_, S, y, &e_shadow, nullptr, &shadow_saturation);
fft_.ZeroPaddedFft(e_shadow, Aec3Fft::Window::kHanning, &E_shadow);
- if (!(main_filter_converged_ || shadow_filter_converged_)) {
- const auto sum_of_squares = [](float a, float b) { return a + b * b; };
- const float y2 = std::accumulate(y.begin(), y.end(), 0.f, sum_of_squares);
+ // Check for filter convergence.
+ const auto sum_of_squares = [](float a, float b) { return a + b * b; };
+ const float y2 = std::accumulate(y.begin(), y.end(), 0.f, sum_of_squares);
+ const float e2_main =
+ std::accumulate(e_main.begin(), e_main.end(), 0.f, sum_of_squares);
+ const float e2_shadow =
+ std::accumulate(e_shadow.begin(), e_shadow.end(), 0.f, sum_of_squares);
- if (!main_filter_converged_) {
- const float e2_main =
- std::accumulate(e_main.begin(), e_main.end(), 0.f, sum_of_squares);
- main_filter_converged_ = e2_main > 0.1 * y2;
- }
-
- if (!shadow_filter_converged_) {
- const float e2_shadow = std::accumulate(e_shadow.begin(), e_shadow.end(),
- 0.f, sum_of_squares);
- shadow_filter_converged_ = e2_shadow > 0.1 * y2;
- }
- }
+ constexpr float kConvergenceThreshold = 50 * 50 * kBlockSize;
+ main_filter_converged_ = e2_main < 0.2 * y2 && y2 > kConvergenceThreshold;
+ shadow_filter_converged_ =
+ e2_shadow < 0.05 * y2 && y2 > kConvergenceThreshold;
+ main_filter_once_converged_ =
+ main_filter_once_converged_ || main_filter_converged_;
+ main_filter_diverged_ = e2_main > 1.5f * y2 && y2 > 30.f * 30.f * kBlockSize;
// Compute spectra for future use.
E_shadow.Spectrum(optimization_, output->E2_shadow);
E_main.Spectrum(optimization_, output->E2_main);
- if (main_filter_converged_ || !shadow_filter_converged_) {
- fft_.ZeroPaddedFft(e_main, Aec3Fft::Window::kRectangular,
- &E_main_nonwindowed);
- E_main_nonwindowed.Spectrum(optimization_, output->E2_main_nonwindowed);
- } else {
- fft_.ZeroPaddedFft(e_shadow, Aec3Fft::Window::kRectangular,
- &E_main_nonwindowed);
- E_main_nonwindowed.Spectrum(optimization_, output->E2_main_nonwindowed);
- }
-
// Update the main filter.
std::array<float, kFftLengthBy2Plus1> X2;
render_buffer.SpectralSum(main_filter_.SizePartitions(), &X2);
@@ -205,9 +194,7 @@
data_dumper_->DumpRaw("aec3_subtractor_G_shadow", G.re);
data_dumper_->DumpRaw("aec3_subtractor_G_shadow", G.im);
- main_filter_.DumpFilter("aec3_subtractor_H_main", "aec3_subtractor_h_main");
- shadow_filter_.DumpFilter("aec3_subtractor_H_shadow",
- "aec3_subtractor_h_shadow");
+ DumpFilters();
}
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/subtractor.h b/modules/audio_processing/aec3/subtractor.h
index b3c8506..38fc3c6 100644
--- a/modules/audio_processing/aec3/subtractor.h
+++ b/modules/audio_processing/aec3/subtractor.h
@@ -53,14 +53,14 @@
// Returns the block-wise frequency response for the main adaptive filter.
const std::vector<std::array<float, kFftLengthBy2Plus1>>&
FilterFrequencyResponse() const {
- return main_filter_converged_ || (!shadow_filter_converged_)
+ return main_filter_once_converged_ || (!shadow_filter_converged_)
? main_filter_.FilterFrequencyResponse()
: shadow_filter_.FilterFrequencyResponse();
}
// Returns the estimate of the impulse response for the main adaptive filter.
const std::vector<float>& FilterImpulseResponse() const {
- return main_filter_converged_ || (!shadow_filter_converged_)
+ return main_filter_once_converged_ || (!shadow_filter_converged_)
? main_filter_.FilterImpulseResponse()
: shadow_filter_.FilterImpulseResponse();
}
@@ -69,6 +69,14 @@
return main_filter_converged_ || shadow_filter_converged_;
}
+ bool DivergedFilter() const { return main_filter_diverged_; }
+
+ void DumpFilters() {
+ main_filter_.DumpFilter("aec3_subtractor_H_main", "aec3_subtractor_h_main");
+ shadow_filter_.DumpFilter("aec3_subtractor_H_shadow",
+ "aec3_subtractor_h_shadow");
+ }
+
private:
const Aec3Fft fft_;
ApmDataDumper* data_dumper_;
@@ -79,7 +87,9 @@
MainFilterUpdateGain G_main_;
ShadowFilterUpdateGain G_shadow_;
bool main_filter_converged_ = false;
+ bool main_filter_once_converged_ = false;
bool shadow_filter_converged_ = false;
+ bool main_filter_diverged_ = false;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(Subtractor);
};
diff --git a/modules/audio_processing/aec3/subtractor_output.h b/modules/audio_processing/aec3/subtractor_output.h
index 83f6cf5..8655665 100644
--- a/modules/audio_processing/aec3/subtractor_output.h
+++ b/modules/audio_processing/aec3/subtractor_output.h
@@ -24,9 +24,7 @@
std::array<float, kBlockSize> e_main;
std::array<float, kBlockSize> e_shadow;
FftData E_main;
- FftData E_main_nonwindowed;
std::array<float, kFftLengthBy2Plus1> E2_main;
- std::array<float, kFftLengthBy2Plus1> E2_main_nonwindowed;
std::array<float, kFftLengthBy2Plus1> E2_shadow;
void Reset() {
diff --git a/modules/audio_processing/aec3/subtractor_unittest.cc b/modules/audio_processing/aec3/subtractor_unittest.cc
index 5a8e070..097d7e8 100644
--- a/modules/audio_processing/aec3/subtractor_unittest.cc
+++ b/modules/audio_processing/aec3/subtractor_unittest.cc
@@ -68,7 +68,7 @@
}
render_delay_buffer->PrepareCaptureProcessing();
render_signal_analyzer.Update(*render_delay_buffer->GetRenderBuffer(),
- aec_state.FilterDelay());
+ aec_state.FilterDelayBlocks());
// Handle echo path changes.
if (std::find(blocks_with_echo_path_changes.begin(),
@@ -85,9 +85,9 @@
false, EchoPathVariability::DelayAdjustment::kNone, false));
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
- subtractor.ConvergedFilter(),
+ subtractor.ConvergedFilter(), subtractor.DivergedFilter(),
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2,
- output.s_main, false);
+ output.s_main);
}
const float output_power = std::inner_product(
diff --git a/modules/audio_processing/aec3/suppression_filter.cc b/modules/audio_processing/aec3/suppression_filter.cc
index 8c92bf5..87e3008 100644
--- a/modules/audio_processing/aec3/suppression_filter.cc
+++ b/modules/audio_processing/aec3/suppression_filter.cc
@@ -64,7 +64,6 @@
fft_(),
e_output_old_(NumBandsForRate(sample_rate_hz_)) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
- e_input_old_.fill(0.f);
std::for_each(e_output_old_.begin(), e_output_old_.end(),
[](std::array<float, kFftLengthBy2>& a) { a.fill(0.f); });
}
@@ -76,22 +75,14 @@
const FftData& comfort_noise_high_band,
const std::array<float, kFftLengthBy2Plus1>& suppression_gain,
float high_bands_gain,
+ const FftData& E_lowest_band,
std::vector<std::vector<float>>* e) {
RTC_DCHECK(e);
RTC_DCHECK_EQ(e->size(), NumBandsForRate(sample_rate_hz_));
FftData E;
- std::array<float, kFftLength> e_extended;
- constexpr float kIfftNormalization = 2.f / kFftLength;
// Analysis filterbank.
- std::transform(e_input_old_.begin(), e_input_old_.end(),
- std::begin(kSqrtHanning), e_extended.begin(),
- std::multiplies<float>());
- std::transform((*e)[0].begin(), (*e)[0].end(),
- std::begin(kSqrtHanning) + kFftLengthBy2,
- e_extended.begin() + kFftLengthBy2, std::multiplies<float>());
- std::copy((*e)[0].begin(), (*e)[0].end(), e_input_old_.begin());
- fft_.Fft(&e_extended, &E);
+ E.Assign(E_lowest_band);
// Apply gain.
std::transform(suppression_gain.begin(), suppression_gain.end(), E.re.begin(),
@@ -113,6 +104,9 @@
E.im.begin(), E.im.begin(), std::plus<float>());
// Synthesis filterbank.
+ std::array<float, kFftLength> e_extended;
+ constexpr float kIfftNormalization = 2.f / kFftLength;
+
fft_.Ifft(E, &e_extended);
std::transform(e_output_old_[0].begin(), e_output_old_[0].end(),
std::begin(kSqrtHanning) + kFftLengthBy2, (*e)[0].begin(),
diff --git a/modules/audio_processing/aec3/suppression_filter.h b/modules/audio_processing/aec3/suppression_filter.h
index 5f91dea..237408d 100644
--- a/modules/audio_processing/aec3/suppression_filter.h
+++ b/modules/audio_processing/aec3/suppression_filter.h
@@ -28,13 +28,13 @@
const FftData& comfort_noise_high_bands,
const std::array<float, kFftLengthBy2Plus1>& suppression_gain,
float high_bands_gain,
+ const FftData& E_lowest_band,
std::vector<std::vector<float>>* e);
private:
const int sample_rate_hz_;
const OouraFft ooura_fft_;
const Aec3Fft fft_;
- std::array<float, kFftLengthBy2> e_input_old_;
std::vector<std::array<float, kFftLengthBy2>> e_output_old_;
RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionFilter);
};
diff --git a/modules/audio_processing/aec3/suppression_filter_unittest.cc b/modules/audio_processing/aec3/suppression_filter_unittest.cc
index 51b3f91..eaa608e 100644
--- a/modules/audio_processing/aec3/suppression_filter_unittest.cc
+++ b/modules/audio_processing/aec3/suppression_filter_unittest.cc
@@ -42,10 +42,11 @@
TEST(SuppressionFilter, NullOutput) {
FftData cn;
FftData cn_high_bands;
+ FftData E;
std::array<float, kFftLengthBy2Plus1> gain;
EXPECT_DEATH(SuppressionFilter(16000).ApplyGain(cn, cn_high_bands, gain, 1.0f,
- nullptr),
+ E, nullptr),
"");
}
@@ -62,7 +63,10 @@
FftData cn;
FftData cn_high_bands;
std::array<float, kFftLengthBy2Plus1> gain;
+ std::array<float, kFftLengthBy2> e_old_;
+ Aec3Fft fft;
+ e_old_.fill(0.f);
gain.fill(1.f);
cn.re.fill(1.f);
cn.im.fill(1.f);
@@ -71,7 +75,12 @@
std::vector<std::vector<float>> e(3, std::vector<float>(kBlockSize, 0.f));
std::vector<std::vector<float>> e_ref = e;
- filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e);
+
+ FftData E;
+ fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E);
+ std::copy(e[0].begin(), e[0].end(), e_old_.begin());
+
+ filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
for (size_t k = 0; k < e.size(); ++k) {
EXPECT_EQ(e_ref[k], e[k]);
@@ -83,8 +92,11 @@
SuppressionFilter filter(48000);
FftData cn;
FftData cn_high_bands;
+ std::array<float, kFftLengthBy2> e_old_;
+ Aec3Fft fft;
std::array<float, kFftLengthBy2Plus1> gain;
std::vector<std::vector<float>> e(3, std::vector<float>(kBlockSize, 0.f));
+ e_old_.fill(0.f);
gain.fill(1.f);
std::for_each(gain.begin() + 10, gain.end(), [](float& a) { a = 0.f; });
@@ -103,7 +115,12 @@
e[0]);
e0_input =
std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_input);
- filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e);
+
+ FftData E;
+ fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E);
+ std::copy(e[0].begin(), e[0].end(), e_old_.begin());
+
+ filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
e0_output =
std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_output);
}
@@ -116,10 +133,12 @@
TEST(SuppressionFilter, SignalTransparency) {
SuppressionFilter filter(48000);
FftData cn;
+ std::array<float, kFftLengthBy2> e_old_;
+ Aec3Fft fft;
FftData cn_high_bands;
std::array<float, kFftLengthBy2Plus1> gain;
std::vector<std::vector<float>> e(3, std::vector<float>(kBlockSize, 0.f));
-
+ e_old_.fill(0.f);
gain.fill(1.f);
std::for_each(gain.begin() + 30, gain.end(), [](float& a) { a = 0.f; });
@@ -137,7 +156,12 @@
e[0]);
e0_input =
std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_input);
- filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e);
+
+ FftData E;
+ fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E);
+ std::copy(e[0].begin(), e[0].end(), e_old_.begin());
+
+ filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
e0_output =
std::inner_product(e[0].begin(), e[0].end(), e[0].begin(), e0_output);
}
@@ -150,6 +174,8 @@
SuppressionFilter filter(48000);
FftData cn;
FftData cn_high_bands;
+ std::array<float, kFftLengthBy2> e_old_;
+ Aec3Fft fft;
std::array<float, kFftLengthBy2Plus1> gain;
std::vector<std::vector<float>> e(3, std::vector<float>(kBlockSize, 0.f));
@@ -167,7 +193,11 @@
}
}
- filter.ApplyGain(cn, cn_high_bands, gain, 1.f, &e);
+ FftData E;
+ fft.PaddedFft(e[0], e_old_, Aec3Fft::Window::kSqrtHanning, &E);
+ std::copy(e[0].begin(), e[0].end(), e_old_.begin());
+
+ filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
if (k > 2) {
for (size_t j = 0; j < 2; ++j) {
for (size_t i = 0; i < kBlockSize; ++i) {
diff --git a/modules/audio_processing/aec3/suppression_gain.cc b/modules/audio_processing/aec3/suppression_gain.cc
index 53fd575..9591c67 100644
--- a/modules/audio_processing/aec3/suppression_gain.cc
+++ b/modules/audio_processing/aec3/suppression_gain.cc
@@ -1,3 +1,4 @@
+
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
@@ -20,21 +21,13 @@
#include <numeric>
#include "modules/audio_processing/aec3/vector_math.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
-// Reduce gain to avoid narrow band echo leakage.
-void NarrowBandAttenuation(int narrow_bin,
- std::array<float, kFftLengthBy2Plus1>* gain) {
- const int upper_bin =
- std::min(narrow_bin + 6, static_cast<int>(kFftLengthBy2Plus1 - 1));
- for (int k = std::max(0, narrow_bin - 6); k <= upper_bin; ++k) {
- (*gain)[k] = std::min((*gain)[k], 0.001f);
- }
-}
-
// Adjust the gains according to the presence of known external filters.
void AdjustForExternalFilters(std::array<float, kFftLengthBy2Plus1>* gain) {
// Limit the low frequency gains to avoid the impact of the high-pass filter
@@ -107,19 +100,60 @@
return std::min(gain_below_8_khz, anti_howling_gain);
}
+// Scales the echo according to assessed audibility at the other end.
+void WeightEchoForAudibility(const EchoCanceller3Config& config,
+ rtc::ArrayView<const float> echo,
+ rtc::ArrayView<float> weighted_echo,
+ rtc::ArrayView<float> one_by_weighted_echo) {
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, echo.size());
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, weighted_echo.size());
+ RTC_DCHECK_EQ(kFftLengthBy2Plus1, one_by_weighted_echo.size());
+
+ auto weigh = [](float threshold, float normalizer, size_t begin, size_t end,
+ rtc::ArrayView<const float> echo,
+ rtc::ArrayView<float> weighted_echo,
+ rtc::ArrayView<float> one_by_weighted_echo) {
+ for (size_t k = begin; k < end; ++k) {
+ if (echo[k] < threshold) {
+ float tmp = (threshold - echo[k]) * normalizer;
+ weighted_echo[k] = echo[k] * std::max(0.f, 1.f - tmp * tmp);
+ } else {
+ weighted_echo[k] = echo[k];
+ }
+ one_by_weighted_echo[k] =
+ weighted_echo[k] > 0.f ? 1.f / weighted_echo[k] : 1.f;
+ }
+ };
+
+ float threshold = config.echo_audibility.floor_power *
+ config.echo_audibility.audibility_threshold_lf;
+ float normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
+ weigh(threshold, normalizer, 0, 3, echo, weighted_echo, one_by_weighted_echo);
+
+ threshold = config.echo_audibility.floor_power *
+ config.echo_audibility.audibility_threshold_mf;
+ normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
+ weigh(threshold, normalizer, 3, 7, echo, weighted_echo, one_by_weighted_echo);
+
+ threshold = config.echo_audibility.floor_power *
+ config.echo_audibility.audibility_threshold_hf;
+ normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
+ weigh(threshold, normalizer, 7, kFftLengthBy2Plus1, echo, weighted_echo,
+ one_by_weighted_echo);
+}
+
// Computes the gain to reduce the echo to a non audible level.
void GainToNoAudibleEcho(
const EchoCanceller3Config& config,
bool low_noise_render,
bool saturated_echo,
- bool saturating_echo_path,
bool linear_echo_estimate,
const std::array<float, kFftLengthBy2Plus1>& nearend,
- const std::array<float, kFftLengthBy2Plus1>& echo,
+ const std::array<float, kFftLengthBy2Plus1>& weighted_echo,
const std::array<float, kFftLengthBy2Plus1>& masker,
const std::array<float, kFftLengthBy2Plus1>& min_gain,
const std::array<float, kFftLengthBy2Plus1>& max_gain,
- const std::array<float, kFftLengthBy2Plus1>& one_by_echo,
+ const std::array<float, kFftLengthBy2Plus1>& one_by_weighted_echo,
std::array<float, kFftLengthBy2Plus1>* gain) {
float nearend_masking_margin = 0.f;
if (linear_echo_estimate) {
@@ -133,8 +167,6 @@
RTC_DCHECK_LE(0.f, nearend_masking_margin);
RTC_DCHECK_GT(1.f, nearend_masking_margin);
- const float one_by_one_minus_nearend_masking_margin =
- 1.f / (1.0f - nearend_masking_margin);
const float masker_margin =
linear_echo_estimate ? config.gain_mask.m1 : config.gain_mask.m8;
@@ -142,15 +174,17 @@
for (size_t k = 0; k < gain->size(); ++k) {
const float unity_gain_masker = std::max(nearend[k], masker[k]);
RTC_DCHECK_LE(0.f, nearend_masking_margin * unity_gain_masker);
- if (echo[k] <= nearend_masking_margin * unity_gain_masker ||
+ if (weighted_echo[k] <= nearend_masking_margin * unity_gain_masker ||
unity_gain_masker <= 0.f) {
(*gain)[k] = 1.f;
} else {
RTC_DCHECK_LT(0.f, unity_gain_masker);
- (*gain)[k] = std::max(0.f, (1.f - 5.f * echo[k] / unity_gain_masker) *
- one_by_one_minus_nearend_masking_margin);
(*gain)[k] =
- std::max(masker_margin * masker[k] * one_by_echo[k], (*gain)[k]);
+ std::max(0.f, (1.f - config.gain_mask.gain_curve_slope *
+ weighted_echo[k] / unity_gain_masker) *
+ config.gain_mask.gain_curve_offset);
+ (*gain)[k] = std::max(masker_margin * masker[k] * one_by_weighted_echo[k],
+ (*gain)[k]);
}
(*gain)[k] = std::min(std::max((*gain)[k], min_gain[k]), max_gain[k]);
@@ -167,6 +201,20 @@
const std::array<float, kFftLengthBy2Plus1>& last_masker,
const std::array<float, kFftLengthBy2Plus1>& gain,
std::array<float, kFftLengthBy2Plus1>* masker) {
+ // Apply masking over time.
+ float masking_factor = config.gain_mask.temporal_masking_lf;
+ auto limit = config.gain_mask.temporal_masking_lf_bands;
+ std::transform(
+ comfort_noise.begin(), comfort_noise.begin() + limit, last_masker.begin(),
+ masker->begin(),
+ [masking_factor](float a, float b) { return a + masking_factor * b; });
+ masking_factor = config.gain_mask.temporal_masking_hf;
+ std::transform(
+ comfort_noise.begin() + limit, comfort_noise.end(),
+ last_masker.begin() + limit, masker->begin() + limit,
+ [masking_factor](float a, float b) { return a + masking_factor * b; });
+
+ // Apply masking only between lower frequency bands.
std::array<float, kFftLengthBy2Plus1> side_band_masker;
float max_nearend_after_gain = 0.f;
for (size_t k = 0; k < gain.size(); ++k) {
@@ -174,10 +222,8 @@
max_nearend_after_gain =
std::max(max_nearend_after_gain, nearend_after_gain);
side_band_masker[k] = nearend_after_gain + comfort_noise[k];
- (*masker)[k] = comfort_noise[k] + config.gain_mask.m4 * last_masker[k];
}
- // Apply masking only between lower frequency bands.
RTC_DCHECK_LT(kUpperAccurateBandPlus1, gain.size());
for (size_t k = 1; k < kUpperAccurateBandPlus1; ++k) {
(*masker)[k] += config.gain_mask.m5 *
@@ -209,27 +255,24 @@
} // namespace
+int SuppressionGain::instance_count_ = 0;
+
// TODO(peah): Add further optimizations, in particular for the divisions.
void SuppressionGain::LowerBandGain(
bool low_noise_render,
- const rtc::Optional<int>& narrow_peak_band,
const AecState& aec_state,
const std::array<float, kFftLengthBy2Plus1>& nearend,
const std::array<float, kFftLengthBy2Plus1>& echo,
const std::array<float, kFftLengthBy2Plus1>& comfort_noise,
std::array<float, kFftLengthBy2Plus1>* gain) {
const bool saturated_echo = aec_state.SaturatedEcho();
- const bool saturating_echo_path = aec_state.SaturatingEchoPath();
const bool linear_echo_estimate = aec_state.UsableLinearEstimate();
- // Count the number of blocks since saturation.
- no_saturation_counter_ = saturated_echo ? 0 : no_saturation_counter_ + 1;
-
- // Precompute 1/echo (note that when the echo is zero, the precomputed value
- // is never used).
- std::array<float, kFftLengthBy2Plus1> one_by_echo;
- std::transform(echo.begin(), echo.end(), one_by_echo.begin(),
- [](float a) { return a > 0.f ? 1.f / a : 1.f; });
+ // Weight echo power in terms of audibility. // Precompute 1/weighted echo
+ // (note that when the echo is zero, the precomputed value is never used).
+ std::array<float, kFftLengthBy2Plus1> weighted_echo;
+ std::array<float, kFftLengthBy2Plus1> one_by_weighted_echo;
+ WeightEchoForAudibility(config_, echo, weighted_echo, one_by_weighted_echo);
// Compute the minimum gain as the attenuating gain to put the signal just
// above the zero sample values.
@@ -237,9 +280,9 @@
const float min_echo_power =
low_noise_render ? config_.echo_audibility.low_render_limit
: config_.echo_audibility.normal_render_limit;
- if (no_saturation_counter_ > 10) {
+ if (!saturated_echo) {
for (size_t k = 0; k < nearend.size(); ++k) {
- const float denom = std::min(nearend[k], echo[k]);
+ const float denom = std::min(nearend[k], weighted_echo[k]);
min_gain[k] = denom > 0.f ? min_echo_power / denom : 1.f;
min_gain[k] = std::min(min_gain[k], 1.f);
}
@@ -259,44 +302,47 @@
// Iteratively compute the gain required to attenuate the echo to a non
// noticeable level.
gain->fill(0.f);
+ std::array<float, kFftLengthBy2Plus1> masker;
for (int k = 0; k < 2; ++k) {
- std::array<float, kFftLengthBy2Plus1> masker;
MaskingPower(config_, nearend, comfort_noise, last_masker_, *gain, &masker);
GainToNoAudibleEcho(config_, low_noise_render, saturated_echo,
- saturating_echo_path, linear_echo_estimate, nearend,
- echo, masker, min_gain, max_gain, one_by_echo, gain);
+ linear_echo_estimate, nearend, weighted_echo, masker,
+ min_gain, max_gain, one_by_weighted_echo, gain);
AdjustForExternalFilters(gain);
- if (narrow_peak_band) {
- NarrowBandAttenuation(*narrow_peak_band, gain);
- }
}
// Adjust the gain for frequencies which have not yet converged.
AdjustNonConvergedFrequencies(gain);
// Update the allowed maximum gain increase.
- UpdateGainIncrease(low_noise_render, linear_echo_estimate, echo, *gain);
-
- // Adjust gain dynamics.
- const float gain_bound =
- std::max(0.001f, *std::min_element(gain->begin(), gain->end()) * 10000.f);
- std::for_each(gain->begin(), gain->end(),
- [gain_bound](float& a) { a = std::min(a, gain_bound); });
+ UpdateGainIncrease(low_noise_render, linear_echo_estimate, saturated_echo,
+ weighted_echo, *gain);
// Store data required for the gain computation of the next block.
- std::copy(echo.begin(), echo.end(), last_echo_.begin());
+ std::copy(weighted_echo.begin(), weighted_echo.end(), last_echo_.begin());
std::copy(gain->begin(), gain->end(), last_gain_.begin());
MaskingPower(config_, nearend, comfort_noise, last_masker_, *gain,
&last_masker_);
aec3::VectorMath(optimization_).Sqrt(*gain);
+
+ // Debug outputs for the purpose of development and analysis.
+ data_dumper_->DumpRaw("aec3_suppressor_min_gain", min_gain);
+ data_dumper_->DumpRaw("aec3_suppressor_max_gain", max_gain);
+ data_dumper_->DumpRaw("aec3_suppressor_masker", masker);
+ data_dumper_->DumpRaw("aec3_suppressor_last_masker", last_masker_);
}
SuppressionGain::SuppressionGain(const EchoCanceller3Config& config,
- Aec3Optimization optimization)
- : optimization_(optimization),
+ Aec3Optimization optimization,
+ int sample_rate_hz)
+ : data_dumper_(
+ new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
+ optimization_(optimization),
config_(config),
state_change_duration_blocks_(
- static_cast<int>(config_.filter.config_change_duration_blocks)) {
+ static_cast<int>(config_.filter.config_change_duration_blocks)),
+ coherence_gain_(sample_rate_hz,
+ config_.suppressor.bands_with_reliable_coherence) {
RTC_DCHECK_LT(0, state_change_duration_blocks_);
one_by_state_change_duration_blocks_ = 1.f / state_change_duration_blocks_;
last_gain_.fill(1.f);
@@ -305,10 +351,15 @@
last_echo_.fill(0.f);
}
+SuppressionGain::~SuppressionGain() = default;
+
void SuppressionGain::GetGain(
- const std::array<float, kFftLengthBy2Plus1>& nearend,
- const std::array<float, kFftLengthBy2Plus1>& echo,
- const std::array<float, kFftLengthBy2Plus1>& comfort_noise,
+ const std::array<float, kFftLengthBy2Plus1>& nearend_spectrum,
+ const std::array<float, kFftLengthBy2Plus1>& echo_spectrum,
+ const std::array<float, kFftLengthBy2Plus1>& comfort_noise_spectrum,
+ const FftData& linear_aec_fft,
+ const FftData& render_fft,
+ const FftData& capture_fft,
const RenderSignalAnalyzer& render_signal_analyzer,
const AecState& aec_state,
const std::vector<std::vector<float>>& render,
@@ -321,9 +372,21 @@
bool low_noise_render = low_render_detector_.Detect(render);
const rtc::Optional<int> narrow_peak_band =
render_signal_analyzer.NarrowPeakBand();
- LowerBandGain(low_noise_render, narrow_peak_band, aec_state, nearend, echo,
- comfort_noise, low_band_gain);
+ LowerBandGain(low_noise_render, aec_state, nearend_spectrum, echo_spectrum,
+ comfort_noise_spectrum, low_band_gain);
+ // Adjust the gain for bands where the coherence indicates not echo.
+ if (config_.suppressor.bands_with_reliable_coherence > 0) {
+ std::array<float, kFftLengthBy2Plus1> G_coherence;
+ coherence_gain_.ComputeGain(linear_aec_fft, render_fft, capture_fft,
+ G_coherence);
+ for (size_t k = 0; k < config_.suppressor.bands_with_reliable_coherence;
+ ++k) {
+ (*low_band_gain)[k] = std::max((*low_band_gain)[k], G_coherence[k]);
+ }
+ }
+
+ // Limit the gain of the lower bands during start up and after resets.
const float gain_upper_bound = aec_state.SuppressionGainLimit();
if (gain_upper_bound < 1.f) {
for (size_t k = 0; k < low_band_gain->size(); ++k) {
@@ -348,6 +411,7 @@
void SuppressionGain::UpdateGainIncrease(
bool low_noise_render,
bool linear_echo_estimate,
+ bool saturated_echo,
const std::array<float, kFftLengthBy2Plus1>& echo,
const std::array<float, kFftLengthBy2Plus1>& new_gain) {
float max_inc;
@@ -374,7 +438,7 @@
rate_dec = p.nonlinear.rate_dec;
min_inc = p.nonlinear.min_inc;
min_dec = p.nonlinear.min_dec;
- } else if (initial_state_ && no_saturation_counter_ > 10) {
+ } else if (initial_state_ && !saturated_echo) {
if (initial_state_change_counter_ > 0) {
float change_factor =
initial_state_change_counter_ * one_by_state_change_duration_blocks_;
@@ -404,7 +468,7 @@
rate_dec = p.low_noise.rate_dec;
min_inc = p.low_noise.min_inc;
min_dec = p.low_noise.min_dec;
- } else if (no_saturation_counter_ > 10) {
+ } else if (!saturated_echo) {
max_inc = p.normal.max_inc;
max_dec = p.normal.max_dec;
rate_inc = p.normal.rate_inc;
diff --git a/modules/audio_processing/aec3/suppression_gain.h b/modules/audio_processing/aec3/suppression_gain.h
index 6624c1c..7b34d0a 100644
--- a/modules/audio_processing/aec3/suppression_gain.h
+++ b/modules/audio_processing/aec3/suppression_gain.h
@@ -17,6 +17,7 @@
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec_state.h"
+#include "modules/audio_processing/aec3/coherence_gain.h"
#include "modules/audio_processing/aec3/render_signal_analyzer.h"
#include "rtc_base/constructormagic.h"
@@ -25,22 +26,27 @@
class SuppressionGain {
public:
SuppressionGain(const EchoCanceller3Config& config,
- Aec3Optimization optimization);
- void GetGain(const std::array<float, kFftLengthBy2Plus1>& nearend,
- const std::array<float, kFftLengthBy2Plus1>& echo,
- const std::array<float, kFftLengthBy2Plus1>& comfort_noise,
- const RenderSignalAnalyzer& render_signal_analyzer,
- const AecState& aec_state,
- const std::vector<std::vector<float>>& render,
- float* high_bands_gain,
- std::array<float, kFftLengthBy2Plus1>* low_band_gain);
+ Aec3Optimization optimization,
+ int sample_rate_hz);
+ ~SuppressionGain();
+ void GetGain(
+ const std::array<float, kFftLengthBy2Plus1>& nearend_spectrum,
+ const std::array<float, kFftLengthBy2Plus1>& echo_spectrum,
+ const std::array<float, kFftLengthBy2Plus1>& comfort_noise_spectrum,
+ const FftData& linear_aec_fft,
+ const FftData& render_fft,
+ const FftData& capture_fft,
+ const RenderSignalAnalyzer& render_signal_analyzer,
+ const AecState& aec_state,
+ const std::vector<std::vector<float>>& render,
+ float* high_bands_gain,
+ std::array<float, kFftLengthBy2Plus1>* low_band_gain);
// Toggles the usage of the initial state.
void SetInitialState(bool state);
private:
void LowerBandGain(bool stationary_with_low_power,
- const rtc::Optional<int>& narrow_peak_band,
const AecState& aec_state,
const std::array<float, kFftLengthBy2Plus1>& nearend,
const std::array<float, kFftLengthBy2Plus1>& echo,
@@ -51,6 +57,7 @@
void UpdateGainIncrease(
bool low_noise_render,
bool linear_echo_estimate,
+ bool saturated_echo,
const std::array<float, kFftLengthBy2Plus1>& echo,
const std::array<float, kFftLengthBy2Plus1>& new_gain);
@@ -62,6 +69,8 @@
float average_power_ = 32768.f * 32768.f;
};
+ static int instance_count_;
+ std::unique_ptr<ApmDataDumper> data_dumper_;
const Aec3Optimization optimization_;
const EchoCanceller3Config config_;
const int state_change_duration_blocks_;
@@ -70,11 +79,11 @@
std::array<float, kFftLengthBy2Plus1> last_masker_;
std::array<float, kFftLengthBy2Plus1> gain_increase_;
std::array<float, kFftLengthBy2Plus1> last_echo_;
-
LowNoiseRenderDetector low_render_detector_;
- size_t no_saturation_counter_ = 0;
bool initial_state_ = true;
int initial_state_change_counter_ = 0;
+ CoherenceGain coherence_gain_;
+
RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionGain);
};
diff --git a/modules/audio_processing/aec3/suppression_gain_limiter.cc b/modules/audio_processing/aec3/suppression_gain_limiter.cc
index 643bb58..52218eb 100644
--- a/modules/audio_processing/aec3/suppression_gain_limiter.cc
+++ b/modules/audio_processing/aec3/suppression_gain_limiter.cc
@@ -38,7 +38,16 @@
recent_reset_ = true;
}
-void SuppressionGainUpperLimiter::Update(bool render_activity) {
+void SuppressionGainUpperLimiter::Update(bool render_activity,
+ bool transparent_mode) {
+ if (transparent_mode) {
+ active_render_seen_ = true;
+ call_startup_phase_ = false;
+ recent_reset_ = false;
+ suppressor_gain_limit_ = 1.f;
+ return;
+ }
+
if (recent_reset_ && !call_startup_phase_) {
// Only enforce 250 ms full suppression after in-call resets,
constexpr int kMuteFramesAfterReset = kNumBlocksPerSecond / 4;
diff --git a/modules/audio_processing/aec3/suppression_gain_limiter.h b/modules/audio_processing/aec3/suppression_gain_limiter.h
index 7a3f228..e02f491 100644
--- a/modules/audio_processing/aec3/suppression_gain_limiter.h
+++ b/modules/audio_processing/aec3/suppression_gain_limiter.h
@@ -27,7 +27,7 @@
void Reset();
// Updates the limiting behavior for the current capture bloc.
- void Update(bool render_activity);
+ void Update(bool render_activity, bool transparent_mode);
// Returns the current suppressor gain limit.
float Limit() const { return suppressor_gain_limit_; }
diff --git a/modules/audio_processing/aec3/suppression_gain_unittest.cc b/modules/audio_processing/aec3/suppression_gain_unittest.cc
index 0e48102..128c61e 100644
--- a/modules/audio_processing/aec3/suppression_gain_unittest.cc
+++ b/modules/audio_processing/aec3/suppression_gain_unittest.cc
@@ -29,15 +29,25 @@
std::array<float, kFftLengthBy2Plus1> E2;
std::array<float, kFftLengthBy2Plus1> R2;
std::array<float, kFftLengthBy2Plus1> N2;
+ FftData E;
+ FftData X;
+ FftData Y;
E2.fill(0.f);
R2.fill(0.f);
N2.fill(0.f);
+ E.re.fill(0.f);
+ E.im.fill(0.f);
+ X.re.fill(0.f);
+ X.im.fill(0.f);
+ Y.re.fill(0.f);
+ Y.im.fill(0.f);
+
float high_bands_gain;
AecState aec_state(EchoCanceller3Config{});
EXPECT_DEATH(
- SuppressionGain(EchoCanceller3Config{}, DetectOptimization())
- .GetGain(E2, R2, N2, RenderSignalAnalyzer((EchoCanceller3Config{})),
- aec_state,
+ SuppressionGain(EchoCanceller3Config{}, DetectOptimization(), 16000)
+ .GetGain(E2, R2, N2, E, X, Y,
+ RenderSignalAnalyzer((EchoCanceller3Config{})), aec_state,
std::vector<std::vector<float>>(
3, std::vector<float>(kBlockSize, 0.f)),
&high_bands_gain, nullptr),
@@ -48,8 +58,8 @@
// Does a sanity check that the gains are correctly computed.
TEST(SuppressionGain, BasicGainComputation) {
- SuppressionGain suppression_gain(EchoCanceller3Config(),
- DetectOptimization());
+ SuppressionGain suppression_gain(EchoCanceller3Config(), DetectOptimization(),
+ 16000);
RenderSignalAnalyzer analyzer(EchoCanceller3Config{});
float high_bands_gain;
std::array<float, kFftLengthBy2Plus1> E2;
@@ -58,6 +68,9 @@
std::array<float, kFftLengthBy2Plus1> N2;
std::array<float, kFftLengthBy2Plus1> g;
std::array<float, kBlockSize> s;
+ FftData E;
+ FftData X;
+ FftData Y;
std::vector<std::vector<float>> x(1, std::vector<float>(kBlockSize, 0.f));
EchoCanceller3Config config;
AecState aec_state(config);
@@ -73,21 +86,27 @@
R2.fill(0.1f);
N2.fill(100.f);
s.fill(10.f);
+ E.re.fill(sqrtf(E2[0]));
+ E.im.fill(0.f);
+ X.re.fill(sqrtf(R2[0]));
+ X.im.fill(0.f);
+ Y.re.fill(sqrtf(Y2[0]));
+ Y.im.fill(0.f);
// Ensure that the gain is no longer forced to zero.
for (int k = 0; k <= kNumBlocksPerSecond / 5 + 1; ++k) {
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
- subtractor.ConvergedFilter(),
- *render_delay_buffer->GetRenderBuffer(), E2, Y2, s, false);
+ subtractor.ConvergedFilter(), subtractor.DivergedFilter(),
+ *render_delay_buffer->GetRenderBuffer(), E2, Y2, s);
}
for (int k = 0; k < 100; ++k) {
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
- subtractor.ConvergedFilter(),
- *render_delay_buffer->GetRenderBuffer(), E2, Y2, s, false);
- suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x,
+ subtractor.ConvergedFilter(), subtractor.DivergedFilter(),
+ *render_delay_buffer->GetRenderBuffer(), E2, Y2, s);
+ suppression_gain.GetGain(E2, R2, N2, E, X, Y, analyzer, aec_state, x,
&high_bands_gain, &g);
}
std::for_each(g.begin(), g.end(),
@@ -98,12 +117,16 @@
Y2.fill(100.f);
R2.fill(0.1f);
N2.fill(0.f);
+ E.re.fill(sqrtf(E2[0]));
+ X.re.fill(sqrtf(R2[0]));
+ Y.re.fill(sqrtf(Y2[0]));
+
for (int k = 0; k < 100; ++k) {
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
- subtractor.ConvergedFilter(),
- *render_delay_buffer->GetRenderBuffer(), E2, Y2, s, false);
- suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x,
+ subtractor.ConvergedFilter(), subtractor.DivergedFilter(),
+ *render_delay_buffer->GetRenderBuffer(), E2, Y2, s);
+ suppression_gain.GetGain(E2, R2, N2, E, X, Y, analyzer, aec_state, x,
&high_bands_gain, &g);
}
std::for_each(g.begin(), g.end(),
@@ -112,9 +135,11 @@
// Ensure that a strong echo is suppressed.
E2.fill(1000000000.f);
R2.fill(10000000000000.f);
- N2.fill(0.f);
+ E.re.fill(sqrtf(E2[0]));
+ X.re.fill(sqrtf(R2[0]));
+
for (int k = 0; k < 10; ++k) {
- suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x,
+ suppression_gain.GetGain(E2, R2, N2, E, X, Y, analyzer, aec_state, x,
&high_bands_gain, &g);
}
std::for_each(g.begin(), g.end(),
diff --git a/modules/audio_processing/aec_dump/BUILD.gn b/modules/audio_processing/aec_dump/BUILD.gn
index 7afaaf4..152c290 100644
--- a/modules/audio_processing/aec_dump/BUILD.gn
+++ b/modules/audio_processing/aec_dump/BUILD.gn
@@ -29,7 +29,6 @@
deps = [
"..:aec_dump_interface",
- "../..:module_api",
"../../../test:test_support",
]
}
@@ -63,11 +62,12 @@
deps = [
":aec_dump",
"..:aec_dump_interface",
- "../../../modules:module_api",
+ "../../../api/audio:audio_frame_api",
"../../../rtc_base:checks",
"../../../rtc_base:protobuf_utils",
"../../../rtc_base:rtc_base_approved",
"../../../rtc_base:rtc_task_queue",
+ "../../../rtc_base/system:file_wrapper",
"../../../system_wrappers",
]
@@ -82,7 +82,6 @@
":aec_dump_impl",
"..:aec_dump_interface",
"..:audioproc_debug_proto",
- "../../../modules:module_api",
"../../../rtc_base:rtc_task_queue",
"../../../test:fileutils",
"../../../test:test_support",
diff --git a/modules/audio_processing/aec_dump/aec_dump_impl.cc b/modules/audio_processing/aec_dump/aec_dump_impl.cc
index 4deb192..ec35f0a 100644
--- a/modules/audio_processing/aec_dump/aec_dump_impl.cc
+++ b/modules/audio_processing/aec_dump/aec_dump_impl.cc
@@ -48,6 +48,10 @@
pb_cfg->set_intelligibility_enhancer_enabled(
config.intelligibility_enhancer_enabled);
+ pb_cfg->set_pre_amplifier_enabled(config.pre_amplifier_enabled);
+ pb_cfg->set_pre_amplifier_fixed_gain_factor(
+ config.pre_amplifier_fixed_gain_factor);
+
pb_cfg->set_experiments_description(config.experiments_description);
}
diff --git a/modules/audio_processing/aec_dump/aec_dump_impl.h b/modules/audio_processing/aec_dump/aec_dump_impl.h
index 36d72e9..0d88a7e 100644
--- a/modules/audio_processing/aec_dump/aec_dump_impl.h
+++ b/modules/audio_processing/aec_dump/aec_dump_impl.h
@@ -15,16 +15,16 @@
#include <string>
#include <vector>
+#include "api/audio/audio_frame.h"
#include "modules/audio_processing/aec_dump/capture_stream_info.h"
#include "modules/audio_processing/aec_dump/write_to_file_task.h"
#include "modules/audio_processing/include/aec_dump.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/ignore_wundef.h"
#include "rtc_base/platform_file.h"
#include "rtc_base/race_checker.h"
+#include "rtc_base/system/file_wrapper.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/thread_annotations.h"
-#include "system_wrappers/include/file_wrapper.h"
// Files generated at build-time by the protobuf compiler.
RTC_PUSH_IGNORING_WUNDEF()
diff --git a/modules/audio_processing/aec_dump/aec_dump_unittest.cc b/modules/audio_processing/aec_dump/aec_dump_unittest.cc
index 965ac03..98640b9 100644
--- a/modules/audio_processing/aec_dump/aec_dump_unittest.cc
+++ b/modules/audio_processing/aec_dump/aec_dump_unittest.cc
@@ -12,7 +12,6 @@
#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/task_queue.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_processing/aec_dump/capture_stream_info.h b/modules/audio_processing/aec_dump/capture_stream_info.h
index 91bb1fa..da8fb58 100644
--- a/modules/audio_processing/aec_dump/capture_stream_info.h
+++ b/modules/audio_processing/aec_dump/capture_stream_info.h
@@ -15,9 +15,9 @@
#include <utility>
#include <vector>
+#include "api/audio/audio_frame.h"
#include "modules/audio_processing/aec_dump/write_to_file_task.h"
#include "modules/audio_processing/include/aec_dump.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/ignore_wundef.h"
#include "rtc_base/logging.h"
diff --git a/modules/audio_processing/aec_dump/mock_aec_dump.h b/modules/audio_processing/aec_dump/mock_aec_dump.h
index 8cfabdd..c2088c0 100644
--- a/modules/audio_processing/aec_dump/mock_aec_dump.h
+++ b/modules/audio_processing/aec_dump/mock_aec_dump.h
@@ -14,7 +14,6 @@
#include <memory>
#include "modules/audio_processing/include/aec_dump.h"
-#include "modules/include/module_common_types.h"
#include "test/gmock.h"
namespace webrtc {
diff --git a/modules/audio_processing/aec_dump/write_to_file_task.h b/modules/audio_processing/aec_dump/write_to_file_task.h
index 7301473..711afb2 100644
--- a/modules/audio_processing/aec_dump/write_to_file_task.h
+++ b/modules/audio_processing/aec_dump/write_to_file_task.h
@@ -19,8 +19,8 @@
#include "rtc_base/event.h"
#include "rtc_base/ignore_wundef.h"
#include "rtc_base/platform_file.h"
+#include "rtc_base/system/file_wrapper.h"
#include "rtc_base/task_queue.h"
-#include "system_wrappers/include/file_wrapper.h"
// Files generated at build-time by the protobuf compiler.
RTC_PUSH_IGNORING_WUNDEF()
diff --git a/modules/audio_processing/agc/agc.cc b/modules/audio_processing/agc/agc.cc
index e161676..12c8cfb 100644
--- a/modules/audio_processing/agc/agc.cc
+++ b/modules/audio_processing/agc/agc.cc
@@ -18,7 +18,6 @@
#include "modules/audio_processing/agc/loudness_histogram.h"
#include "modules/audio_processing/agc/utility.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
namespace webrtc {
diff --git a/modules/audio_processing/agc/agc_manager_direct.cc b/modules/audio_processing/agc/agc_manager_direct.cc
index 5ba5f4f..2d6ee81 100644
--- a/modules/audio_processing/agc/agc_manager_direct.cc
+++ b/modules/audio_processing/agc/agc_manager_direct.cc
@@ -18,7 +18,6 @@
#include "modules/audio_processing/agc/gain_map_internal.h"
#include "modules/audio_processing/gain_control_impl.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_minmax.h"
diff --git a/modules/audio_processing/agc/legacy/gain_control.h b/modules/audio_processing/agc/legacy/gain_control.h
index 0f121b1..05b7ff6 100644
--- a/modules/audio_processing/agc/legacy/gain_control.h
+++ b/modules/audio_processing/agc/legacy/gain_control.h
@@ -208,7 +208,7 @@
* This function creates and returns an AGC instance, which will contain the
* state information for one (duplex) channel.
*/
-void* WebRtcAgc_Create();
+void* WebRtcAgc_Create(void);
/*
* This function frees the AGC instance created at the beginning.
diff --git a/modules/audio_processing/agc/loudness_histogram.cc b/modules/audio_processing/agc/loudness_histogram.cc
index 63d5f7c..0ed5850 100644
--- a/modules/audio_processing/agc/loudness_histogram.cc
+++ b/modules/audio_processing/agc/loudness_histogram.cc
@@ -13,7 +13,6 @@
#include <cmath>
#include <cstring>
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
namespace webrtc {
diff --git a/modules/audio_processing/agc/mock_agc.h b/modules/audio_processing/agc/mock_agc.h
index b27d28c..cf2a859 100644
--- a/modules/audio_processing/agc/mock_agc.h
+++ b/modules/audio_processing/agc/mock_agc.h
@@ -13,7 +13,6 @@
#include "modules/audio_processing/agc/agc.h"
-#include "modules/include/module_common_types.h"
#include "test/gmock.h"
namespace webrtc {
diff --git a/modules/audio_processing/agc2/BUILD.gn b/modules/audio_processing/agc2/BUILD.gn
index aca80d4..6f92f84 100644
--- a/modules/audio_processing/agc2/BUILD.gn
+++ b/modules/audio_processing/agc2/BUILD.gn
@@ -8,9 +8,47 @@
import("../../../webrtc.gni")
-rtc_source_set("agc2") {
+group("agc2") {
+ deps = [
+ ":adaptive_digital",
+ ":fixed_digital",
+ ]
+}
+
+rtc_source_set("adaptive_digital") {
sources = [
- "agc2_common.h",
+ "adaptive_agc.cc",
+ "adaptive_agc.h",
+ "adaptive_digital_gain_applier.cc",
+ "adaptive_digital_gain_applier.h",
+ "adaptive_mode_level_estimator.cc",
+ "adaptive_mode_level_estimator.h",
+ "saturation_protector.cc",
+ "saturation_protector.h",
+ ]
+
+ configs += [ "..:apm_debug_dump" ]
+
+ deps = [
+ ":common",
+ ":gain_applier",
+ ":noise_level_estimator",
+ "..:aec_core",
+ "..:apm_logging",
+ "..:audio_frame_view",
+ "../../../api:array_view",
+ "../../../common_audio",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:rtc_base_approved",
+ "../../../rtc_base:safe_minmax",
+ "../vad",
+ "../vad:vad_with_level",
+ "rnn_vad",
+ ]
+}
+
+rtc_source_set("fixed_digital") {
+ sources = [
"fixed_digital_level_estimator.cc",
"fixed_digital_level_estimator.h",
"fixed_gain_controller.cc",
@@ -24,6 +62,7 @@
configs += [ "..:apm_debug_dump" ]
deps = [
+ ":common",
"..:apm_logging",
"..:audio_frame_view",
"../../../api:array_view",
@@ -36,13 +75,74 @@
]
}
+rtc_source_set("common") {
+ sources = [
+ "agc2_common.h",
+ ]
+ deps = [
+ "../../../rtc_base:rtc_base_approved",
+ ]
+}
+
+rtc_source_set("noise_level_estimator") {
+ sources = [
+ "biquad_filter.cc",
+ "biquad_filter.h",
+ "down_sampler.cc",
+ "down_sampler.h",
+ "noise_level_estimator.cc",
+ "noise_level_estimator.h",
+ "noise_spectrum_estimator.cc",
+ "noise_spectrum_estimator.h",
+ "signal_classifier.cc",
+ "signal_classifier.h",
+ ]
+ deps = [
+ "..:aec_core",
+ "..:apm_logging",
+ "..:audio_frame_view",
+ "../../../api:array_view",
+ "../../../common_audio",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:macromagic",
+ ]
+
+ configs += [ "..:apm_debug_dump" ]
+}
+
+rtc_source_set("gain_applier") {
+ sources = [
+ "gain_applier.cc",
+ "gain_applier.h",
+ ]
+ deps = [
+ ":common",
+ "..:audio_frame_view",
+ "../../../rtc_base:safe_minmax",
+ ]
+}
+
+rtc_source_set("test_utils") {
+ testonly = true
+ visibility = [ ":*" ]
+ sources = [
+ "agc2_testing_common.cc",
+ "agc2_testing_common.h",
+ "vector_float_frame.cc",
+ "vector_float_frame.h",
+ ]
+ deps = [
+ "..:audio_frame_view",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:rtc_base_approved",
+ ]
+}
+
rtc_source_set("fixed_digital_unittests") {
testonly = true
configs += [ "..:apm_debug_dump" ]
sources = [
- "agc2_testing_common.cc",
- "agc2_testing_common.h",
"agc2_testing_common_unittest.cc",
"compute_interpolated_gain_curve.cc",
"compute_interpolated_gain_curve.h",
@@ -53,11 +153,11 @@
"limiter.cc",
"limiter.h",
"limiter_unittest.cc",
- "vector_float_frame.cc",
- "vector_float_frame.h",
]
deps = [
- ":agc2",
+ ":common",
+ ":fixed_digital",
+ ":test_utils",
"..:apm_logging",
"..:audio_frame_view",
"../../../api:array_view",
@@ -67,3 +167,49 @@
"../../../rtc_base:rtc_base_tests_utils",
]
}
+
+rtc_source_set("adaptive_digital_unittests") {
+ testonly = true
+ configs += [ "..:apm_debug_dump" ]
+
+ sources = [
+ "adaptive_digital_gain_applier_unittest.cc",
+ "adaptive_mode_level_estimator_unittest.cc",
+ "gain_applier_unittest.cc",
+ "saturation_protector_unittest.cc",
+ ]
+ deps = [
+ ":adaptive_digital",
+ ":common",
+ ":gain_applier",
+ ":test_utils",
+ "..:apm_logging",
+ "..:audio_frame_view",
+ "../../../api:array_view",
+ "../../../common_audio",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:rtc_base_approved",
+ "../../../rtc_base:rtc_base_tests_utils",
+ "../vad:vad_with_level",
+ ]
+}
+
+rtc_source_set("noise_estimator_unittests") {
+ testonly = true
+ configs += [ "..:apm_debug_dump" ]
+
+ sources = [
+ "noise_level_estimator_unittest.cc",
+ "signal_classifier_unittest.cc",
+ ]
+ deps = [
+ ":noise_level_estimator",
+ ":test_utils",
+ "..:apm_logging",
+ "..:audio_frame_view",
+ "../../../api:array_view",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:rtc_base_approved",
+ "../../../rtc_base:rtc_base_tests_utils",
+ ]
+}
diff --git a/modules/audio_processing/agc2/adaptive_agc.cc b/modules/audio_processing/agc2/adaptive_agc.cc
new file mode 100644
index 0000000..45e8853
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_agc.cc
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/adaptive_agc.h"
+
+#include <algorithm>
+#include <numeric>
+
+#include "common_audio/include/audio_util.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "modules/audio_processing/vad/voice_activity_detector.h"
+
+namespace webrtc {
+
+AdaptiveAgc::AdaptiveAgc(ApmDataDumper* apm_data_dumper)
+ : speech_level_estimator_(apm_data_dumper),
+ gain_applier_(apm_data_dumper),
+ apm_data_dumper_(apm_data_dumper),
+ noise_level_estimator_(apm_data_dumper) {
+ RTC_DCHECK(apm_data_dumper);
+}
+
+AdaptiveAgc::~AdaptiveAgc() = default;
+
+void AdaptiveAgc::Process(AudioFrameView<float> float_frame) {
+ // TODO(webrtc:7494): Remove this loop. Remove the vectors from
+ // VadWithData after we move to a VAD that outputs an estimate every
+ // kFrameDurationMs ms.
+ //
+ // Some VADs are 'bursty'. They return several estimates for some
+ // frames, and no estimates for other frames. We want to feed all to
+ // the level estimator, but only care about the last level it
+ // produces.
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability> vad_results =
+ vad_.AnalyzeFrame(float_frame);
+ for (const auto& vad_result : vad_results) {
+ apm_data_dumper_->DumpRaw("agc2_vad_probability",
+ vad_result.speech_probability);
+ apm_data_dumper_->DumpRaw("agc2_vad_rms_dbfs", vad_result.speech_rms_dbfs);
+
+ apm_data_dumper_->DumpRaw("agc2_vad_peak_dbfs",
+ vad_result.speech_peak_dbfs);
+ speech_level_estimator_.UpdateEstimation(vad_result);
+ }
+
+ const float speech_level_dbfs = speech_level_estimator_.LatestLevelEstimate();
+
+ const float noise_level_dbfs = noise_level_estimator_.Analyze(float_frame);
+
+ apm_data_dumper_->DumpRaw("agc2_noise_estimate_dbfs", noise_level_dbfs);
+
+ // The gain applier applies the gain.
+ gain_applier_.Process(speech_level_dbfs, noise_level_dbfs, vad_results,
+ float_frame);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/adaptive_agc.h b/modules/audio_processing/agc2/adaptive_agc.h
new file mode 100644
index 0000000..a91aa2a
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_agc.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_AGC_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_AGC_H_
+
+#include <memory>
+
+#include "modules/audio_processing/agc2/adaptive_digital_gain_applier.h"
+#include "modules/audio_processing/agc2/adaptive_mode_level_estimator.h"
+#include "modules/audio_processing/agc2/noise_level_estimator.h"
+#include "modules/audio_processing/include/audio_frame_view.h"
+#include "modules/audio_processing/vad/vad_with_level.h"
+
+namespace webrtc {
+class ApmDataDumper;
+
+class AdaptiveAgc {
+ public:
+ explicit AdaptiveAgc(ApmDataDumper* apm_data_dumper);
+ void Process(AudioFrameView<float> float_frame);
+ ~AdaptiveAgc();
+
+ private:
+ AdaptiveModeLevelEstimator speech_level_estimator_;
+ VadWithLevel vad_;
+ AdaptiveDigitalGainApplier gain_applier_;
+ ApmDataDumper* const apm_data_dumper_;
+ NoiseLevelEstimator noise_level_estimator_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_AGC_H_
diff --git a/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc b/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc
new file mode 100644
index 0000000..20b5a27
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_digital_gain_applier.cc
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/adaptive_digital_gain_applier.h"
+
+#include <algorithm>
+
+#include "common_audio/include/audio_util.h"
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/numerics/safe_minmax.h"
+
+namespace webrtc {
+namespace {
+
+// This function maps input level to desired applied gain. We want to
+// boost the signal so that peaks are at -kHeadroomDbfs. We can't
+// apply more than kMaxGainDb gain.
+float ComputeGainDb(float input_level_dbfs) {
+ // If the level is very low, boost it as much as we can.
+ if (input_level_dbfs < -(kHeadroomDbfs + kMaxGainDb)) {
+ return kMaxGainDb;
+ }
+
+ // We expect to end up here most of the time: the level is below
+ // -headroom, but we can boost it to -headroom.
+ if (input_level_dbfs < -kHeadroomDbfs) {
+ return -kHeadroomDbfs - input_level_dbfs;
+ }
+
+ // Otherwise, the level is too high and we can't boost. The
+ // LevelEstimator is responsible for not reporting bogus gain
+ // values.
+ RTC_DCHECK_LE(input_level_dbfs, 0.f);
+ return 0.f;
+}
+
+// We require 'gain + noise_level <= kMaxNoiseLevelDbfs'.
+float LimitGainByNoise(float target_gain,
+ float input_noise_level_dbfs,
+ ApmDataDumper* apm_data_dumper) {
+ const float noise_headroom_db = kMaxNoiseLevelDbfs - input_noise_level_dbfs;
+ apm_data_dumper->DumpRaw("agc2_noise_headroom_db", noise_headroom_db);
+ return std::min(target_gain, std::max(noise_headroom_db, 0.f));
+}
+
+// Computes how the gain should change during this frame.
+// Return the gain difference in db to 'last_gain_db'.
+float ComputeGainChangeThisFrameDb(float target_gain_db,
+ float last_gain_db,
+ bool gain_increase_allowed) {
+ float target_gain_difference_db = target_gain_db - last_gain_db;
+ if (!gain_increase_allowed) {
+ target_gain_difference_db = std::min(target_gain_difference_db, 0.f);
+ }
+
+ return rtc::SafeClamp(target_gain_difference_db, -kMaxGainChangePerFrameDb,
+ kMaxGainChangePerFrameDb);
+}
+} // namespace
+
+AdaptiveDigitalGainApplier::AdaptiveDigitalGainApplier(
+ ApmDataDumper* apm_data_dumper)
+ : gain_applier_(false, DbToRatio(last_gain_db_)),
+ apm_data_dumper_(apm_data_dumper) {}
+
+void AdaptiveDigitalGainApplier::Process(
+ float input_level_dbfs,
+ float input_noise_level_dbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability> vad_results,
+ AudioFrameView<float> float_frame) {
+ RTC_DCHECK_GE(input_level_dbfs, -150.f);
+ RTC_DCHECK_LE(input_level_dbfs, 0.f);
+ RTC_DCHECK_GE(float_frame.num_channels(), 1);
+ RTC_DCHECK_GE(float_frame.samples_per_channel(), 1);
+
+ const float target_gain_db =
+ LimitGainByNoise(ComputeGainDb(input_level_dbfs), input_noise_level_dbfs,
+ apm_data_dumper_);
+
+ // TODO(webrtc:7494): Remove this construct. Remove the vectors from
+ // VadWithData after we move to a VAD that outputs an estimate every
+ // kFrameDurationMs ms.
+ //
+ // Forbid increasing the gain when there is no speech. For some
+ // VADs, 'vad_results' has either many or 0 results. If there are 0
+ // results, keep the old flag. If there are many results, and at
+ // least one is confident speech, we allow attenuation.
+ if (!vad_results.empty()) {
+ gain_increase_allowed_ = std::all_of(
+ vad_results.begin(), vad_results.end(),
+ [](const VadWithLevel::LevelAndProbability& vad_result) {
+ return vad_result.speech_probability > kVadConfidenceThreshold;
+ });
+ }
+
+ const float gain_change_this_frame_db = ComputeGainChangeThisFrameDb(
+ target_gain_db, last_gain_db_, gain_increase_allowed_);
+
+ apm_data_dumper_->DumpRaw("agc2_want_to_change_by_db",
+ target_gain_db - last_gain_db_);
+ apm_data_dumper_->DumpRaw("agc2_will_change_by_db",
+ gain_change_this_frame_db);
+
+ // Optimization: avoid calling math functions if gain does not
+ // change.
+ if (gain_change_this_frame_db != 0.f) {
+ gain_applier_.SetGainFactor(
+ DbToRatio(last_gain_db_ + gain_change_this_frame_db));
+ }
+ gain_applier_.ApplyGain(float_frame);
+
+ // Remember that the gain has changed for the next iteration.
+ last_gain_db_ = last_gain_db_ + gain_change_this_frame_db;
+ apm_data_dumper_->DumpRaw("agc2_applied_gain_db", last_gain_db_);
+}
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/adaptive_digital_gain_applier.h b/modules/audio_processing/agc2/adaptive_digital_gain_applier.h
new file mode 100644
index 0000000..b06c65b
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_digital_gain_applier.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_APPLIER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_APPLIER_H_
+
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/agc2/gain_applier.h"
+#include "modules/audio_processing/include/audio_frame_view.h"
+#include "modules/audio_processing/vad/vad_with_level.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+
+class AdaptiveDigitalGainApplier {
+ public:
+ explicit AdaptiveDigitalGainApplier(ApmDataDumper* apm_data_dumper);
+ // Decide what gain to apply.
+ void Process(
+ float input_level_dbfs,
+ float input_noise_level_dbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability> vad_results,
+ AudioFrameView<float> float_frame);
+
+ private:
+ float last_gain_db_ = kInitialAdaptiveDigitalGainDb;
+ GainApplier gain_applier_;
+
+ // For some combinations of noise and speech probability, increasing
+ // the level is not allowed. Since we may get VAD results in bursts,
+ // we keep track of this variable until the next VAD results come
+ // in.
+ bool gain_increase_allowed_ = true;
+ ApmDataDumper* apm_data_dumper_ = nullptr;
+};
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_DIGITAL_GAIN_APPLIER_H_
diff --git a/modules/audio_processing/agc2/adaptive_digital_gain_applier_unittest.cc b/modules/audio_processing/agc2/adaptive_digital_gain_applier_unittest.cc
new file mode 100644
index 0000000..ebb040e
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_digital_gain_applier_unittest.cc
@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/adaptive_digital_gain_applier.h"
+
+#include <algorithm>
+
+#include "common_audio/include/audio_util.h"
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/agc2/vector_float_frame.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/gunit.h"
+
+namespace webrtc {
+namespace {
+// Constants used in place of estimated noise levels.
+constexpr float kNoNoiseDbfs = -90.f;
+constexpr float kWithNoiseDbfs = -20.f;
+
+// Runs gain applier and returns the applied gain in linear scale.
+float RunOnConstantLevel(int num_iterations,
+ VadWithLevel::LevelAndProbability vad_data,
+ float input_level_dbfs,
+ AdaptiveDigitalGainApplier* gain_applier) {
+ float gain_linear = 0.f;
+
+ for (int i = 0; i < num_iterations; ++i) {
+ VectorFloatFrame fake_audio(1, 1, 1.f);
+ gain_applier->Process(
+ input_level_dbfs, kNoNoiseDbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability>(&vad_data, 1),
+ fake_audio.float_frame_view());
+ gain_linear = fake_audio.float_frame_view().channel(0)[0];
+ }
+ return gain_linear;
+}
+
+constexpr VadWithLevel::LevelAndProbability kVadSpeech(1.f, -20.f, 0.f);
+} // namespace
+
+TEST(AutomaticGainController2AdaptiveGainApplier, GainApplierShouldNotCrash) {
+ static_assert(
+ std::is_trivially_destructible<VadWithLevel::LevelAndProbability>::value,
+ "");
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveDigitalGainApplier gain_applier(&apm_data_dumper);
+
+ // Make one call with reasonable audio level values and settings.
+ VectorFloatFrame fake_audio(2, 480, 10000.f);
+ gain_applier.Process(
+ -5.0, kNoNoiseDbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability>(&kVadSpeech, 1),
+ fake_audio.float_frame_view());
+}
+
+// Check that the output is -kHeadroom dBFS.
+TEST(AutomaticGainController2AdaptiveGainApplier, TargetLevelIsReached) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveDigitalGainApplier gain_applier(&apm_data_dumper);
+
+ constexpr float initial_level_dbfs = -5.f;
+
+ const float applied_gain =
+ RunOnConstantLevel(200, kVadSpeech, initial_level_dbfs, &gain_applier);
+
+ EXPECT_NEAR(applied_gain, DbToRatio(-kHeadroomDbfs - initial_level_dbfs),
+ 0.1f);
+}
+
+// Check that the output is -kHeadroom dBFS
+TEST(AutomaticGainController2AdaptiveGainApplier, GainApproachesMaxGain) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveDigitalGainApplier gain_applier(&apm_data_dumper);
+
+ constexpr float initial_level_dbfs = -kHeadroomDbfs - kMaxGainDb - 10.f;
+ // A few extra frames for safety.
+ constexpr int kNumFramesToAdapt =
+ static_cast<int>(kMaxGainDb / kMaxGainChangePerFrameDb) + 10;
+
+ const float applied_gain = RunOnConstantLevel(
+ kNumFramesToAdapt, kVadSpeech, initial_level_dbfs, &gain_applier);
+ EXPECT_NEAR(applied_gain, DbToRatio(kMaxGainDb), 0.1f);
+
+ const float applied_gain_db = 20.f * std::log10(applied_gain);
+ EXPECT_NEAR(applied_gain_db, kMaxGainDb, 0.1f);
+}
+
+TEST(AutomaticGainController2AdaptiveGainApplier, GainDoesNotChangeFast) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveDigitalGainApplier gain_applier(&apm_data_dumper);
+
+ constexpr float initial_level_dbfs = -25.f;
+ // A few extra frames for safety.
+ constexpr int kNumFramesToAdapt =
+ static_cast<int>(initial_level_dbfs / kMaxGainChangePerFrameDb) + 10;
+
+ const float kMaxChangePerFrameLinear = DbToRatio(kMaxGainChangePerFrameDb);
+
+ float last_gain_linear = 1.f;
+ for (int i = 0; i < kNumFramesToAdapt; ++i) {
+ SCOPED_TRACE(i);
+ VectorFloatFrame fake_audio(1, 1, 1.f);
+ gain_applier.Process(
+ initial_level_dbfs, kNoNoiseDbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability>(&kVadSpeech, 1),
+ fake_audio.float_frame_view());
+ float current_gain_linear = fake_audio.float_frame_view().channel(0)[0];
+ EXPECT_LE(std::abs(current_gain_linear - last_gain_linear),
+ kMaxChangePerFrameLinear);
+ last_gain_linear = current_gain_linear;
+ }
+
+ // Check that the same is true when gain decreases as well.
+ for (int i = 0; i < kNumFramesToAdapt; ++i) {
+ SCOPED_TRACE(i);
+ VectorFloatFrame fake_audio(1, 1, 1.f);
+ gain_applier.Process(
+ 0.f, kNoNoiseDbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability>(&kVadSpeech, 1),
+ fake_audio.float_frame_view());
+ float current_gain_linear = fake_audio.float_frame_view().channel(0)[0];
+ EXPECT_LE(std::abs(current_gain_linear - last_gain_linear),
+ kMaxChangePerFrameLinear);
+ last_gain_linear = current_gain_linear;
+ }
+}
+
+TEST(AutomaticGainController2AdaptiveGainApplier, GainIsRampedInAFrame) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveDigitalGainApplier gain_applier(&apm_data_dumper);
+
+ constexpr float initial_level_dbfs = -25.f;
+ constexpr int num_samples = 480;
+
+ VectorFloatFrame fake_audio(1, num_samples, 1.f);
+ gain_applier.Process(
+ initial_level_dbfs, kNoNoiseDbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability>(&kVadSpeech, 1),
+ fake_audio.float_frame_view());
+ float maximal_difference = 0.f;
+ float current_value = 1.f * DbToRatio(kInitialAdaptiveDigitalGainDb);
+ for (const auto& x : fake_audio.float_frame_view().channel(0)) {
+ const float difference = std::abs(x - current_value);
+ maximal_difference = std::max(maximal_difference, difference);
+ current_value = x;
+ }
+
+ const float kMaxChangePerFrameLinear = DbToRatio(kMaxGainChangePerFrameDb);
+ const float kMaxChangePerSample = kMaxChangePerFrameLinear / num_samples;
+
+ EXPECT_LE(maximal_difference, kMaxChangePerSample);
+}
+
+TEST(AutomaticGainController2AdaptiveGainApplier, NoiseLimitsGain) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveDigitalGainApplier gain_applier(&apm_data_dumper);
+
+ constexpr float initial_level_dbfs = -25.f;
+ constexpr int num_samples = 480;
+ constexpr int num_initial_frames =
+ kInitialAdaptiveDigitalGainDb / kMaxGainChangePerFrameDb;
+ constexpr int num_frames = 50;
+
+ ASSERT_GT(kWithNoiseDbfs, kMaxNoiseLevelDbfs) << "kWithNoiseDbfs is too low";
+
+ for (int i = 0; i < num_initial_frames + num_frames; ++i) {
+ VectorFloatFrame fake_audio(1, num_samples, 1.f);
+ gain_applier.Process(
+ initial_level_dbfs, kWithNoiseDbfs,
+ rtc::ArrayView<const VadWithLevel::LevelAndProbability>(&kVadSpeech, 1),
+ fake_audio.float_frame_view());
+
+ // Wait so that the adaptive gain applier has time to lower the gain.
+ if (i > num_initial_frames) {
+ const float maximal_ratio =
+ *std::max_element(fake_audio.float_frame_view().channel(0).begin(),
+ fake_audio.float_frame_view().channel(0).end());
+
+ EXPECT_NEAR(maximal_ratio, 1.f, 0.001f);
+ }
+ }
+}
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc b/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc
new file mode 100644
index 0000000..6aa2e91
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_mode_level_estimator.cc
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/adaptive_mode_level_estimator.h"
+
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/numerics/safe_minmax.h"
+
+namespace webrtc {
+
+AdaptiveModeLevelEstimator::AdaptiveModeLevelEstimator(
+ ApmDataDumper* apm_data_dumper)
+ : saturation_protector_(apm_data_dumper),
+ apm_data_dumper_(apm_data_dumper) {}
+
+void AdaptiveModeLevelEstimator::UpdateEstimation(
+ const VadWithLevel::LevelAndProbability& vad_data) {
+ RTC_DCHECK_GT(vad_data.speech_rms_dbfs, -150.f);
+ RTC_DCHECK_LT(vad_data.speech_rms_dbfs, 50.f);
+ RTC_DCHECK_GT(vad_data.speech_peak_dbfs, -150.f);
+ RTC_DCHECK_LT(vad_data.speech_peak_dbfs, 50.f);
+ RTC_DCHECK_GE(vad_data.speech_probability, 0.f);
+ RTC_DCHECK_LE(vad_data.speech_probability, 1.f);
+
+ if (vad_data.speech_probability < kVadConfidenceThreshold) {
+ DebugDumpEstimate();
+ return;
+ }
+
+ const bool buffer_is_full = buffer_size_ms_ >= kFullBufferSizeMs;
+ if (!buffer_is_full) {
+ buffer_size_ms_ += kFrameDurationMs;
+ }
+
+ const float leak_factor = buffer_is_full ? kFullBufferLeakFactor : 1.f;
+
+ estimate_numerator_ = estimate_numerator_ * leak_factor +
+ vad_data.speech_rms_dbfs * vad_data.speech_probability;
+ estimate_denominator_ =
+ estimate_denominator_ * leak_factor + vad_data.speech_probability;
+
+ last_estimate_with_offset_dbfs_ = estimate_numerator_ / estimate_denominator_;
+
+ saturation_protector_.UpdateMargin(vad_data, last_estimate_with_offset_dbfs_);
+ DebugDumpEstimate();
+}
+
+float AdaptiveModeLevelEstimator::LatestLevelEstimate() const {
+ return rtc::SafeClamp<float>(
+ last_estimate_with_offset_dbfs_ + saturation_protector_.LastMargin(),
+ -90.f, 0.f);
+}
+
+void AdaptiveModeLevelEstimator::DebugDumpEstimate() {
+ apm_data_dumper_->DumpRaw("agc2_adaptive_level_estimate_with_offset_dbfs",
+ last_estimate_with_offset_dbfs_);
+ apm_data_dumper_->DumpRaw("agc2_adaptive_level_estimate_dbfs",
+ LatestLevelEstimate());
+ saturation_protector_.DebugDumpEstimate();
+}
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/adaptive_mode_level_estimator.h b/modules/audio_processing/agc2/adaptive_mode_level_estimator.h
new file mode 100644
index 0000000..9762f1f
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_mode_level_estimator.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_H_
+
+#include "modules/audio_processing/agc2/saturation_protector.h"
+#include "modules/audio_processing/vad/vad_with_level.h"
+
+namespace webrtc {
+class ApmDataDumper;
+
+class AdaptiveModeLevelEstimator {
+ public:
+ explicit AdaptiveModeLevelEstimator(ApmDataDumper* apm_data_dumper);
+ void UpdateEstimation(const VadWithLevel::LevelAndProbability& vad_data);
+ float LatestLevelEstimate() const;
+
+ private:
+ void DebugDumpEstimate();
+
+ size_t buffer_size_ms_ = 0;
+ float last_estimate_with_offset_dbfs_ = kInitialSpeechLevelEstimateDbfs;
+ float estimate_numerator_ = 0.f;
+ float estimate_denominator_ = 0.f;
+ SaturationProtector saturation_protector_;
+ ApmDataDumper* const apm_data_dumper_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_ADAPTIVE_MODE_LEVEL_ESTIMATOR_H_
diff --git a/modules/audio_processing/agc2/adaptive_mode_level_estimator_unittest.cc b/modules/audio_processing/agc2/adaptive_mode_level_estimator_unittest.cc
new file mode 100644
index 0000000..71909d0
--- /dev/null
+++ b/modules/audio_processing/agc2/adaptive_mode_level_estimator_unittest.cc
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/adaptive_mode_level_estimator.h"
+
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/gunit.h"
+
+namespace webrtc {
+namespace {
+void RunOnConstantLevel(int num_iterations,
+ VadWithLevel::LevelAndProbability vad_data,
+ AdaptiveModeLevelEstimator* level_estimator) {
+ for (int i = 0; i < num_iterations; ++i) {
+ level_estimator->UpdateEstimation(vad_data); // By copy
+ }
+}
+} // namespace
+
+TEST(AutomaticGainController2AdaptiveModeLevelEstimator,
+ EstimatorShouldNotCrash) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveModeLevelEstimator level_estimator(&apm_data_dumper);
+
+ VadWithLevel::LevelAndProbability vad_data(1.f, -20.f, -10.f);
+ level_estimator.UpdateEstimation(vad_data);
+ static_cast<void>(level_estimator.LatestLevelEstimate());
+}
+
+TEST(AutomaticGainController2AdaptiveModeLevelEstimator, LevelShouldStabilize) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveModeLevelEstimator level_estimator(&apm_data_dumper);
+
+ constexpr float kSpeechRmsDbfs = -15.f;
+ RunOnConstantLevel(
+ 100,
+ VadWithLevel::LevelAndProbability(
+ 1.f, kSpeechRmsDbfs - kInitialSaturationMarginDb, kSpeechRmsDbfs),
+ &level_estimator);
+
+ EXPECT_NEAR(level_estimator.LatestLevelEstimate(), kSpeechRmsDbfs, 0.1f);
+}
+
+TEST(AutomaticGainController2AdaptiveModeLevelEstimator,
+ EstimatorIgnoresZeroProbabilityFrames) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveModeLevelEstimator level_estimator(&apm_data_dumper);
+
+ // Run for one second of fake audio.
+ constexpr float kSpeechRmsDbfs = -25.f;
+ RunOnConstantLevel(
+ 100,
+ VadWithLevel::LevelAndProbability(
+ 1.f, kSpeechRmsDbfs - kInitialSaturationMarginDb, kSpeechRmsDbfs),
+ &level_estimator);
+
+ // Run for one more second, but mark as not speech.
+ constexpr float kNoiseRmsDbfs = 0.f;
+ RunOnConstantLevel(
+ 100, VadWithLevel::LevelAndProbability(0.f, kNoiseRmsDbfs, kNoiseRmsDbfs),
+ &level_estimator);
+
+ // Level should not have changed.
+ EXPECT_NEAR(level_estimator.LatestLevelEstimate(), kSpeechRmsDbfs, 0.1f);
+}
+
+TEST(AutomaticGainController2AdaptiveModeLevelEstimator, TimeToAdapt) {
+ ApmDataDumper apm_data_dumper(0);
+ AdaptiveModeLevelEstimator level_estimator(&apm_data_dumper);
+
+ // Run for one 'window size' interval
+ constexpr float kInitialSpeechRmsDbfs = -30.f;
+ RunOnConstantLevel(
+ kFullBufferSizeMs / kFrameDurationMs,
+ VadWithLevel::LevelAndProbability(
+ 1.f, kInitialSpeechRmsDbfs - kInitialSaturationMarginDb,
+ kInitialSpeechRmsDbfs),
+ &level_estimator);
+
+ // Run for one half 'window size' interval. This should not be enough to
+ // adapt.
+ constexpr float kDifferentSpeechRmsDbfs = -10.f;
+ // It should at most differ by 25% after one 'window size' interval.
+ const float kMaxDifferenceDb =
+ 0.25 * std::abs(kDifferentSpeechRmsDbfs - kInitialSpeechRmsDbfs);
+ RunOnConstantLevel(
+ static_cast<int>(kFullBufferSizeMs / kFrameDurationMs / 2),
+ VadWithLevel::LevelAndProbability(
+ 1.f, kDifferentSpeechRmsDbfs - kInitialSaturationMarginDb,
+ kDifferentSpeechRmsDbfs),
+ &level_estimator);
+ EXPECT_GT(
+ std::abs(kDifferentSpeechRmsDbfs - level_estimator.LatestLevelEstimate()),
+ kMaxDifferenceDb);
+
+ // Run for some more time. Afterwards, we should have adapted.
+ RunOnConstantLevel(
+ static_cast<int>(3 * kFullBufferSizeMs / kFrameDurationMs),
+ VadWithLevel::LevelAndProbability(
+ 1.f, kDifferentSpeechRmsDbfs - kInitialSaturationMarginDb,
+ kDifferentSpeechRmsDbfs),
+ &level_estimator);
+ EXPECT_NEAR(level_estimator.LatestLevelEstimate(), kDifferentSpeechRmsDbfs,
+ kMaxDifferenceDb);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/agc2_common.h b/modules/audio_processing/agc2/agc2_common.h
index ad0ab4e..3a499d9 100644
--- a/modules/audio_processing/agc2/agc2_common.h
+++ b/modules/audio_processing/agc2/agc2_common.h
@@ -19,7 +19,7 @@
constexpr float kMinFloatS16Value = -32768.f;
constexpr float kMaxFloatS16Value = 32767.f;
-constexpr double kMaxAbsFloatS16Value = 32768.0;
+constexpr float kMaxAbsFloatS16Value = 32768.0f;
constexpr size_t kFrameDurationMs = 10;
constexpr size_t kSubFramesInFrame = 20;
@@ -27,6 +27,44 @@
constexpr float kAttackFilterConstant = 0.f;
+// Adaptive digital gain applier settings below.
+constexpr float kMaxGainChangePerSecondDb = 3.f;
+constexpr float kMaxGainChangePerFrameDb =
+ kMaxGainChangePerSecondDb * kFrameDurationMs / 1000.f;
+constexpr float kHeadroomDbfs = 1.f;
+constexpr float kMaxGainDb = 30.f;
+constexpr float kInitialAdaptiveDigitalGainDb = 8.f;
+
+// This parameter must be tuned together with the noise estimator.
+constexpr float kMaxNoiseLevelDbfs = -50.f;
+
+// Used in the Level Estimator for deciding when to update the speech
+// level estimate. Also used in the adaptive digital gain applier to
+// decide when to allow target gain reduction.
+constexpr float kVadConfidenceThreshold = 0.9f;
+
+// The amount of 'memory' of the Level Estimator. Decides leak factors.
+constexpr size_t kFullBufferSizeMs = 1000;
+constexpr float kFullBufferLeakFactor = 1.f - 1.f / kFullBufferSizeMs;
+
+constexpr float kInitialSpeechLevelEstimateDbfs = -30.f;
+
+// Saturation Protector settings.
+constexpr float kInitialSaturationMarginDb = 17.f;
+
+constexpr size_t kPeakEnveloperSuperFrameLengthMs = 500;
+
+constexpr size_t kPeakEnveloperBufferSize =
+ kFullBufferSizeMs / kPeakEnveloperSuperFrameLengthMs + 1;
+
+// This value is 10 ** (-1/20 * frame_size_ms / satproc_attack_ms),
+// where satproc_attack_ms is 5000.
+constexpr float kSaturationProtectorAttackConstant = 0.9988493699365052f;
+
+// This value is 10 ** (-1/20 * frame_size_ms / satproc_decay_ms),
+// where satproc_decay_ms is 1000.
+constexpr float kSaturationProtectorDecayConstant = 0.9997697679981565f;
+
// This is computed from kDecayMs by
// 10 ** (-1/20 * subframe_duration / kDecayMs).
// |subframe_duration| is |kFrameDurationMs / kSubFramesInFrame|.
diff --git a/modules/audio_processing/agc2/agc2_testing_common.h b/modules/audio_processing/agc2/agc2_testing_common.h
index a176282..8c4f400 100644
--- a/modules/audio_processing/agc2/agc2_testing_common.h
+++ b/modules/audio_processing/agc2/agc2_testing_common.h
@@ -11,9 +11,13 @@
#ifndef MODULES_AUDIO_PROCESSING_AGC2_AGC2_TESTING_COMMON_H_
#define MODULES_AUDIO_PROCESSING_AGC2_AGC2_TESTING_COMMON_H_
+#include <math.h>
+
+#include <limits>
#include <vector>
#include "rtc_base/basictypes.h"
+#include "rtc_base/checks.h"
namespace webrtc {
@@ -26,8 +30,49 @@
constexpr float kLimiterMaxInputLevelDbFs = 1.f;
constexpr float kLimiterKneeSmoothnessDb = 1.f;
constexpr float kLimiterCompressionRatio = 5.f;
+constexpr float kPi = 3.1415926536f;
std::vector<double> LinSpace(const double l, const double r, size_t num_points);
+
+class SineGenerator {
+ public:
+ SineGenerator(float frequency, int rate)
+ : frequency_(frequency), rate_(rate) {}
+ float operator()() {
+ x_radians_ += frequency_ / rate_ * 2 * kPi;
+ if (x_radians_ > 2 * kPi) {
+ x_radians_ -= 2 * kPi;
+ }
+ return 1000.f * sinf(x_radians_);
+ }
+
+ private:
+ float frequency_;
+ int rate_;
+ float x_radians_ = 0.f;
+};
+
+class PulseGenerator {
+ public:
+ PulseGenerator(float frequency, int rate)
+ : samples_period_(
+ static_cast<int>(static_cast<float>(rate) / frequency)) {
+ RTC_DCHECK_GT(rate, frequency);
+ }
+ float operator()() {
+ sample_counter_++;
+ if (sample_counter_ >= samples_period_) {
+ sample_counter_ -= samples_period_;
+ }
+ return static_cast<float>(
+ sample_counter_ == 0 ? std::numeric_limits<int16_t>::max() : 10.f);
+ }
+
+ private:
+ int samples_period_;
+ int sample_counter_ = 0;
+};
+
} // namespace test
} // namespace webrtc
diff --git a/modules/audio_processing/agc2/biquad_filter.cc b/modules/audio_processing/agc2/biquad_filter.cc
new file mode 100644
index 0000000..c15c644
--- /dev/null
+++ b/modules/audio_processing/agc2/biquad_filter.cc
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/biquad_filter.h"
+
+namespace webrtc {
+
+// This method applies a biquad filter to an input signal x to produce an
+// output signal y. The biquad coefficients are specified at the construction
+// of the object.
+void BiQuadFilter::Process(rtc::ArrayView<const float> x,
+ rtc::ArrayView<float> y) {
+ for (size_t k = 0; k < x.size(); ++k) {
+ // Use temporary variable for x[k] to allow in-place function call
+ // (that x and y refer to the same array).
+ const float tmp = x[k];
+ y[k] = coefficients_.b[0] * tmp + coefficients_.b[1] * biquad_state_.b[0] +
+ coefficients_.b[2] * biquad_state_.b[1] -
+ coefficients_.a[0] * biquad_state_.a[0] -
+ coefficients_.a[1] * biquad_state_.a[1];
+ biquad_state_.b[1] = biquad_state_.b[0];
+ biquad_state_.b[0] = tmp;
+ biquad_state_.a[1] = biquad_state_.a[0];
+ biquad_state_.a[0] = y[k];
+ }
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/biquad_filter.h b/modules/audio_processing/agc2/biquad_filter.h
new file mode 100644
index 0000000..4fd5e2e
--- /dev/null
+++ b/modules/audio_processing/agc2/biquad_filter.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_BIQUAD_FILTER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_BIQUAD_FILTER_H_
+
+#include "api/array_view.h"
+#include "rtc_base/arraysize.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+class BiQuadFilter {
+ public:
+ struct BiQuadCoefficients {
+ float b[3];
+ float a[2];
+ };
+
+ BiQuadFilter() = default;
+
+ void Initialize(const BiQuadCoefficients& coefficients) {
+ coefficients_ = coefficients;
+ }
+
+ // Produces a filtered output y of the input x. Both x and y need to
+ // have the same length.
+ void Process(rtc::ArrayView<const float> x, rtc::ArrayView<float> y);
+
+ private:
+ struct BiQuadState {
+ BiQuadState() {
+ std::fill(b, b + arraysize(b), 0.f);
+ std::fill(a, a + arraysize(a), 0.f);
+ }
+
+ float b[2];
+ float a[2];
+ };
+
+ BiQuadState biquad_state_;
+ BiQuadCoefficients coefficients_;
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(BiQuadFilter);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_BIQUAD_FILTER_H_
diff --git a/modules/audio_processing/agc2/down_sampler.cc b/modules/audio_processing/agc2/down_sampler.cc
new file mode 100644
index 0000000..50486e0
--- /dev/null
+++ b/modules/audio_processing/agc2/down_sampler.cc
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/down_sampler.h"
+
+#include <string.h>
+#include <algorithm>
+
+#include "modules/audio_processing/agc2/biquad_filter.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+
+constexpr int kChunkSizeMs = 10;
+constexpr int kSampleRate8kHz = 8000;
+constexpr int kSampleRate16kHz = 16000;
+constexpr int kSampleRate32kHz = 32000;
+constexpr int kSampleRate48kHz = 48000;
+
+// Bandlimiter coefficients computed based on that only
+// the first 40 bins of the spectrum for the downsampled
+// signal are used.
+// [B,A] = butter(2,(41/64*4000)/8000)
+const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_16kHz = {
+ {0.1455f, 0.2911f, 0.1455f},
+ {-0.6698f, 0.2520f}};
+
+// [B,A] = butter(2,(41/64*4000)/16000)
+const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_32kHz = {
+ {0.0462f, 0.0924f, 0.0462f},
+ {-1.3066f, 0.4915f}};
+
+// [B,A] = butter(2,(41/64*4000)/24000)
+const BiQuadFilter::BiQuadCoefficients kLowPassFilterCoefficients_48kHz = {
+ {0.0226f, 0.0452f, 0.0226f},
+ {-1.5320f, 0.6224f}};
+
+} // namespace
+
+DownSampler::DownSampler(ApmDataDumper* data_dumper)
+ : data_dumper_(data_dumper) {
+ Initialize(48000);
+}
+void DownSampler::Initialize(int sample_rate_hz) {
+ RTC_DCHECK(
+ sample_rate_hz == kSampleRate8kHz || sample_rate_hz == kSampleRate16kHz ||
+ sample_rate_hz == kSampleRate32kHz || sample_rate_hz == kSampleRate48kHz);
+
+ sample_rate_hz_ = sample_rate_hz;
+ down_sampling_factor_ = rtc::CheckedDivExact(sample_rate_hz_, 8000);
+
+ /// Note that the down sampling filter is not used if the sample rate is 8
+ /// kHz.
+ if (sample_rate_hz_ == kSampleRate16kHz) {
+ low_pass_filter_.Initialize(kLowPassFilterCoefficients_16kHz);
+ } else if (sample_rate_hz_ == kSampleRate32kHz) {
+ low_pass_filter_.Initialize(kLowPassFilterCoefficients_32kHz);
+ } else if (sample_rate_hz_ == kSampleRate48kHz) {
+ low_pass_filter_.Initialize(kLowPassFilterCoefficients_48kHz);
+ }
+}
+
+void DownSampler::DownSample(rtc::ArrayView<const float> in,
+ rtc::ArrayView<float> out) {
+ data_dumper_->DumpWav("lc_down_sampler_input", in, sample_rate_hz_, 1);
+ RTC_DCHECK_EQ(sample_rate_hz_ * kChunkSizeMs / 1000, in.size());
+ RTC_DCHECK_EQ(kSampleRate8kHz * kChunkSizeMs / 1000, out.size());
+ const size_t kMaxNumFrames = kSampleRate48kHz * kChunkSizeMs / 1000;
+ float x[kMaxNumFrames];
+
+ // Band-limit the signal to 4 kHz.
+ if (sample_rate_hz_ != kSampleRate8kHz) {
+ low_pass_filter_.Process(in, rtc::ArrayView<float>(x, in.size()));
+
+ // Downsample the signal.
+ size_t k = 0;
+ for (size_t j = 0; j < out.size(); ++j) {
+ RTC_DCHECK_GT(kMaxNumFrames, k);
+ out[j] = x[k];
+ k += down_sampling_factor_;
+ }
+ } else {
+ std::copy(in.data(), in.data() + in.size(), out.data());
+ }
+
+ data_dumper_->DumpWav("lc_down_sampler_output", out, kSampleRate8kHz, 1);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/down_sampler.h b/modules/audio_processing/agc2/down_sampler.h
new file mode 100644
index 0000000..a609ea8
--- /dev/null
+++ b/modules/audio_processing/agc2/down_sampler.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_DOWN_SAMPLER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_DOWN_SAMPLER_H_
+
+#include "api/array_view.h"
+#include "modules/audio_processing/agc2/biquad_filter.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+
+class DownSampler {
+ public:
+ explicit DownSampler(ApmDataDumper* data_dumper);
+ void Initialize(int sample_rate_hz);
+
+ void DownSample(rtc::ArrayView<const float> in, rtc::ArrayView<float> out);
+
+ private:
+ ApmDataDumper* data_dumper_;
+ int sample_rate_hz_;
+ int down_sampling_factor_;
+ BiQuadFilter low_pass_filter_;
+
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DownSampler);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_DOWN_SAMPLER_H_
diff --git a/modules/audio_processing/agc2/fixed_gain_controller.cc b/modules/audio_processing/agc2/fixed_gain_controller.cc
index a565613..e59ae34 100644
--- a/modules/audio_processing/agc2/fixed_gain_controller.cc
+++ b/modules/audio_processing/agc2/fixed_gain_controller.cc
@@ -54,12 +54,8 @@
gain_curve_applier_.SetSampleRate(sample_rate_hz);
}
-void FixedGainController::EnableLimiter(bool enable_limiter) {
- enable_limiter_ = enable_limiter;
-}
-
void FixedGainController::Process(AudioFrameView<float> signal) {
- // Apply fixed digital gain; interpolate if necessary. One of the
+ // Apply fixed digital gain. One of the
// planned usages of the FGC is to only use the limiter. In that
// case, the gain would be 1.0. Not doing the multiplications speeds
// it up considerably. Hence the check.
@@ -72,16 +68,13 @@
}
}
- // Use the limiter (if configured to).
- if (enable_limiter_) {
- gain_curve_applier_.Process(signal);
+ // Use the limiter.
+ gain_curve_applier_.Process(signal);
- // Dump data for debug.
- const auto channel_view = signal.channel(0);
- apm_data_dumper_->DumpRaw("agc2_fixed_digital_gain_curve_applier",
- channel_view.size(), channel_view.data());
- }
-
+ // Dump data for debug.
+ const auto channel_view = signal.channel(0);
+ apm_data_dumper_->DumpRaw("agc2_fixed_digital_gain_curve_applier",
+ channel_view.size(), channel_view.data());
// Hard-clipping.
for (size_t k = 0; k < signal.num_channels(); ++k) {
rtc::ArrayView<float> channel_view = signal.channel(k);
diff --git a/modules/audio_processing/agc2/fixed_gain_controller.h b/modules/audio_processing/agc2/fixed_gain_controller.h
index fd80348..2b92cfc 100644
--- a/modules/audio_processing/agc2/fixed_gain_controller.h
+++ b/modules/audio_processing/agc2/fixed_gain_controller.h
@@ -27,13 +27,11 @@
// with any other method call).
void SetGain(float gain_to_apply_db);
void SetSampleRate(size_t sample_rate_hz);
- void EnableLimiter(bool enable_limiter);
private:
float gain_to_apply_ = 1.f;
ApmDataDumper* apm_data_dumper_ = nullptr;
GainCurveApplier gain_curve_applier_;
- bool enable_limiter_ = true;
};
} // namespace webrtc
diff --git a/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc b/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc
index 1d6c2ae..d688f6a 100644
--- a/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc
+++ b/modules/audio_processing/agc2/fixed_gain_controller_unittest.cc
@@ -49,34 +49,20 @@
std::unique_ptr<FixedGainController> CreateFixedGainController(
float gain_to_apply,
- size_t rate,
- bool enable_limiter) {
+ size_t rate) {
std::unique_ptr<FixedGainController> fgc =
rtc::MakeUnique<FixedGainController>(&test_data_dumper);
fgc->SetGain(gain_to_apply);
fgc->SetSampleRate(rate);
- fgc->EnableLimiter(enable_limiter);
return fgc;
}
} // namespace
-TEST(AutomaticGainController2FixedDigital, CreateUseWithoutLimiter) {
- const int kSampleRate = 48000;
- std::unique_ptr<FixedGainController> fixed_gc =
- CreateFixedGainController(kGainToApplyDb, kSampleRate, false);
- VectorFloatFrame vectors_with_float_frame(
- 1, rtc::CheckedDivExact(kSampleRate, 100), kInputLevelLinear);
- auto float_frame = vectors_with_float_frame.float_frame_view();
- fixed_gc->Process(float_frame);
- const auto channel = float_frame.channel(0);
- EXPECT_LT(kInputLevelLinear, channel[0]);
-}
-
-TEST(AutomaticGainController2FixedDigital, CreateUseWithLimiter) {
+TEST(AutomaticGainController2FixedDigital, CreateUse) {
const int kSampleRate = 44000;
std::unique_ptr<FixedGainController> fixed_gc =
- CreateFixedGainController(kGainToApplyDb, kSampleRate, true);
+ CreateFixedGainController(kGainToApplyDb, kSampleRate);
VectorFloatFrame vectors_with_float_frame(
1, rtc::CheckedDivExact(kSampleRate, 100), kInputLevelLinear);
auto float_frame = vectors_with_float_frame.float_frame_view();
@@ -96,7 +82,7 @@
// Since |test::kLimiterMaxInputLevelDbFs| > |gain_db|, the
// limiter will not saturate the signal.
std::unique_ptr<FixedGainController> fixed_gc_no_saturation =
- CreateFixedGainController(gain_db, kSampleRate, true);
+ CreateFixedGainController(gain_db, kSampleRate);
// Saturation not expected.
SCOPED_TRACE(std::to_string(gain_db));
@@ -112,7 +98,7 @@
// Since |test::kLimiterMaxInputLevelDbFs| < |gain|, the limiter
// will saturate the signal.
std::unique_ptr<FixedGainController> fixed_gc_saturation =
- CreateFixedGainController(gain_db, kSampleRate, true);
+ CreateFixedGainController(gain_db, kSampleRate);
// Saturation expected.
SCOPED_TRACE(std::to_string(gain_db));
@@ -135,7 +121,7 @@
// Since |gain| > |test::kLimiterMaxInputLevelDbFs|, the limiter will
// not saturate the signal.
std::unique_ptr<FixedGainController> fixed_gc_no_saturation =
- CreateFixedGainController(gain_db, kSampleRate, true);
+ CreateFixedGainController(gain_db, kSampleRate);
// Saturation not expected.
SCOPED_TRACE(std::to_string(gain_db));
@@ -151,7 +137,7 @@
// Singe |gain| < |test::kLimiterMaxInputLevelDbFs|, the limiter will
// saturate the signal.
std::unique_ptr<FixedGainController> fixed_gc_saturation =
- CreateFixedGainController(gain_db, kSampleRate, true);
+ CreateFixedGainController(gain_db, kSampleRate);
// Saturation expected.
SCOPED_TRACE(std::to_string(gain_db));
@@ -170,7 +156,7 @@
constexpr float kGainDbFactor10 = 20.f;
std::unique_ptr<FixedGainController> fixed_gc_no_saturation =
- CreateFixedGainController(kGainDbNoChange, kSampleRate, false);
+ CreateFixedGainController(kGainDbNoChange, kSampleRate);
// Signal level is unchanged with 0 db gain.
EXPECT_FLOAT_EQ(
diff --git a/modules/audio_processing/agc2/gain_applier.cc b/modules/audio_processing/agc2/gain_applier.cc
new file mode 100644
index 0000000..38eb1de
--- /dev/null
+++ b/modules/audio_processing/agc2/gain_applier.cc
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/gain_applier.h"
+
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "rtc_base/numerics/safe_minmax.h"
+
+namespace webrtc {
+namespace {
+
+// Returns true when the gain factor is so close to 1 that it would
+// not affect int16 samples.
+bool GainCloseToOne(float gain_factor) {
+ return 1.f - 1.f / kMaxFloatS16Value <= gain_factor &&
+ gain_factor <= 1.f + 1.f / kMaxFloatS16Value;
+}
+
+void ClipSignal(AudioFrameView<float> signal) {
+ for (size_t k = 0; k < signal.num_channels(); ++k) {
+ rtc::ArrayView<float> channel_view = signal.channel(k);
+ for (auto& sample : channel_view) {
+ sample = rtc::SafeClamp(sample, kMinFloatS16Value, kMaxFloatS16Value);
+ }
+ }
+}
+
+void ApplyGainWithRamping(float last_gain_linear,
+ float gain_at_end_of_frame_linear,
+ float inverse_samples_per_channel,
+ AudioFrameView<float> float_frame) {
+ // Do not modify the signal.
+ if (last_gain_linear == gain_at_end_of_frame_linear &&
+ GainCloseToOne(gain_at_end_of_frame_linear)) {
+ return;
+ }
+
+ // Gain is constant and different from 1.
+ if (last_gain_linear == gain_at_end_of_frame_linear) {
+ for (size_t k = 0; k < float_frame.num_channels(); ++k) {
+ rtc::ArrayView<float> channel_view = float_frame.channel(k);
+ for (auto& sample : channel_view) {
+ sample *= gain_at_end_of_frame_linear;
+ }
+ }
+ return;
+ }
+
+ // The gain changes. We have to change slowly to avoid discontinuities.
+ const float increment = (gain_at_end_of_frame_linear - last_gain_linear) *
+ inverse_samples_per_channel;
+ float gain = last_gain_linear;
+ for (size_t i = 0; i < float_frame.samples_per_channel(); ++i) {
+ for (size_t ch = 0; ch < float_frame.num_channels(); ++ch) {
+ float_frame.channel(ch)[i] *= gain;
+ }
+ gain += increment;
+ }
+}
+
+} // namespace
+
+GainApplier::GainApplier(bool hard_clip_samples, float initial_gain_factor)
+ : hard_clip_samples_(hard_clip_samples),
+ last_gain_factor_(initial_gain_factor),
+ current_gain_factor_(initial_gain_factor) {}
+
+void GainApplier::ApplyGain(AudioFrameView<float> signal) {
+ if (static_cast<int>(signal.samples_per_channel()) != samples_per_channel_) {
+ Initialize(signal.samples_per_channel());
+ }
+
+ ApplyGainWithRamping(last_gain_factor_, current_gain_factor_,
+ inverse_samples_per_channel_, signal);
+
+ last_gain_factor_ = current_gain_factor_;
+
+ if (hard_clip_samples_) {
+ ClipSignal(signal);
+ }
+}
+
+void GainApplier::SetGainFactor(float gain_factor) {
+ RTC_DCHECK_GT(gain_factor, 0.f);
+ current_gain_factor_ = gain_factor;
+}
+
+void GainApplier::Initialize(size_t samples_per_channel) {
+ RTC_DCHECK_GT(samples_per_channel, 0);
+ samples_per_channel_ = static_cast<int>(samples_per_channel);
+ inverse_samples_per_channel_ = 1.f / samples_per_channel_;
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/gain_applier.h b/modules/audio_processing/agc2/gain_applier.h
new file mode 100644
index 0000000..a4f56ce
--- /dev/null
+++ b/modules/audio_processing/agc2/gain_applier.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_GAIN_APPLIER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_GAIN_APPLIER_H_
+
+#include "modules/audio_processing/include/audio_frame_view.h"
+
+namespace webrtc {
+class GainApplier {
+ public:
+ GainApplier(bool hard_clip_samples, float initial_gain_factor);
+
+ void ApplyGain(AudioFrameView<float> signal);
+ void SetGainFactor(float gain_factor);
+
+ private:
+ void Initialize(size_t samples_per_channel);
+
+ // Whether to clip samples after gain is applied. If 'true', result
+ // will fit in FloatS16 range.
+ const bool hard_clip_samples_;
+ float last_gain_factor_;
+
+ // If this value is not equal to 'last_gain_factor', gain will be
+ // ramped from 'last_gain_factor_' to this value during the next
+ // 'ApplyGain'.
+ float current_gain_factor_;
+ int samples_per_channel_ = -1;
+ float inverse_samples_per_channel_ = -1.f;
+};
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_GAIN_APPLIER_H_
diff --git a/modules/audio_processing/agc2/gain_applier_unittest.cc b/modules/audio_processing/agc2/gain_applier_unittest.cc
new file mode 100644
index 0000000..bbf3a85
--- /dev/null
+++ b/modules/audio_processing/agc2/gain_applier_unittest.cc
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/gain_applier.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <limits>
+
+#include "modules/audio_processing/agc2/vector_float_frame.h"
+#include "rtc_base/gunit.h"
+
+namespace webrtc {
+TEST(AutomaticGainController2GainApplier, InitialGainIsRespected) {
+ constexpr float initial_signal_level = 123.f;
+ constexpr float gain_factor = 10.f;
+ VectorFloatFrame fake_audio(1, 1, initial_signal_level);
+ GainApplier gain_applier(true, gain_factor);
+
+ gain_applier.ApplyGain(fake_audio.float_frame_view());
+ EXPECT_NEAR(fake_audio.float_frame_view().channel(0)[0],
+ initial_signal_level * gain_factor, 0.1f);
+}
+
+TEST(AutomaticGainController2GainApplier, ClippingIsDone) {
+ constexpr float initial_signal_level = 30000.f;
+ constexpr float gain_factor = 10.f;
+ VectorFloatFrame fake_audio(1, 1, initial_signal_level);
+ GainApplier gain_applier(true, gain_factor);
+
+ gain_applier.ApplyGain(fake_audio.float_frame_view());
+ EXPECT_NEAR(fake_audio.float_frame_view().channel(0)[0],
+ std::numeric_limits<int16_t>::max(), 0.1f);
+}
+
+TEST(AutomaticGainController2GainApplier, ClippingIsNotDone) {
+ constexpr float initial_signal_level = 30000.f;
+ constexpr float gain_factor = 10.f;
+ VectorFloatFrame fake_audio(1, 1, initial_signal_level);
+ GainApplier gain_applier(false, gain_factor);
+
+ gain_applier.ApplyGain(fake_audio.float_frame_view());
+
+ EXPECT_NEAR(fake_audio.float_frame_view().channel(0)[0],
+ initial_signal_level * gain_factor, 0.1f);
+}
+
+TEST(AutomaticGainController2GainApplier, RampingIsDone) {
+ constexpr float initial_signal_level = 30000.f;
+ constexpr float initial_gain_factor = 1.f;
+ constexpr float target_gain_factor = 0.5f;
+ constexpr int num_channels = 3;
+ constexpr int samples_per_channel = 4;
+ VectorFloatFrame fake_audio(num_channels, samples_per_channel,
+ initial_signal_level);
+ GainApplier gain_applier(false, initial_gain_factor);
+
+ gain_applier.SetGainFactor(target_gain_factor);
+ gain_applier.ApplyGain(fake_audio.float_frame_view());
+
+ // The maximal gain change should be close to that in linear interpolation.
+ for (size_t channel = 0; channel < num_channels; ++channel) {
+ float max_signal_change = 0.f;
+ float last_signal_level = initial_signal_level;
+ for (const auto sample : fake_audio.float_frame_view().channel(channel)) {
+ const float current_change = fabs(last_signal_level - sample);
+ max_signal_change =
+ std::max(max_signal_change, current_change);
+ last_signal_level = sample;
+ }
+ const float total_gain_change =
+ fabs((initial_gain_factor - target_gain_factor) * initial_signal_level);
+ EXPECT_NEAR(max_signal_change, total_gain_change / samples_per_channel,
+ 0.1f);
+ }
+
+ // Next frame should have the desired level.
+ VectorFloatFrame next_fake_audio_frame(num_channels, samples_per_channel,
+ initial_signal_level);
+ gain_applier.ApplyGain(next_fake_audio_frame.float_frame_view());
+
+ // The last sample should have the new gain.
+ EXPECT_NEAR(next_fake_audio_frame.float_frame_view().channel(0)[0],
+ initial_signal_level * target_gain_factor, 0.1f);
+}
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/interpolated_gain_curve.cc b/modules/audio_processing/agc2/interpolated_gain_curve.cc
index 69602b5..7f1f16c 100644
--- a/modules/audio_processing/agc2/interpolated_gain_curve.cc
+++ b/modules/audio_processing/agc2/interpolated_gain_curve.cc
@@ -20,19 +20,33 @@
namespace {
void LogRegionStats(const InterpolatedGainCurve::Stats& stats) {
using Region = InterpolatedGainCurve::GainCurveRegion;
+ const int duration_s =
+ stats.region_duration_frames / (1000 / kFrameDurationMs);
- std::string histogram_name = "WebRTC.Audio.AGC2.FixedDigitalGainCurveRegion.";
- if (stats.region == Region::kIdentity) {
- histogram_name += "Identity";
- } else if (stats.region == Region::kKnee) {
- histogram_name += "Knee";
- } else if (stats.region == Region::kLimiter) {
- histogram_name += "Limiter";
- } else {
- histogram_name += "Saturation";
+ switch (stats.region) {
+ case Region::kIdentity: {
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Audio.Agc2.FixedDigitalGainCurveRegion.Identity", duration_s);
+ break;
+ }
+ case Region::kKnee: {
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Audio.Agc2.FixedDigitalGainCurveRegion.Knee", duration_s);
+ break;
+ }
+ case Region::kLimiter: {
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Audio.Agc2.FixedDigitalGainCurveRegion.Limiter", duration_s);
+ break;
+ }
+ case Region::kSaturation: {
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Audio.Agc2.FixedDigitalGainCurveRegion.Saturation",
+ duration_s);
+ break;
+ }
+ default: { RTC_NOTREACHED(); }
}
- RTC_HISTOGRAM_COUNTS_10000(histogram_name,
- stats.region_duration_frames / 100);
}
} // namespace
diff --git a/modules/audio_processing/agc2/noise_level_estimator.cc b/modules/audio_processing/agc2/noise_level_estimator.cc
new file mode 100644
index 0000000..d9aaf1f
--- /dev/null
+++ b/modules/audio_processing/agc2/noise_level_estimator.cc
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/noise_level_estimator.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <numeric>
+
+#include "common_audio/include/audio_util.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+
+namespace webrtc {
+
+namespace {
+constexpr int kFramesPerSecond = 100;
+
+float FrameEnergy(const AudioFrameView<const float>& audio) {
+ float energy = 0.f;
+ for (size_t k = 0; k < audio.num_channels(); ++k) {
+ float channel_energy =
+ std::accumulate(audio.channel(k).begin(), audio.channel(k).end(), 0.f,
+ [](float a, float b) -> float { return a + b * b; });
+ energy = std::max(channel_energy, energy);
+ }
+ return energy;
+}
+
+float EnergyToDbfs(float signal_energy, size_t num_samples) {
+ const float rms = std::sqrt(signal_energy / num_samples);
+ return FloatS16ToDbfs(rms);
+}
+} // namespace
+
+NoiseLevelEstimator::NoiseLevelEstimator(ApmDataDumper* data_dumper)
+ : signal_classifier_(data_dumper) {
+ Initialize(48000);
+}
+
+NoiseLevelEstimator::~NoiseLevelEstimator() {}
+
+void NoiseLevelEstimator::Initialize(int sample_rate_hz) {
+ sample_rate_hz_ = sample_rate_hz;
+ noise_energy_ = 1.f;
+ first_update_ = true;
+ min_noise_energy_ = sample_rate_hz * 2.f * 2.f / kFramesPerSecond;
+ noise_energy_hold_counter_ = 0;
+ signal_classifier_.Initialize(sample_rate_hz);
+}
+
+float NoiseLevelEstimator::Analyze(const AudioFrameView<const float>& frame) {
+ const int rate =
+ static_cast<int>(frame.samples_per_channel() * kFramesPerSecond);
+ if (rate != sample_rate_hz_) {
+ Initialize(rate);
+ }
+ const float frame_energy = FrameEnergy(frame);
+ if (frame_energy <= 0.f) {
+ RTC_DCHECK_GE(frame_energy, 0.f);
+ return EnergyToDbfs(noise_energy_, frame.samples_per_channel());
+ }
+
+ if (first_update_) {
+ // Initialize the noise energy to the frame energy.
+ first_update_ = false;
+ return EnergyToDbfs(
+ noise_energy_ = std::max(frame_energy, min_noise_energy_),
+ frame.samples_per_channel());
+ }
+
+ const SignalClassifier::SignalType signal_type =
+ signal_classifier_.Analyze(frame.channel(0));
+
+ // Update the noise estimate in a minimum statistics-type manner.
+ if (signal_type == SignalClassifier::SignalType::kStationary) {
+ if (frame_energy > noise_energy_) {
+ // Leak the estimate upwards towards the frame energy if no recent
+ // downward update.
+ noise_energy_hold_counter_ = std::max(noise_energy_hold_counter_ - 1, 0);
+
+ if (noise_energy_hold_counter_ == 0) {
+ noise_energy_ = std::min(noise_energy_ * 1.01f, frame_energy);
+ }
+ } else {
+ // Update smoothly downwards with a limited maximum update magnitude.
+ noise_energy_ =
+ std::max(noise_energy_ * 0.9f,
+ noise_energy_ + 0.05f * (frame_energy - noise_energy_));
+ noise_energy_hold_counter_ = 1000;
+ }
+ } else {
+ // For a non-stationary signal, leak the estimate downwards in order to
+ // avoid estimate locking due to incorrect signal classification.
+ noise_energy_ = noise_energy_ * 0.99f;
+ }
+
+ // Ensure a minimum of the estimate.
+ return EnergyToDbfs(
+ noise_energy_ = std::max(noise_energy_, min_noise_energy_),
+ frame.samples_per_channel());
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/noise_level_estimator.h b/modules/audio_processing/agc2/noise_level_estimator.h
new file mode 100644
index 0000000..24067a1
--- /dev/null
+++ b/modules/audio_processing/agc2/noise_level_estimator.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_NOISE_LEVEL_ESTIMATOR_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_NOISE_LEVEL_ESTIMATOR_H_
+
+#include "modules/audio_processing/agc2/signal_classifier.h"
+#include "modules/audio_processing/include/audio_frame_view.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+class ApmDataDumper;
+
+class NoiseLevelEstimator {
+ public:
+ NoiseLevelEstimator(ApmDataDumper* data_dumper);
+ ~NoiseLevelEstimator();
+ // Returns the estimated noise level in dBFS.
+ float Analyze(const AudioFrameView<const float>& frame);
+
+ private:
+ void Initialize(int sample_rate_hz);
+
+ int sample_rate_hz_;
+ float min_noise_energy_;
+ bool first_update_;
+ float noise_energy_;
+ int noise_energy_hold_counter_;
+ SignalClassifier signal_classifier_;
+
+ RTC_DISALLOW_COPY_AND_ASSIGN(NoiseLevelEstimator);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_NOISE_LEVEL_ESTIMATOR_H_
diff --git a/modules/audio_processing/agc2/noise_level_estimator_unittest.cc b/modules/audio_processing/agc2/noise_level_estimator_unittest.cc
new file mode 100644
index 0000000..c4fd33b
--- /dev/null
+++ b/modules/audio_processing/agc2/noise_level_estimator_unittest.cc
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/noise_level_estimator.h"
+
+#include <array>
+#include <functional>
+#include <limits>
+
+#include "modules/audio_processing/agc2/agc2_testing_common.h"
+#include "modules/audio_processing/agc2/vector_float_frame.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/random.h"
+
+namespace webrtc {
+namespace {
+Random rand_gen(42);
+ApmDataDumper data_dumper(0);
+constexpr int kNumIterations = 200;
+constexpr int kFramesPerSecond = 100;
+
+// Runs the noise estimator on audio generated by 'sample_generator'
+// for kNumIterations. Returns the last noise level estimate.
+float RunEstimator(std::function<float()> sample_generator, int rate) {
+ NoiseLevelEstimator estimator(&data_dumper);
+ const size_t samples_per_channel =
+ rtc::CheckedDivExact(rate, kFramesPerSecond);
+ VectorFloatFrame signal(1, static_cast<int>(samples_per_channel), 0.f);
+
+ for (int i = 0; i < kNumIterations; ++i) {
+ AudioFrameView<float> frame_view = signal.float_frame_view();
+ for (size_t j = 0; j < samples_per_channel; ++j) {
+ frame_view.channel(0)[j] = sample_generator();
+ }
+ estimator.Analyze(frame_view);
+ }
+ return estimator.Analyze(signal.float_frame_view());
+}
+
+float WhiteNoiseGenerator() {
+ return static_cast<float>(rand_gen.Rand(std::numeric_limits<int16_t>::min(),
+ std::numeric_limits<int16_t>::max()));
+}
+} // namespace
+
+// White random noise is stationary, but does not trigger the detector
+// every frame due to the randomness.
+TEST(AutomaticGainController2NoiseEstimator, RandomNoise) {
+ for (const auto rate : {8000, 16000, 32000, 48000}) {
+ const float noise_level = RunEstimator(WhiteNoiseGenerator, rate);
+ EXPECT_NEAR(noise_level, -5.f, 1.f);
+ }
+}
+
+// Sine curves are (very) stationary. They trigger the detector all
+// the time. Except for a few initial frames.
+TEST(AutomaticGainController2NoiseEstimator, SineTone) {
+ for (const auto rate : {8000, 16000, 32000, 48000}) {
+ test::SineGenerator gen(600.f, rate);
+ const float noise_level = RunEstimator(gen, rate);
+ EXPECT_NEAR(noise_level, -33.f, 1.f);
+ }
+}
+
+// Pulses are transient if they are far enough apart. They shouldn't
+// trigger the noise detector.
+TEST(AutomaticGainController2NoiseEstimator, PulseTone) {
+ for (const auto rate : {8000, 16000, 32000, 48000}) {
+ test::PulseGenerator gen(20.f, rate);
+ const int noise_level = RunEstimator(gen, rate);
+ EXPECT_NEAR(noise_level, -79.f, 1.f);
+ }
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/noise_spectrum_estimator.cc b/modules/audio_processing/agc2/noise_spectrum_estimator.cc
new file mode 100644
index 0000000..9e08126
--- /dev/null
+++ b/modules/audio_processing/agc2/noise_spectrum_estimator.cc
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/noise_spectrum_estimator.h"
+
+#include <string.h>
+#include <algorithm>
+
+#include "api/array_view.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/arraysize.h"
+
+namespace webrtc {
+namespace {
+constexpr float kMinNoisePower = 100.f;
+} // namespace
+
+NoiseSpectrumEstimator::NoiseSpectrumEstimator(ApmDataDumper* data_dumper)
+ : data_dumper_(data_dumper) {
+ Initialize();
+}
+
+void NoiseSpectrumEstimator::Initialize() {
+ std::fill(noise_spectrum_, noise_spectrum_ + arraysize(noise_spectrum_),
+ kMinNoisePower);
+}
+
+void NoiseSpectrumEstimator::Update(rtc::ArrayView<const float> spectrum,
+ bool first_update) {
+ RTC_DCHECK_EQ(65, spectrum.size());
+
+ if (first_update) {
+ // Initialize the noise spectral estimate with the signal spectrum.
+ std::copy(spectrum.data(), spectrum.data() + spectrum.size(),
+ noise_spectrum_);
+ } else {
+ // Smoothly update the noise spectral estimate towards the signal spectrum
+ // such that the magnitude of the updates are limited.
+ for (size_t k = 0; k < spectrum.size(); ++k) {
+ if (noise_spectrum_[k] < spectrum[k]) {
+ noise_spectrum_[k] = std::min(
+ 1.01f * noise_spectrum_[k],
+ noise_spectrum_[k] + 0.05f * (spectrum[k] - noise_spectrum_[k]));
+ } else {
+ noise_spectrum_[k] = std::max(
+ 0.99f * noise_spectrum_[k],
+ noise_spectrum_[k] + 0.05f * (spectrum[k] - noise_spectrum_[k]));
+ }
+ }
+ }
+
+ // Ensure that the noise spectal estimate does not become too low.
+ for (auto& v : noise_spectrum_) {
+ v = std::max(v, kMinNoisePower);
+ }
+
+ data_dumper_->DumpRaw("lc_noise_spectrum", 65, noise_spectrum_);
+ data_dumper_->DumpRaw("lc_signal_spectrum", spectrum);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/noise_spectrum_estimator.h b/modules/audio_processing/agc2/noise_spectrum_estimator.h
new file mode 100644
index 0000000..fd1cc13
--- /dev/null
+++ b/modules/audio_processing/agc2/noise_spectrum_estimator.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_NOISE_SPECTRUM_ESTIMATOR_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_NOISE_SPECTRUM_ESTIMATOR_H_
+
+#include "api/array_view.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+
+class NoiseSpectrumEstimator {
+ public:
+ explicit NoiseSpectrumEstimator(ApmDataDumper* data_dumper);
+ void Initialize();
+ void Update(rtc::ArrayView<const float> spectrum, bool first_update);
+
+ rtc::ArrayView<const float> GetNoiseSpectrum() const {
+ return rtc::ArrayView<const float>(noise_spectrum_);
+ }
+
+ private:
+ ApmDataDumper* data_dumper_;
+ float noise_spectrum_[65];
+
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(NoiseSpectrumEstimator);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_NOISE_SPECTRUM_ESTIMATOR_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/BUILD.gn b/modules/audio_processing/agc2/rnn_vad/BUILD.gn
new file mode 100644
index 0000000..e05dcab
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/BUILD.gn
@@ -0,0 +1,103 @@
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../../webrtc.gni")
+
+group("rnn_vad") {
+ deps = [
+ ":lib",
+ ]
+}
+
+source_set("lib") {
+ sources = [
+ "common.h",
+ "lp_residual.cc",
+ "lp_residual.h",
+ "pitch_info.h",
+ "pitch_search.cc",
+ "pitch_search.h",
+ "pitch_search_internal.cc",
+ "pitch_search_internal.h",
+ "ring_buffer.h",
+ "sequence_buffer.h",
+ "symmetric_matrix_buffer.h",
+ ]
+ deps = [
+ "../../../../api:array_view",
+ "../../../../rtc_base:checks",
+ ]
+}
+
+if (rtc_include_tests) {
+ source_set("lib_test") {
+ testonly = true
+ sources = [
+ "test_utils.cc",
+ "test_utils.h",
+ ]
+ deps = [
+ "../../../../api:array_view",
+ "../../../../rtc_base:checks",
+ "../../../../rtc_base:ptr_util",
+ "../../../../test:fileutils",
+ "../../../../test:test_support",
+ ]
+ }
+
+ unittest_resources = [
+ "../../../../resources/audio_processing/agc2/rnn_vad/pitch_buf_24k.dat",
+ "../../../../resources/audio_processing/agc2/rnn_vad/pitch_lp_res.dat",
+ ]
+
+ if (is_ios) {
+ bundle_data("unittests_bundle_data") {
+ testonly = true
+ sources = unittest_resources
+ outputs = [
+ "{{bundle_resources_dir}}/{{source_file_part}}",
+ ]
+ }
+ }
+
+ rtc_source_set("unittests") {
+ testonly = true
+ sources = [
+ "lp_residual_unittest.cc",
+ "pitch_search_internal_unittest.cc",
+ "pitch_search_unittest.cc",
+ "ring_buffer_unittest.cc",
+ "sequence_buffer_unittest.cc",
+ "symmetric_matrix_buffer_unittest.cc",
+ ]
+ deps = [
+ ":lib",
+ ":lib_test",
+ "../../../../api:array_view",
+ "../../../../test:test_support",
+ ]
+ data = unittest_resources
+ if (is_ios) {
+ deps += [ ":unittests_bundle_data" ]
+ }
+ }
+
+ rtc_executable("rnn_vad_tool") {
+ testonly = true
+ sources = [
+ "rnn_vad_tool.cc",
+ ]
+ deps = [
+ ":lib",
+ "../../../../api:array_view",
+ "../../../../common_audio:common_audio",
+ "../../../../rtc_base:rtc_base_approved",
+ "../../../../test:test_support",
+ ]
+ }
+}
diff --git a/modules/audio_processing/agc2/rnn_vad/common.h b/modules/audio_processing/agc2/rnn_vad/common.h
new file mode 100644
index 0000000..252bf84
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/common.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_COMMON_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_COMMON_H_
+
+namespace webrtc {
+namespace rnn_vad {
+
+constexpr size_t kSampleRate24kHz = 24000;
+constexpr size_t kFrameSize10ms24kHz = kSampleRate24kHz / 100;
+constexpr size_t kFrameSize20ms24kHz = kFrameSize10ms24kHz * 2;
+
+// Pitch analysis params.
+constexpr size_t kMinPitch24kHz = kSampleRate24kHz / 800; // 0.00125 s.
+constexpr size_t kMaxPitch24kHz = kSampleRate24kHz / 62.5; // 0.016 s.
+constexpr size_t kBufSize24kHz = kMaxPitch24kHz + kFrameSize20ms24kHz;
+static_assert((kBufSize24kHz & 1) == 0, "The buffer size must be even.");
+
+// Define a higher minimum pitch period for the initial search. This is used to
+// avoid searching for very short periods, for which a refinement step is
+// responsible.
+constexpr size_t kInitialMinPitch24kHz = 3 * kMinPitch24kHz;
+static_assert(kMinPitch24kHz < kInitialMinPitch24kHz, "");
+static_assert(kInitialMinPitch24kHz < kMaxPitch24kHz, "");
+
+// 12 kHz analysis.
+constexpr size_t kSampleRate12kHz = 12000;
+constexpr size_t kFrameSize10ms12kHz = kSampleRate12kHz / 100;
+constexpr size_t kFrameSize20ms12kHz = kFrameSize10ms12kHz * 2;
+constexpr size_t kBufSize12kHz = kBufSize24kHz / 2;
+constexpr size_t kInitialMinPitch12kHz = kInitialMinPitch24kHz / 2;
+constexpr size_t kMaxPitch12kHz = kMaxPitch24kHz / 2;
+
+// 48 kHz constants.
+constexpr size_t kMinPitch48kHz = kMinPitch24kHz * 2;
+constexpr size_t kMaxPitch48kHz = kMaxPitch24kHz * 2;
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_COMMON_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/lp_residual.cc b/modules/audio_processing/agc2/rnn_vad/lp_residual.cc
new file mode 100644
index 0000000..483336d
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/lp_residual.cc
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/lp_residual.h"
+
+#include <algorithm>
+#include <array>
+#include <numeric>
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace {
+
+// Computes cross-correlation coefficients between |x| and |y| and writes them
+// in |x_corr|. The lag values are in {0, ..., max_lag - 1}, where max_lag
+// equals the size of |x_corr|.
+// The |x| and |y| sub-arrays used to compute a cross-correlation coefficients
+// for a lag l have both size "size of |x| - l" - i.e., the longest sub-array is
+// used. |x| and |y| must have the same size.
+void ComputeCrossCorrelation(
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<const float> y,
+ rtc::ArrayView<float, kNumLpcCoefficients> x_corr) {
+ constexpr size_t max_lag = x_corr.size();
+ RTC_DCHECK_EQ(x.size(), y.size());
+ RTC_DCHECK_LT(max_lag, x.size());
+ for (size_t lag = 0; lag < max_lag; ++lag)
+ x_corr[lag] =
+ std::inner_product(x.begin(), x.end() - lag, y.begin() + lag, 0.f);
+}
+
+// Applies denoising to the auto-correlation coefficients.
+void DenoiseAutoCorrelation(
+ rtc::ArrayView<float, kNumLpcCoefficients> auto_corr) {
+ // Assume -40 dB white noise floor.
+ auto_corr[0] *= 1.0001f;
+ for (size_t i = 1; i < kNumLpcCoefficients; ++i)
+ auto_corr[i] -= auto_corr[i] * (0.008f * i) * (0.008f * i);
+}
+
+// Computes the initial inverse filter coefficients given the auto-correlation
+// coefficients of an input frame.
+void ComputeInitialInverseFilterCoefficients(
+ rtc::ArrayView<const float, kNumLpcCoefficients> auto_corr,
+ rtc::ArrayView<float, kNumLpcCoefficients - 1> lpc_coeffs) {
+ float error = auto_corr[0];
+ for (size_t i = 0; i < kNumLpcCoefficients - 1; ++i) {
+ float reflection_coeff = 0.f;
+ for (size_t j = 0; j < i; ++j)
+ reflection_coeff += lpc_coeffs[j] * auto_corr[i - j];
+ reflection_coeff += auto_corr[i + 1];
+ reflection_coeff /= -error;
+ // Update LPC coefficients and total error.
+ lpc_coeffs[i] = reflection_coeff;
+ for (size_t j = 0; j<(i + 1)>> 1; ++j) {
+ const float tmp1 = lpc_coeffs[j];
+ const float tmp2 = lpc_coeffs[i - 1 - j];
+ lpc_coeffs[j] = tmp1 + reflection_coeff * tmp2;
+ lpc_coeffs[i - 1 - j] = tmp2 + reflection_coeff * tmp1;
+ }
+ error -= reflection_coeff * reflection_coeff * error;
+ if (error < 0.001f * auto_corr[0])
+ break;
+ }
+}
+
+} // namespace
+
+void ComputeAndPostProcessLpcCoefficients(
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<float, kNumLpcCoefficients> lpc_coeffs) {
+ std::array<float, kNumLpcCoefficients> auto_corr;
+ ComputeCrossCorrelation(x, x, {auto_corr.data(), auto_corr.size()});
+ if (auto_corr[0] == 0.f) { // Empty frame.
+ std::fill(lpc_coeffs.begin(), lpc_coeffs.end(), 0);
+ return;
+ }
+ DenoiseAutoCorrelation({auto_corr.data(), auto_corr.size()});
+ std::array<float, kNumLpcCoefficients - 1> lpc_coeffs_pre{};
+ ComputeInitialInverseFilterCoefficients(
+ {auto_corr.data(), auto_corr.size()},
+ {lpc_coeffs_pre.data(), lpc_coeffs_pre.size()});
+ // LPC coefficients post-processing.
+ // TODO(bugs.webrtc.org/9076): Consider removing these steps.
+ float c1 = 1.f;
+ for (size_t i = 0; i < kNumLpcCoefficients - 1; ++i) {
+ c1 *= 0.9f;
+ lpc_coeffs_pre[i] *= c1;
+ }
+ const float c2 = 0.8f;
+ lpc_coeffs[0] = lpc_coeffs_pre[0] + c2;
+ lpc_coeffs[1] = lpc_coeffs_pre[1] + c2 * lpc_coeffs_pre[0];
+ lpc_coeffs[2] = lpc_coeffs_pre[2] + c2 * lpc_coeffs_pre[1];
+ lpc_coeffs[3] = lpc_coeffs_pre[3] + c2 * lpc_coeffs_pre[2];
+ lpc_coeffs[4] = c2 * lpc_coeffs_pre[3];
+}
+
+void ComputeLpResidual(
+ rtc::ArrayView<const float, kNumLpcCoefficients> lpc_coeffs,
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<float> y) {
+ RTC_DCHECK_LT(kNumLpcCoefficients, x.size());
+ RTC_DCHECK_EQ(x.size(), y.size());
+ std::array<float, kNumLpcCoefficients> input_chunk;
+ input_chunk.fill(0.f);
+ for (size_t i = 0; i < y.size(); ++i) {
+ const float sum = std::inner_product(input_chunk.begin(), input_chunk.end(),
+ lpc_coeffs.begin(), x[i]);
+ // Circular shift and add a new sample.
+ for (size_t j = kNumLpcCoefficients - 1; j > 0; --j)
+ input_chunk[j] = input_chunk[j - 1];
+ input_chunk[0] = x[i];
+ // Copy result.
+ y[i] = sum;
+ }
+}
+
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/lp_residual.h b/modules/audio_processing/agc2/rnn_vad/lp_residual.h
new file mode 100644
index 0000000..bffafd2
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/lp_residual.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_LP_RESIDUAL_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_LP_RESIDUAL_H_
+
+#include "api/array_view.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+// LPC inverse filter length.
+constexpr size_t kNumLpcCoefficients = 5;
+
+// Given a frame |x|, computes a post-processed version of LPC coefficients
+// tailored for pitch estimation.
+void ComputeAndPostProcessLpcCoefficients(
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<float, kNumLpcCoefficients> lpc_coeffs);
+
+// Computes the LP residual for the input frame |x| and the LPC coefficients
+// |lpc_coeffs|. |y| and |x| can point to the same array for in-place
+// computation.
+void ComputeLpResidual(
+ rtc::ArrayView<const float, kNumLpcCoefficients> lpc_coeffs,
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<float> y);
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_LP_RESIDUAL_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/lp_residual_unittest.cc b/modules/audio_processing/agc2/rnn_vad/lp_residual_unittest.cc
new file mode 100644
index 0000000..23f1e14
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/lp_residual_unittest.cc
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/lp_residual.h"
+
+#include <array>
+
+#include "modules/audio_processing/agc2/rnn_vad/common.h"
+#include "modules/audio_processing/agc2/rnn_vad/test_utils.h"
+// TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+// #include "test/fpe_observer.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+
+TEST(RnnVadTest, LpResidualOfEmptyFrame) {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ // Input frame (empty, i.e., all samples set to 0).
+ std::array<float, kFrameSize10ms24kHz> empty_frame;
+ empty_frame.fill(0.f);
+ // Compute inverse filter coefficients.
+ std::array<float, kNumLpcCoefficients> lpc_coeffs;
+ ComputeAndPostProcessLpcCoefficients({empty_frame},
+ {lpc_coeffs.data(), lpc_coeffs.size()});
+ // Compute LP residual.
+ std::array<float, kFrameSize10ms24kHz> lp_residual;
+ ComputeLpResidual({lpc_coeffs.data(), lpc_coeffs.size()}, {empty_frame},
+ {lp_residual});
+}
+
+// TODO(bugs.webrtc.org/9076): Remove when the issue is fixed.
+TEST(RnnVadTest, LpResidualPipelineBitExactness) {
+ // Pitch buffer 24 kHz data reader.
+ auto pitch_buf_24kHz_reader = CreatePitchBuffer24kHzReader();
+ const size_t num_frames = pitch_buf_24kHz_reader.second;
+ std::array<float, kBufSize24kHz> pitch_buf_data;
+ rtc::ArrayView<float, kBufSize24kHz> pitch_buf_data_view(
+ pitch_buf_data.data(), pitch_buf_data.size());
+ // Read ground-truth.
+ auto lp_residual_reader = CreateLpResidualAndPitchPeriodGainReader();
+ ASSERT_EQ(num_frames, lp_residual_reader.second);
+ std::array<float, kBufSize24kHz> expected_lp_residual;
+ rtc::ArrayView<float, kBufSize24kHz> expected_lp_residual_view(
+ expected_lp_residual.data(), expected_lp_residual.size());
+ // Init pipeline.
+ std::array<float, kNumLpcCoefficients> lpc_coeffs;
+ rtc::ArrayView<float, kNumLpcCoefficients> lpc_coeffs_view(
+ lpc_coeffs.data(), kNumLpcCoefficients);
+ std::array<float, kBufSize24kHz> computed_lp_residual;
+ rtc::ArrayView<float, kBufSize24kHz> computed_lp_residual_view(
+ computed_lp_residual.data(), computed_lp_residual.size());
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ for (size_t i = 0; i < num_frames; ++i) {
+ SCOPED_TRACE(i);
+ // Read input and expected output.
+ pitch_buf_24kHz_reader.first->ReadChunk(pitch_buf_data_view);
+ lp_residual_reader.first->ReadChunk(expected_lp_residual_view);
+ // Skip pitch gain and period.
+ float unused;
+ lp_residual_reader.first->ReadValue(&unused);
+ lp_residual_reader.first->ReadValue(&unused);
+ // Run pipeline.
+ ComputeAndPostProcessLpcCoefficients(pitch_buf_data_view,
+ lpc_coeffs_view);
+ ComputeLpResidual(lpc_coeffs_view, pitch_buf_data_view,
+ computed_lp_residual_view);
+ // Compare.
+ ExpectNearAbsolute(expected_lp_residual_view, computed_lp_residual_view,
+ kFloatMin);
+ }
+ }
+}
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_info.h b/modules/audio_processing/agc2/rnn_vad/pitch_info.h
new file mode 100644
index 0000000..f0998d1
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_info.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_INFO_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_INFO_H_
+
+namespace webrtc {
+namespace rnn_vad {
+
+// Stores pitch period and gain information. The pitch gain measures the
+// strength of the pitch (the higher, the stronger).
+struct PitchInfo {
+ PitchInfo() : period(0), gain(0.f) {}
+ PitchInfo(size_t p, float g) : period(p), gain(g) {}
+ size_t period;
+ float gain;
+};
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_INFO_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_search.cc b/modules/audio_processing/agc2/rnn_vad/pitch_search.cc
new file mode 100644
index 0000000..4d83588
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_search.cc
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/pitch_search.h"
+#include "modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+PitchInfo PitchSearch(rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ PitchInfo prev_pitch_48kHz) {
+ // Perform the initial pitch search at 12 kHz.
+ std::array<float, kBufSize12kHz> pitch_buf_decimated;
+ Decimate2x(pitch_buf,
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()});
+ // Compute auto-correlation terms.
+ std::array<float, kNumInvertedLags12kHz> auto_corr;
+ ComputePitchAutoCorrelation(
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()}, kMaxPitch12kHz,
+ {auto_corr.data(), auto_corr.size()});
+ // Search for pitch at 12 kHz.
+ std::array<size_t, 2> pitch_candidates_inv_lags = FindBestPitchPeriods(
+ {auto_corr.data(), auto_corr.size()},
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()}, kMaxPitch12kHz);
+
+ // Refine the pitch period estimation.
+ // The refinement is done using the pitch buffer that contains 24 kHz samples.
+ // Therefore, adapt the inverted lags in |pitch_candidates_inv_lags| from 12
+ // to 24 kHz.
+ for (size_t i = 0; i < pitch_candidates_inv_lags.size(); ++i)
+ pitch_candidates_inv_lags[i] *= 2;
+ size_t pitch_inv_lag_48kHz = RefinePitchPeriod48kHz(
+ pitch_buf,
+ {pitch_candidates_inv_lags.data(), pitch_candidates_inv_lags.size()});
+ // Look for stronger harmonics to find the final pitch period and its gain.
+ RTC_DCHECK_LT(pitch_inv_lag_48kHz, kMaxPitch48kHz);
+ return CheckLowerPitchPeriodsAndComputePitchGain(
+ pitch_buf, kMaxPitch48kHz - pitch_inv_lag_48kHz, prev_pitch_48kHz);
+}
+
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_search.h b/modules/audio_processing/agc2/rnn_vad/pitch_search.h
new file mode 100644
index 0000000..a0af0eb
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_search.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_SEARCH_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_SEARCH_H_
+
+#include "api/array_view.h"
+#include "modules/audio_processing/agc2/rnn_vad/common.h"
+#include "modules/audio_processing/agc2/rnn_vad/pitch_info.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+// Searches the pitch period and gain. Return the pitch estimation data for
+// 48 kHz.
+PitchInfo PitchSearch(rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ PitchInfo prev_pitch_48kHz);
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_SEARCH_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc b/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc
new file mode 100644
index 0000000..1ff4621
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.cc
@@ -0,0 +1,407 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h"
+
+#include <algorithm>
+#include <cmath>
+#include <numeric>
+#include <utility>
+
+#include "modules/audio_processing/agc2/rnn_vad/common.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace {
+
+// Converts a lag to an inverted lag (only for 24kHz).
+size_t GetInvertedLag(size_t lag) {
+ RTC_DCHECK_LE(lag, kMaxPitch24kHz);
+ return kMaxPitch24kHz - lag;
+}
+
+float ComputeAutoCorrelationCoeff(rtc::ArrayView<const float> pitch_buf,
+ size_t inv_lag,
+ size_t max_pitch_period) {
+ RTC_DCHECK_LT(inv_lag, pitch_buf.size());
+ RTC_DCHECK_LT(max_pitch_period, pitch_buf.size());
+ RTC_DCHECK_LE(inv_lag, max_pitch_period);
+ // TODO(bugs.webrtc.org/9076): Maybe optimize using vectorization.
+ return std::inner_product(pitch_buf.begin() + max_pitch_period,
+ pitch_buf.end(), pitch_buf.begin() + inv_lag, 0.f);
+}
+
+// Computes a pseudo-interpolation offset for an estimated pitch period |lag| by
+// looking at the auto-correlation coefficients in the neighborhood of |lag|.
+// (namely, |prev_auto_corr|, |lag_auto_corr| and |next_auto_corr|). The output
+// is a lag in {-1, 0, +1}.
+// TODO(bugs.webrtc.org/9076): Consider removing pseudo-i since it
+// is relevant only if the spectral analysis works at a sample rate that is
+// twice as that of the pitch buffer (not so important instead for the estimated
+// pitch period feature fed into the RNN).
+int GetPitchPseudoInterpolationOffset(size_t lag,
+ float prev_auto_corr,
+ float lag_auto_corr,
+ float next_auto_corr) {
+ const float& a = prev_auto_corr;
+ const float& b = lag_auto_corr;
+ const float& c = next_auto_corr;
+
+ int offset = 0;
+ if ((c - a) > 0.7f * (b - a)) {
+ offset = 1; // |c| is the largest auto-correlation coefficient.
+ } else if ((a - c) > 0.7f * (b - c)) {
+ offset = -1; // |a| is the largest auto-correlation coefficient.
+ }
+ return offset;
+}
+
+// Refines a pitch period |lag| encoded as lag with pseudo-interpolation. The
+// output sample rate is twice as that of |lag|.
+size_t PitchPseudoInterpolationLagPitchBuf(
+ size_t lag,
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf) {
+ int offset = 0;
+ // Cannot apply pseudo-interpolation at the boundaries.
+ if (lag > 0 && lag < kMaxPitch24kHz) {
+ offset = GetPitchPseudoInterpolationOffset(
+ lag,
+ ComputeAutoCorrelationCoeff(pitch_buf, GetInvertedLag(lag - 1),
+ kMaxPitch24kHz),
+ ComputeAutoCorrelationCoeff(pitch_buf, GetInvertedLag(lag),
+ kMaxPitch24kHz),
+ ComputeAutoCorrelationCoeff(pitch_buf, GetInvertedLag(lag + 1),
+ kMaxPitch24kHz));
+ }
+ return 2 * lag + offset;
+}
+
+// Refines a pitch period |inv_lag| encoded as inverted lag with
+// pseudo-interpolation. The output sample rate is twice as that of
+// |inv_lag|.
+size_t PitchPseudoInterpolationInvLagAutoCorr(
+ size_t inv_lag,
+ rtc::ArrayView<const float> auto_corr) {
+ int offset = 0;
+ // Cannot apply pseudo-interpolation at the boundaries.
+ if (inv_lag > 0 && inv_lag < auto_corr.size() - 1) {
+ offset = GetPitchPseudoInterpolationOffset(inv_lag, auto_corr[inv_lag + 1],
+ auto_corr[inv_lag],
+ auto_corr[inv_lag - 1]);
+ }
+ // TODO(bugs.webrtc.org/9076): When retraining, check if |offset| below should
+ // be subtracted since |inv_lag| is an inverted lag but offset is a lag.
+ return 2 * inv_lag + offset;
+}
+
+// Integer multipliers used in CheckLowerPitchPeriodsAndComputePitchGain() when
+// looking for sub-harmonics.
+// The values have been chosen to serve the following algorithm. Given the
+// initial pitch period T, we examine whether one of its harmonics is the true
+// fundamental frequency. We consider T/k with k in {2, ..., 15}. For each of
+// these harmonics, in addition to the pitch gain of itself, we choose one
+// multiple of its pitch period, n*T/k, to validate it (by averaging their pitch
+// gains). The multiplier n is chosen so that n*T/k is used only one time over
+// all k. When for example k = 4, we should also expect a peak at 3*T/4. When
+// k = 8 instead we don't want to look at 2*T/8, since we have already checked
+// T/4 before. Instead, we look at T*3/8.
+// The array can be generate in Python as follows:
+// from fractions import Fraction
+// # Smallest positive integer not in X.
+// def mex(X):
+// for i in range(1, int(max(X)+2)):
+// if i not in X:
+// return i
+// # Visited multiples of the period.
+// S = {1}
+// for n in range(2, 16):
+// sn = mex({n * i for i in S} | {1})
+// S = S | {Fraction(1, n), Fraction(sn, n)}
+// print(sn, end=', ')
+constexpr std::array<size_t, 14> kSubHarmonicMultipliers = {
+ {3, 2, 3, 2, 5, 2, 3, 2, 3, 2, 5, 2, 3, 2}};
+
+// Initial pitch period candidate thresholds for ComputePitchGainThreshold() for
+// a sample rate of 24 kHz. Computed as [5*k*k for k in range(16)].
+constexpr std::array<size_t, 14> kInitialPitchPeriodThresholds = {
+ {20, 45, 80, 125, 180, 245, 320, 405, 500, 605, 720, 845, 980, 1125}};
+
+} // namespace
+
+void Decimate2x(rtc::ArrayView<const float, kBufSize24kHz> src,
+ rtc::ArrayView<float, kBufSize12kHz> dst) {
+ // TODO(bugs.webrtc.org/9076): Consider adding anti-aliasing filter.
+ static_assert(2 * dst.size() == src.size(), "");
+ for (size_t i = 0; i < dst.size(); ++i)
+ dst[i] = src[2 * i];
+}
+
+float ComputePitchGainThreshold(size_t candidate_pitch_period,
+ size_t pitch_period_ratio,
+ size_t initial_pitch_period,
+ float initial_pitch_gain,
+ size_t prev_pitch_period,
+ size_t prev_pitch_gain) {
+ // Map arguments to more compact aliases.
+ const size_t& t1 = candidate_pitch_period;
+ const size_t& k = pitch_period_ratio;
+ const size_t& t0 = initial_pitch_period;
+ const float& g0 = initial_pitch_gain;
+ const size_t& t_prev = prev_pitch_period;
+ const size_t& g_prev = prev_pitch_gain;
+
+ // Validate input.
+ RTC_DCHECK_GE(k, 2);
+
+ // Compute a term that lowers the threshold when |t1| is close to the last
+ // estimated period |t_prev| - i.e., pitch tracking.
+ float lower_threshold_term = 0;
+ if (abs(static_cast<int>(t1) - static_cast<int>(t_prev)) <= 1) {
+ // The candidate pitch period is within 1 sample from the previous one.
+ // Make the candidate at |t1| very easy to be accepted.
+ lower_threshold_term = g_prev;
+ } else if (abs(static_cast<int>(t1) - static_cast<int>(t_prev)) == 2 &&
+ t0 > kInitialPitchPeriodThresholds[k - 2]) {
+ // The candidate pitch period is 2 samples far from the previous one and the
+ // period |t0| (from which |t1| has been derived) is greater than a
+ // threshold. Make |t1| easy to be accepted.
+ lower_threshold_term = 0.5f * g_prev;
+ }
+ // Set the threshold based on the gain of the initial estimate |t0|. Also
+ // reduce the chance of false positives caused by a bias towards high
+ // frequencies (originating from short-term correlations).
+ float threshold = std::max(0.3f, 0.7f * g0 - lower_threshold_term);
+ if (t1 < 3 * kMinPitch24kHz) { // High frequency.
+ threshold = std::max(0.4f, 0.85f * g0 - lower_threshold_term);
+ } else if (t1 < 2 * kMinPitch24kHz) { // Even higher frequency.
+ threshold = std::max(0.5f, 0.9f * g0 - lower_threshold_term);
+ }
+ return threshold;
+}
+
+void ComputeSlidingFrameSquareEnergies(
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ rtc::ArrayView<float, kMaxPitch24kHz + 1> yy_values) {
+ float yy =
+ ComputeAutoCorrelationCoeff(pitch_buf, kMaxPitch24kHz, kMaxPitch24kHz);
+ yy_values[0] = yy;
+ for (size_t i = 1; i < yy_values.size(); ++i) {
+ RTC_DCHECK_LE(i, kMaxPitch24kHz + kFrameSize20ms24kHz);
+ RTC_DCHECK_LE(i, kMaxPitch24kHz);
+ const float old_coeff = pitch_buf[kMaxPitch24kHz + kFrameSize20ms24kHz - i];
+ const float new_coeff = pitch_buf[kMaxPitch24kHz - i];
+ yy -= old_coeff * old_coeff;
+ yy += new_coeff * new_coeff;
+ yy = std::max(0.f, yy);
+ yy_values[i] = yy;
+ }
+}
+
+// TODO(bugs.webrtc.org/9076): Optimize using FFT and/or vectorization.
+void ComputePitchAutoCorrelation(
+ rtc::ArrayView<const float, kBufSize12kHz> pitch_buf,
+ size_t max_pitch_period,
+ rtc::ArrayView<float, kNumInvertedLags12kHz> auto_corr) {
+ RTC_DCHECK_GT(max_pitch_period, auto_corr.size());
+ RTC_DCHECK_LT(max_pitch_period, pitch_buf.size());
+ // Compute auto-correlation coefficients.
+ for (size_t inv_lag = 0; inv_lag < auto_corr.size(); ++inv_lag) {
+ auto_corr[inv_lag] =
+ ComputeAutoCorrelationCoeff(pitch_buf, inv_lag, max_pitch_period);
+ }
+}
+
+std::array<size_t, 2> FindBestPitchPeriods(
+ rtc::ArrayView<const float> auto_corr,
+ rtc::ArrayView<const float> pitch_buf,
+ size_t max_pitch_period) {
+ // Stores a pitch candidate period and strength information.
+ struct PitchCandidate {
+ // Pitch period encoded as inverted lag.
+ size_t period_inverted_lag = 0;
+ // Pitch strength encoded as a ratio.
+ float strength_numerator = -1.f;
+ float strength_denominator = 0.f;
+ // Compare the strength of two pitch candidates.
+ bool HasStrongerPitchThan(const PitchCandidate& b) const {
+ // Comparing the numerator/denominator ratios without using divisions.
+ return strength_numerator * b.strength_denominator >
+ b.strength_numerator * strength_denominator;
+ }
+ };
+
+ RTC_DCHECK_GT(max_pitch_period, auto_corr.size());
+ RTC_DCHECK_LT(max_pitch_period, pitch_buf.size());
+ const size_t frame_size = pitch_buf.size() - max_pitch_period;
+ // TODO(bugs.webrtc.org/9076): Maybe optimize using vectorization.
+ float yy =
+ std::inner_product(pitch_buf.begin(), pitch_buf.begin() + frame_size + 1,
+ pitch_buf.begin(), 1.f);
+ // Search best and second best pitches by looking at the scaled
+ // auto-correlation.
+ PitchCandidate candidate;
+ PitchCandidate best;
+ PitchCandidate second_best;
+ second_best.period_inverted_lag = 1;
+ for (size_t inv_lag = 0; inv_lag < auto_corr.size(); ++inv_lag) {
+ // A pitch candidate must have positive correlation.
+ if (auto_corr[inv_lag] > 0) {
+ candidate.period_inverted_lag = inv_lag;
+ candidate.strength_numerator = auto_corr[inv_lag] * auto_corr[inv_lag];
+ candidate.strength_denominator = yy;
+ if (candidate.HasStrongerPitchThan(second_best)) {
+ if (candidate.HasStrongerPitchThan(best)) {
+ second_best = best;
+ best = candidate;
+ } else {
+ second_best = candidate;
+ }
+ }
+ }
+ // Update |squared_energy_y| for the next inverted lag.
+ const float old_coeff = pitch_buf[inv_lag];
+ const float new_coeff = pitch_buf[inv_lag + frame_size];
+ yy -= old_coeff * old_coeff;
+ yy += new_coeff * new_coeff;
+ yy = std::max(0.f, yy);
+ }
+ return {{best.period_inverted_lag, second_best.period_inverted_lag}};
+}
+
+size_t RefinePitchPeriod48kHz(
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ rtc::ArrayView<const size_t, 2> inv_lags) {
+ // Compute the auto-correlation terms only for neighbors of the given pitch
+ // candidates (similar to what is done in ComputePitchAutoCorrelation(), but
+ // for a few lag values).
+ std::array<float, kNumInvertedLags24kHz> auto_corr;
+ auto_corr.fill(0.f); // Zeros become ignored lags in FindBestPitchPeriods().
+ auto is_neighbor = [](size_t i, size_t j) {
+ return ((i > j) ? (i - j) : (j - i)) <= 2;
+ };
+ for (size_t inv_lag = 0; inv_lag < auto_corr.size(); ++inv_lag) {
+ if (is_neighbor(inv_lag, inv_lags[0]) || is_neighbor(inv_lag, inv_lags[1]))
+ auto_corr[inv_lag] =
+ ComputeAutoCorrelationCoeff(pitch_buf, inv_lag, kMaxPitch24kHz);
+ }
+ // Find best pitch at 24 kHz.
+ const auto pitch_candidates_inv_lags = FindBestPitchPeriods(
+ {auto_corr.data(), auto_corr.size()},
+ {pitch_buf.data(), pitch_buf.size()}, kMaxPitch24kHz);
+ const auto inv_lag = pitch_candidates_inv_lags[0]; // Refine the best.
+ // Pseudo-interpolation.
+ return PitchPseudoInterpolationInvLagAutoCorr(
+ inv_lag, {auto_corr.data(), auto_corr.size()});
+}
+
+PitchInfo CheckLowerPitchPeriodsAndComputePitchGain(
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ size_t initial_pitch_period_48kHz,
+ PitchInfo prev_pitch_48kHz) {
+ RTC_DCHECK_LE(kMinPitch48kHz, initial_pitch_period_48kHz);
+ RTC_DCHECK_LE(initial_pitch_period_48kHz, kMaxPitch48kHz);
+ // Stores information for a refined pitch candidate.
+ struct RefinedPitchCandidate {
+ RefinedPitchCandidate() {}
+ RefinedPitchCandidate(size_t period_24kHz, float gain, float xy, float yy)
+ : period_24kHz(period_24kHz), gain(gain), xy(xy), yy(yy) {}
+ size_t period_24kHz;
+ // Pitch strength information.
+ float gain;
+ // Additional pitch strength information used for the final estimation of
+ // pitch gain.
+ float xy; // Cross-correlation.
+ float yy; // Auto-correlation.
+ };
+
+ // Initialize.
+ std::array<float, kMaxPitch24kHz + 1> yy_values;
+ ComputeSlidingFrameSquareEnergies(pitch_buf,
+ {yy_values.data(), yy_values.size()});
+ const float xx = yy_values[0];
+ // Helper lambdas.
+ const auto pitch_gain = [](float xy, float yy, float xx) {
+ RTC_DCHECK_LE(0.f, xx * yy);
+ return xy / std::sqrt(1.f + xx * yy);
+ };
+ // Initial pitch candidate gain.
+ RefinedPitchCandidate best_pitch;
+ best_pitch.period_24kHz =
+ std::min(initial_pitch_period_48kHz / 2, kMaxPitch24kHz - 1);
+ best_pitch.xy = ComputeAutoCorrelationCoeff(
+ pitch_buf, GetInvertedLag(best_pitch.period_24kHz), kMaxPitch24kHz);
+ best_pitch.yy = yy_values[best_pitch.period_24kHz];
+ best_pitch.gain = pitch_gain(best_pitch.xy, best_pitch.yy, xx);
+
+ // Store the initial pitch period information.
+ const size_t initial_pitch_period = best_pitch.period_24kHz;
+ const float initial_pitch_gain = best_pitch.gain;
+
+ // Given the initial pitch estimation, check lower periods (i.e., harmonics).
+ const auto alternative_period = [](size_t period, size_t k,
+ size_t n) -> size_t {
+ RTC_DCHECK_LT(0, k);
+ return (2 * n * period + k) / (2 * k); // Same as round(n*period/k).
+ };
+ for (size_t k = 2; k < kSubHarmonicMultipliers.size() + 2; ++k) {
+ size_t candidate_pitch_period =
+ alternative_period(initial_pitch_period, k, 1);
+ if (candidate_pitch_period < kMinPitch24kHz)
+ break;
+ // When looking at |candidate_pitch_period|, we also look at one of its
+ // sub-harmonics. |kSubHarmonicMultipliers| is used to know where to look.
+ // |k| == 2 is a special case since |candidate_pitch_secondary_period| might
+ // be greater than the maximum pitch period.
+ size_t candidate_pitch_secondary_period = alternative_period(
+ initial_pitch_period, k, kSubHarmonicMultipliers[k - 2]);
+ if (k == 2 && candidate_pitch_secondary_period > kMaxPitch24kHz)
+ candidate_pitch_secondary_period = initial_pitch_period;
+ RTC_DCHECK_NE(candidate_pitch_period, candidate_pitch_secondary_period)
+ << "The lower pitch period and the additional sub-harmonic must not "
+ << "coincide.";
+ // Compute an auto-correlation score for the primary pitch candidate
+ // |candidate_pitch_period| by also looking at its possible sub-harmonic
+ // |candidate_pitch_secondary_period|.
+ float xy_primary_period = ComputeAutoCorrelationCoeff(
+ pitch_buf, GetInvertedLag(candidate_pitch_period), kMaxPitch24kHz);
+ float xy_secondary_period = ComputeAutoCorrelationCoeff(
+ pitch_buf, GetInvertedLag(candidate_pitch_secondary_period),
+ kMaxPitch24kHz);
+ float xy = 0.5f * (xy_primary_period + xy_secondary_period);
+ float yy = 0.5f * (yy_values[candidate_pitch_period] +
+ yy_values[candidate_pitch_secondary_period]);
+ float candidate_pitch_gain = pitch_gain(xy, yy, xx);
+
+ // Maybe update best period.
+ float threshold = ComputePitchGainThreshold(
+ candidate_pitch_period, k, initial_pitch_period, initial_pitch_gain,
+ prev_pitch_48kHz.period / 2, prev_pitch_48kHz.gain);
+ if (candidate_pitch_gain > threshold) {
+ best_pitch = {candidate_pitch_period, candidate_pitch_gain, xy, yy};
+ }
+ }
+
+ // Final pitch gain and period.
+ best_pitch.xy = std::max(0.f, best_pitch.xy);
+ RTC_DCHECK_LE(0.f, best_pitch.yy);
+ float final_pitch_gain = (best_pitch.yy <= best_pitch.xy)
+ ? 1.f
+ : best_pitch.xy / (best_pitch.yy + 1.f);
+ final_pitch_gain = std::min(best_pitch.gain, final_pitch_gain);
+ size_t final_pitch_period_48kHz = std::max(
+ kMinPitch48kHz,
+ PitchPseudoInterpolationLagPitchBuf(best_pitch.period_24kHz, pitch_buf));
+
+ return {final_pitch_period_48kHz, final_pitch_gain};
+}
+
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h b/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h
new file mode 100644
index 0000000..dfe1b35
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_SEARCH_INTERNAL_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_SEARCH_INTERNAL_H_
+
+#include <array>
+
+#include "api/array_view.h"
+#include "modules/audio_processing/agc2/rnn_vad/common.h"
+#include "modules/audio_processing/agc2/rnn_vad/pitch_info.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+// The inverted lags for the pitch interval [|kInitialMinPitch12kHz|,
+// |kMaxPitch12kHz|] are in the range [0, |kNumInvertedLags|].
+static_assert(kMaxPitch12kHz > kInitialMinPitch12kHz, "");
+static_assert(kMaxPitch24kHz > kInitialMinPitch24kHz, "");
+constexpr size_t kNumInvertedLags12kHz = kMaxPitch12kHz - kInitialMinPitch12kHz;
+constexpr size_t kNumInvertedLags24kHz = kMaxPitch24kHz - kInitialMinPitch24kHz;
+
+// Performs 2x decimation without any anti-aliasing filter.
+void Decimate2x(rtc::ArrayView<const float, kBufSize24kHz> src,
+ rtc::ArrayView<float, kBufSize12kHz> dst);
+
+// Computes a gain threshold for a candidate pitch period given the initial and
+// the previous pitch period and gain estimates and the pitch period ratio used
+// to derive the candidate pitch period from the initial period.
+float ComputePitchGainThreshold(size_t candidate_pitch_period,
+ size_t pitch_period_ratio,
+ size_t initial_pitch_period,
+ float initial_pitch_gain,
+ size_t prev_pitch_period,
+ size_t prev_pitch_gain);
+
+// Computes the sum of squared samples for every sliding frame in the pitch
+// buffer. |yy_values| indexes are lags.
+//
+// The pitch buffer is structured as depicted below:
+// |.........|...........|
+// a b
+// The part on the left, named "a" contains the oldest samples, whereas "b" the
+// most recent ones. The size of "a" corresponds to the maximum pitch period,
+// that of "b" to the frame size (e.g., 16 ms and 20 ms respectively).
+void ComputeSlidingFrameSquareEnergies(
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ rtc::ArrayView<float, kMaxPitch24kHz + 1> yy_values);
+
+// Computes the auto-correlation coefficients for a given pitch interval.
+// |auto_corr| indexes are inverted lags.
+//
+// The auto-correlations coefficients are computed as follows:
+// |.........|...........| <- pitch buffer
+// [ x (fixed) ]
+// [ y_0 ]
+// [ y_{m-1} ]
+// x and y are sub-array of equal length; x is never moved, whereas y slides.
+// The cross-correlation between y_0 and x corresponds to the auto-correlation
+// for the maximum pitch period. Hence, the first value in |auto_corr| has an
+// inverted lag equal to 0 that corresponds to a lag equal to the maximum pitch
+// period.
+void ComputePitchAutoCorrelation(
+ rtc::ArrayView<const float, kBufSize12kHz> pitch_buf,
+ size_t max_pitch_period,
+ rtc::ArrayView<float, kNumInvertedLags12kHz> auto_corr);
+
+// Given the auto-correlation coefficients stored according to
+// ComputePitchAutoCorrelation() (i.e., using inverted lags), returns the best
+// and the second best pitch periods.
+std::array<size_t, 2> FindBestPitchPeriods(
+ rtc::ArrayView<const float> auto_corr,
+ rtc::ArrayView<const float> pitch_buf,
+ size_t max_pitch_period);
+
+// Refines the pitch period estimation given the pitch buffer |pitch_buf| and
+// the initial pitch period estimation |inv_lags|. Returns an inverted lag at
+// 48 kHz.
+size_t RefinePitchPeriod48kHz(
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ rtc::ArrayView<const size_t, 2> inv_lags);
+
+// Refines the pitch period estimation and compute the pitch gain. Returns the
+// refined pitch estimation data at 48 kHz.
+PitchInfo CheckLowerPitchPeriodsAndComputePitchGain(
+ rtc::ArrayView<const float, kBufSize24kHz> pitch_buf,
+ size_t initial_pitch_period_48kHz,
+ PitchInfo prev_pitch_48kHz);
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_PITCH_SEARCH_INTERNAL_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_search_internal_unittest.cc b/modules/audio_processing/agc2/rnn_vad/pitch_search_internal_unittest.cc
new file mode 100644
index 0000000..9a6a267
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_search_internal_unittest.cc
@@ -0,0 +1,531 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/pitch_search_internal.h"
+
+#include <array>
+#include <tuple>
+
+#include "modules/audio_processing/agc2/rnn_vad/test_utils.h"
+// TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+// #include "test/fpe_observer.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+namespace {
+
+// TODO(bugs.webrtc.org/9076): Move to resource file.
+constexpr std::array<float, kBufSize24kHz> kPitchBufferData = {
+ -35.248100281f, -25.836528778f, 5.682674408f, 2.880297661f, -1.648161888f,
+ -4.094896793f, -3.500580072f, -0.896141529f, -2.989939451f, -4.608089447f,
+ -3.721750736f, -2.290785789f, -3.326566458f, -4.370154381f, -3.221047878f,
+ -4.049056530f, -2.846302271f, -1.805017233f, -1.547624588f, -0.809937477f,
+ -1.446955442f, -3.258146763f, -1.849959373f, 0.005283833f, -0.571619749f,
+ -0.630573988f, -0.162780523f, -2.699024916f, -0.856231451f, 2.748089552f,
+ 2.026614428f, -0.474685907f, -0.571918726f, 1.186420918f, 1.770769954f,
+ 2.017296791f, 1.154794335f, 1.082345366f, 1.954892635f, 2.249727726f,
+ 2.643483400f, 1.857815385f, 0.064472735f, 0.015978813f, 0.301099658f,
+ 0.478950322f, -0.669701457f, -0.654453993f, 1.338572979f, -0.493052602f,
+ -1.763812065f, 0.524392128f, 0.010438919f, -1.726593733f, -2.866710663f,
+ -2.065258503f, -3.010460854f, -3.994765282f, -4.102010250f, -3.135548830f,
+ -2.597487926f, -2.255330563f, -1.002008915f, 0.523116589f, 1.430158496f,
+ -1.655169368f, -2.263641357f, 0.766040802f, 1.166070461f, 0.002490997f,
+ 0.401043415f, -0.158550858f, -0.572042346f, 1.365390539f, -1.397871614f,
+ -2.020734787f, -1.979169965f, -1.025816441f, 0.012545407f, -1.042758584f,
+ -1.206598401f, -1.140330791f, -3.060853720f, -3.530077934f, -1.774474382f,
+ -1.342000484f, -3.171817064f, -2.489153862f, -1.593364000f, -2.552185535f,
+ -2.899760723f, -4.698278427f, -4.123534203f, -2.613421679f, -2.061793327f,
+ -4.113687515f, -3.174087524f, -2.367874622f, -4.523970604f, -4.250762939f,
+ -2.752931118f, -1.547106743f, -4.109455109f, -3.893044472f, -2.348384857f,
+ -3.194510698f, -3.502159357f, -2.785978794f, -1.981978416f, -3.279178143f,
+ -3.007923365f, -1.801304340f, -1.839247227f, -1.003675938f, -0.985928297f,
+ -1.647925615f, -2.166392088f, -1.947163343f, 0.488545895f, 1.567199469f,
+ -1.179960012f, -2.710370064f, -2.613196850f, -3.205850124f, -2.796218395f,
+ -0.715085745f, 1.406243801f, -0.779834270f, -2.075612307f, -0.922246933f,
+ -1.849850416f, 0.979040504f, 3.570628166f, 0.945924520f, -2.821768284f,
+ -6.262358189f, -6.154916763f, -0.567943573f, 2.386518955f, 1.673806906f,
+ -3.676584721f, -7.129202843f, -3.311969519f, 1.126702785f, 3.218248606f,
+ 1.600885630f, -1.709451079f, -6.822564125f, -6.011950970f, -0.671678543f,
+ 1.080205441f, -1.342422366f, -3.589303732f, -3.586701870f, -3.425134897f,
+ -1.078015327f, 2.556719542f, 0.469867468f, 0.139251709f, -0.118916273f,
+ -1.284181952f, 0.941113472f, 0.550188303f, -1.767568469f, -5.429461956f,
+ -5.065113068f, -2.111886740f, -3.606999397f, -2.410579205f, 1.013466120f,
+ 1.057218194f, 0.305267453f, 2.898609161f, 5.776575565f, 4.792305946f,
+ -0.863526106f, -2.439013481f, -0.825202525f, -2.297998428f, -0.520106375f,
+ -0.653605103f, -3.204111576f, -2.455038786f, -2.160304308f, 0.622359931f,
+ 3.803062916f, 4.340928555f, 2.390868664f, 1.645600080f, 0.405841053f,
+ -0.153203994f, 3.438643217f, 4.752261162f, 1.552502871f, 1.947945356f,
+ 0.856451511f, -0.606808305f, -1.223945618f, -1.845071912f, -0.204472303f,
+ 1.750840783f, 2.435559034f, -1.253612280f, -2.675215721f, 1.614801407f,
+ 3.002861023f, 1.743503809f, 3.409059286f, 4.303173542f, 2.441751957f,
+ 1.752274275f, 1.874113560f, 2.070837736f, 1.401355743f, -0.330647945f,
+ -0.664121151f, 1.196543574f, 1.506967187f, 0.985752344f, -1.265938520f,
+ -1.433794141f, 0.380195618f, 0.061504841f, 1.079771042f, 1.773771763f,
+ 3.226663589f, 4.170571804f, 4.220288277f, 3.619904041f, 2.316211224f,
+ 2.012817860f, 0.370972633f, 0.517094851f, 1.869508862f, 0.357770681f,
+ -2.991472483f, -3.216646433f, 0.232109070f, 1.803660274f, 2.928784370f,
+ 4.909455776f, 5.913621426f, 4.653719902f, 4.387111187f, 4.793289661f,
+ 4.744520187f, 5.214610100f, 3.996322632f, 2.619040728f, 0.758128643f,
+ -0.092789888f, 0.070066452f, 0.704165459f, 2.042234898f, 2.768569231f,
+ 3.340583324f, 3.212181091f, 2.748130322f, 3.077554941f, 2.189792156f,
+ 2.646749735f, 2.817450523f, 1.611892223f, 1.981805444f, -1.088236094f,
+ -2.187484741f, -0.654897690f, -0.900939941f, 0.148309708f, 1.498139143f,
+ -0.261296749f, -3.220157146f, -1.727450609f, 0.807144105f, -0.809251904f,
+ -2.361308336f, -1.421746969f, -0.793132067f, -0.313778281f, -0.641793191f,
+ -0.999286890f, 0.219423503f, 0.976444781f, 0.152786255f, -0.405437022f,
+ 0.120257735f, -0.392024517f, -0.019678771f, 1.492373466f, 0.926774263f,
+ 0.566291928f, 1.307234287f, 1.496955752f, 1.448441863f, 2.212901354f,
+ 1.314700723f, 0.213681281f, 1.011370897f, 1.827155828f, 0.250772655f,
+ -0.429592669f, 0.435638547f, 1.506532907f, 1.350761652f, -0.387142301f,
+ -1.770648122f, -2.690037489f, -1.788924456f, -2.023291588f, -2.354584694f,
+ -2.587521076f, -2.002159595f, -0.355855435f, 0.825611115f, 3.075081587f,
+ 2.687968254f, 0.074088633f, 0.439936757f, 1.214704275f, 2.670343399f,
+ 1.567362547f, -1.573154926f, -3.216549397f, -3.596383333f, -3.893716335f,
+ -2.456265688f, -4.313135624f, -5.783064842f, -5.344826221f, -3.484399319f,
+ -2.235594273f, -3.568959475f, -2.447141886f, -0.755384564f, -1.178364277f,
+ 1.034289122f, 1.746821165f, -1.159413576f, -2.569937706f, -1.742212296f,
+ -0.270784855f, 1.886857986f, 0.831889153f, 0.636521816f, -0.067433357f,
+ -0.256595969f, 0.907287478f, 1.575596929f, 0.393882513f, -0.510042071f,
+ 0.507258415f, 0.059408009f, 1.776192427f, 1.664948106f, -0.341539711f,
+ -0.072047889f, -0.795555651f, 0.704908550f, 2.127685547f, 1.486027241f,
+ 1.973046541f, 2.456688404f, 2.871328354f, 4.989626408f, 5.076294422f,
+ 4.262395859f, 3.622689009f, 3.241683960f, 4.222597599f, 3.575423479f,
+ 1.997965097f, 1.391216874f, 2.329971790f, 2.898612261f, 3.871258736f,
+ 2.857767582f, 2.960238218f, 3.047467470f, 2.790968180f, 2.183730364f,
+ 1.991029263f, 2.727865934f, 1.561259747f, 0.787606239f, 3.036532879f,
+ 2.430759192f, 1.475822210f, 2.307994127f, 1.857011318f, 1.538355589f,
+ 2.320549965f, 3.305005074f, 2.554165363f, 2.630100727f, 3.506094217f,
+ 4.454113483f, 2.894124269f, 4.061129570f, 4.425602436f, 3.218537807f,
+ 2.712452173f, 5.546891212f, 6.138017654f, 5.897895813f, 5.698192596f,
+ 4.096743584f, 2.661385298f, 3.646550655f, 4.626225948f, 5.025664330f,
+ 3.861543894f, 4.374861717f, 5.388185978f, 3.376737356f, 2.751175404f,
+ 3.299628258f, 2.025987387f, 1.094563961f, 0.128147125f, -4.321690559f,
+ -6.165239811f, -4.245608330f, -2.974690914f, -5.110438824f, -6.619713306f,
+ -6.594148636f, -7.972207069f, -8.034727097f, -7.296438217f, -6.822746754f,
+ -6.375267029f, -7.629575729f, -8.404177666f, -5.002337456f, -7.024040699f,
+ -7.799823761f, -5.423873901f, -4.861459732f, -2.772324085f, 0.002551556f,
+ -1.445306778f, -1.726813316f, 0.889497757f, 1.760663986f, 2.722227097f,
+ 4.755805969f, 4.188167572f, 1.547533512f, 2.444593906f, 1.612852097f,
+ -0.508655310f, 0.046535015f, 1.720140934f, 1.265070438f, 0.976964772f,
+ 2.446830273f, 6.308787823f, 7.798269272f, 5.347163200f, 3.540414810f,
+ 3.510186911f, 4.305843830f, 5.957427025f, 7.200410843f, 7.049768448f,
+ 7.179680824f, 8.508881569f, 9.094768524f, 12.307214737f, 14.215225220f,
+ 11.316717148f, 8.660657883f, 7.528784275f, 7.616339207f, 6.968524933f,
+ 4.246424198f, 0.214603424f, 0.449179649f, 1.695000648f, 0.110423088f,
+ -0.304885864f, -2.038585663f, -5.223299980f, -5.486608505f, -5.728059292f,
+ -4.866038799f, -2.678806305f, -3.464673519f, -3.407086372f, -2.490849733f,
+ -0.161162257f, 0.118952155f, 0.312392950f, -0.341049194f, 0.013419867f,
+ 3.722306252f, 3.901551247f, 1.781876802f, 2.446551561f, 3.659160852f,
+ 2.530288696f, 3.577404499f, 3.201550961f, 0.281389952f, -0.291333675f,
+ 1.386508465f, 2.181721210f, -2.802821159f, -1.531007886f, 1.608560324f,
+ -0.523656845f, -0.281057000f, 0.571323991f, 0.668095112f, -1.637194037f,
+ -2.756963253f, -1.340666890f, -2.180127621f, -1.874165773f, 0.660111070f,
+ 0.197176635f, 0.781580091f, 1.749967933f, 0.674724638f, -2.082683325f,
+ -3.159717083f, -2.898023844f, -4.691623211f, -5.614190102f, -6.157790661f,
+ -7.776132584f, -8.029224396f, -6.940879345f, -7.065263271f, -7.003522396f,
+ -5.691181183f, -7.872379780f, -7.614178658f, -5.778759003f, -4.605045319f,
+ -4.695390224f, -5.865473270f, -5.825413227f, -4.648111820f, -2.193091869f,
+ -0.172003269f, 1.482686043f, -0.915655136f, -2.626194954f, 1.852293015f,
+ 4.184171677f, 4.083235264f, 1.048256874f, -1.361350536f, 0.438748837f,
+ 1.716395378f, 2.916294813f, 2.639499664f, 0.059617281f, -1.883811951f,
+ 2.136622429f, 6.641947269f, 5.951328754f, 3.875293493f, 3.003573895f,
+ 2.687273264f, 4.843512535f, 6.420391560f, 6.014624596f, 3.444208860f,
+ 0.717782736f, 2.659932613f, 5.204012871f, 5.516477585f, 3.315031528f,
+ 0.454023123f, -0.026421070f, 0.802503586f, 2.606507778f, 1.679640770f,
+ -1.917723656f, -3.348850250f, -2.580049515f, -1.783200264f, -0.810425520f,
+ -0.374402523f, -3.705567360f, -5.367071629f, -4.344952106f, -0.968293428f,
+ 1.147591949f, -1.240655184f, -2.621209621f, -2.452539444f, -1.543132067f,
+ 0.422753096f, 1.026433110f, 0.858573675f, -0.695377707f, -0.242624998f,
+ 3.892488956f, 4.100893021f, 3.498974323f, 1.744507313f, -0.912925899f,
+ 0.929271877f, 3.531583786f, 4.938030243f, 4.081199646f, 0.061933577f,
+ -2.232783318f, -1.356980443f, 1.794556737f, 3.510458231f, 1.323192716f,
+ -0.505770206f, 2.126557350f, 2.507567406f, 2.232018232f, 1.872283101f,
+ 1.265762568f, 0.577634692f, 0.021484375f, 3.114191532f, 1.579384208f,
+ 0.930754900f, 0.308351398f, -0.425426602f, 3.359810352f, 2.437057972f,
+ 1.210662127f, 0.708607912f, -1.576705575f, 0.007833481f, -0.178357601f,
+ -0.880272985f, 0.078738928f, 0.339336634f, -0.763550043f, -1.669098496f,
+ -2.083987713f, -1.946106076f, -0.953974366f, -0.856883168f, -1.282670021f,
+ -1.551425457f, -2.249363184f, -2.555188894f, -1.254808664f, -1.368662596f,
+ -1.839509130f, -0.839046180f, -0.452676475f, 0.721064806f, 1.988085508f,
+ 0.456556678f, -0.255003691f, 0.384676337f, 1.075410485f, 0.617453933f,
+ 1.470067143f, 1.493275523f, 0.954153359f, 1.027234554f, -0.434967309f,
+ -0.694453120f, 0.477285773f, 0.436861426f, 1.486879349f, -0.158989906f,
+ 0.361879885f, 3.234876394f, 1.105287671f, -0.982552111f, 1.514200211f,
+ 0.821707547f, -1.142312169f, 1.845819831f, 3.934516191f, 2.251807690f,
+ 0.530044913f, -1.043874860f, -0.891365111f, -0.264675498f, 0.288083673f,
+ 0.606682122f, -1.132072091f, -3.530973911f, -2.005296707f, 0.335011721f,
+ -0.240332901f, -2.763209343f, -2.148519516f, -1.864180326f, -0.814615071f,
+ -1.589591861f, -2.455522776f, -0.756391644f, 0.689822078f, 0.171640277f,
+ -0.225937843f, 0.363246441f, 0.098157287f, -1.638891220f, -0.400456548f,
+ 1.076233864f, 2.288599968f, 2.716089964f, 1.585703373f, 0.846301913f,
+ 0.887506902f, -0.439320147f, -0.823126972f, 0.712436378f, 1.027045608f,
+ 0.360925227f, -2.289939404f, -1.035227180f, 0.931313038f, -0.133454978f,
+ 0.160856903f, 0.700653732f, 0.817580283f, -0.223383546f, 0.713623106f,
+ 1.327106714f, 1.558022618f, 1.346337557f, -0.661301017f, 0.707845926f,
+ 2.435726643f, 0.763329387f, 0.485213757f, 2.295393229f, 4.525130272f,
+ 2.354229450f, -0.043517172f, 1.635316610f, 1.651852608f, 1.240020633f,
+ 0.320237398f, -0.571269870f, -0.686546564f, -1.796948791f, -0.966899753f,
+ -0.404109240f, -1.295783877f, -2.058131218f, -2.279026985f, -2.183017731f,
+ -2.516988277f, -0.276667058f, -0.475267202f, -2.645681143f, -0.504431605f,
+ -1.031255722f, -3.401877880f, -1.075011969f, -0.667404234f, -2.419279575f,
+ -1.230643749f, 1.151491284f, 0.374734998f, -2.004124880f, -1.923788905f,
+ -0.767004371f, 0.512374282f, 2.254727125f, 1.373157024f, 0.633022547f,
+ 0.194831967f, 0.226476192f, 1.294842482f, 0.838023365f, 1.291390896f,
+ 0.128176212f, -1.109287858f, 0.166733295f, 0.847469866f, -0.662097514f,
+ -0.489783406f, 1.523754478f, 1.903803706f, -0.748670340f, 0.721136212f,
+ 1.627746105f, -0.731291413f, 0.646574259f, 1.722917080f, 0.372141778f,
+ -0.063563704f, 0.916404963f, 2.092662811f, 1.699481010f, 0.181074798f,
+ -1.361395121f, 0.581034362f, 1.451567292f, 0.526586652f, 1.206429839f,
+ -1.041464567f, -2.891606331f, 0.638695598f, 1.198848009f, -0.771047413f,
+ -1.074250221f, -0.500067651f, 0.308775485f, 0.552724898f, 1.083443999f,
+ 1.371356130f, 0.360372365f, 3.391613960f, 2.896605730f, 0.799045980f,
+ 0.922905385f, 3.240214348f, 4.740911484f, 2.945639610f, 2.544054747f,
+ 3.048654795f, 3.541822433f, 4.390746117f, 5.632675171f, 7.721554756f,
+ 6.390114784f, 5.962307930f, 5.873732567f, 5.625522137f, 4.857854843f,
+ 3.148367405f, 3.966898203f, 4.309705257f, 3.543770313f, 2.427399397f,
+ 0.324177742f, -1.809771061f, -2.191485405f, 0.006873131f, -0.876847267f,
+ -0.928904057f, 0.889565945f, -0.127671242f, -1.695463657f, -1.193793774f,
+ -1.452976227f, -3.406696558f, -2.564189196f, -2.136555195f, -2.374645710f,
+ -3.230790854f, -3.076714516f, -3.245117664f, -2.254387617f, -0.245034039f,
+ -1.072510719f, -1.887740970f, 0.431427240f, 1.132410765f, -1.015120149f,
+ -0.274977922f, -1.910447717f, -2.865208864f, -0.131696820f};
+
+// TODO(bugs.webrtc.org/9076): Move to resource file.
+constexpr std::array<float, 385> kPitchBufferFrameSquareEnergies = {
+ 5150.291992188f, 5150.894531250f, 5145.122558594f, 5148.914062500f,
+ 5152.802734375f, 5156.541015625f, 5163.048339844f, 5172.149414062f,
+ 5177.349121094f, 5184.365722656f, 5199.292480469f, 5202.612304688f,
+ 5197.510253906f, 5189.979492188f, 5183.533203125f, 5190.677734375f,
+ 5203.943359375f, 5207.876464844f, 5209.395019531f, 5225.451660156f,
+ 5249.794921875f, 5271.816894531f, 5280.045410156f, 5285.289062500f,
+ 5288.319335938f, 5289.758789062f, 5294.285644531f, 5289.979980469f,
+ 5287.337402344f, 5287.237792969f, 5281.462402344f, 5271.676269531f,
+ 5256.257324219f, 5240.524414062f, 5230.869628906f, 5207.531250000f,
+ 5176.040039062f, 5144.021484375f, 5109.295410156f, 5068.527832031f,
+ 5008.909667969f, 4977.587890625f, 4959.000976562f, 4950.016601562f,
+ 4940.795410156f, 4937.358398438f, 4935.286132812f, 4914.154296875f,
+ 4906.706542969f, 4906.924804688f, 4907.674804688f, 4899.855468750f,
+ 4894.340820312f, 4906.948242188f, 4910.065429688f, 4921.032714844f,
+ 4949.294433594f, 4982.643066406f, 5000.996093750f, 5005.875488281f,
+ 5020.441894531f, 5031.938964844f, 5041.877441406f, 5035.990722656f,
+ 5037.362792969f, 5043.038085938f, 5044.236328125f, 5042.322753906f,
+ 5041.990722656f, 5047.362304688f, 5056.785644531f, 5054.579101562f,
+ 5050.326171875f, 5053.495117188f, 5060.186523438f, 5065.591796875f,
+ 5066.717285156f, 5069.499511719f, 5076.201171875f, 5076.687011719f,
+ 5076.316894531f, 5077.581054688f, 5076.226074219f, 5074.094238281f,
+ 5074.039062500f, 5073.663574219f, 5076.283691406f, 5077.278808594f,
+ 5076.094238281f, 5077.806152344f, 5081.035644531f, 5082.431640625f,
+ 5082.995605469f, 5084.653320312f, 5084.936035156f, 5085.394042969f,
+ 5085.735351562f, 5080.651855469f, 5080.542968750f, 5079.969238281f,
+ 5076.432617188f, 5072.439453125f, 5073.252441406f, 5071.974609375f,
+ 5071.458496094f, 5066.017578125f, 5065.670898438f, 5065.144042969f,
+ 5055.592773438f, 5060.104980469f, 5060.505371094f, 5054.157226562f,
+ 5056.915039062f, 5067.208007812f, 5060.940917969f, 5058.419921875f,
+ 5053.248046875f, 5049.823730469f, 5048.573242188f, 5053.195312500f,
+ 5053.444335938f, 5054.143066406f, 5056.270019531f, 5063.881835938f,
+ 5070.784667969f, 5074.042480469f, 5080.785156250f, 5085.663085938f,
+ 5095.979003906f, 5101.596191406f, 5088.784667969f, 5087.686523438f,
+ 5087.946777344f, 5087.369140625f, 5081.445312500f, 5081.519042969f,
+ 5087.940917969f, 5102.099121094f, 5126.864257812f, 5147.613281250f,
+ 5170.079589844f, 5189.276367188f, 5210.265136719f, 5244.745117188f,
+ 5268.821777344f, 5277.381835938f, 5279.768066406f, 5278.750000000f,
+ 5283.853027344f, 5292.671386719f, 5291.744628906f, 5294.732421875f,
+ 5294.322265625f, 5294.267089844f, 5297.530761719f, 5302.179199219f,
+ 5312.768066406f, 5323.202148438f, 5335.357910156f, 5344.610839844f,
+ 5347.597167969f, 5346.077148438f, 5346.071289062f, 5346.083984375f,
+ 5348.088378906f, 5349.661621094f, 5350.157226562f, 5351.855957031f,
+ 5347.257812500f, 5345.171875000f, 5344.617675781f, 5343.106445312f,
+ 5342.778808594f, 5338.655761719f, 5341.668457031f, 5347.518066406f,
+ 5362.014160156f, 5361.167968750f, 5362.926269531f, 5371.575195312f,
+ 5374.099609375f, 5381.186523438f, 5381.963867188f, 5386.806152344f,
+ 5389.590820312f, 5384.562011719f, 5372.485839844f, 5370.576660156f,
+ 5369.640136719f, 5369.698242188f, 5371.199707031f, 5372.644531250f,
+ 5394.006835938f, 5395.366699219f, 5395.259277344f, 5395.398437500f,
+ 5395.895507812f, 5401.420898438f, 5420.036621094f, 5434.017578125f,
+ 5434.215820312f, 5437.827636719f, 5442.944335938f, 5450.980468750f,
+ 5449.246582031f, 5449.135742188f, 5453.259765625f, 5453.792968750f,
+ 5459.676757812f, 5460.213867188f, 5479.227539062f, 5512.076171875f,
+ 5520.272949219f, 5519.662109375f, 5517.395996094f, 5516.550292969f,
+ 5520.786621094f, 5527.268066406f, 5526.668457031f, 5549.916992188f,
+ 5577.750976562f, 5580.141113281f, 5579.533691406f, 5576.632324219f,
+ 5573.938476562f, 5571.166503906f, 5570.603027344f, 5570.708496094f,
+ 5577.238769531f, 5577.625976562f, 5589.325683594f, 5602.189941406f,
+ 5612.587402344f, 5613.887695312f, 5613.588867188f, 5608.100585938f,
+ 5632.956054688f, 5679.322265625f, 5682.149414062f, 5683.846191406f,
+ 5691.708496094f, 5683.279785156f, 5694.248535156f, 5744.740722656f,
+ 5756.655761719f, 5755.952148438f, 5756.665527344f, 5750.700195312f,
+ 5784.060546875f, 5823.021972656f, 5829.233398438f, 5817.804687500f,
+ 5827.333984375f, 5826.451171875f, 5824.887695312f, 5825.734375000f,
+ 5813.386230469f, 5789.609863281f, 5779.115234375f, 5778.762695312f,
+ 5785.748046875f, 5792.981933594f, 5787.567871094f, 5778.096679688f,
+ 5764.337402344f, 5766.734375000f, 5766.489746094f, 5769.543945312f,
+ 5773.183593750f, 5775.720703125f, 5774.311523438f, 5769.303710938f,
+ 5765.815917969f, 5767.521484375f, 5775.251953125f, 5785.067382812f,
+ 5770.117187500f, 5749.073242188f, 5747.606933594f, 5757.671875000f,
+ 5762.530273438f, 5774.506347656f, 5784.737304688f, 5775.916015625f,
+ 5779.816894531f, 5795.064453125f, 5808.736816406f, 5813.699707031f,
+ 5823.773925781f, 5840.490234375f, 5833.751953125f, 5810.150390625f,
+ 5800.072265625f, 5815.070800781f, 5822.964355469f, 5817.615234375f,
+ 5783.978027344f, 5748.952636719f, 5735.553710938f, 5730.132812500f,
+ 5724.260253906f, 5721.703613281f, 5695.653808594f, 5652.838867188f,
+ 5649.729980469f, 5647.268554688f, 5647.265136719f, 5641.350585938f,
+ 5636.762695312f, 5637.900390625f, 5639.662109375f, 5639.672851562f,
+ 5638.901367188f, 5622.253417969f, 5604.906738281f, 5601.475585938f,
+ 5595.938476562f, 5595.687011719f, 5598.612792969f, 5601.322753906f,
+ 5598.558593750f, 5577.227050781f, 5544.295410156f, 5514.978027344f,
+ 5499.678222656f, 5488.303222656f, 5471.735839844f, 5429.718261719f,
+ 5376.806640625f, 5348.682128906f, 5307.851074219f, 5260.914062500f,
+ 5212.738281250f, 5148.544921875f, 5091.187500000f, 5053.512207031f,
+ 5023.785156250f, 5002.202148438f, 4994.252441406f, 4984.498046875f,
+ 4980.251464844f, 4979.796875000f, 4976.738769531f, 4979.579589844f,
+ 4986.528320312f, 4991.153808594f, 4991.462890625f, 4987.881347656f,
+ 4987.417480469f, 4983.885742188f, 4984.341308594f, 4985.302734375f,
+ 4985.303710938f, 4985.449707031f, 4989.282226562f, 4994.246582031f,
+ 4992.635742188f, 4992.064453125f, 4987.331054688f, 4985.806152344f,
+ 4986.047851562f, 4985.968750000f, 4979.141113281f, 4976.958984375f,
+ 4972.650390625f, 4959.916503906f, 4956.325683594f, 4956.408691406f,
+ 4949.288085938f, 4951.827636719f, 4962.202636719f, 4981.184570312f,
+ 4992.152832031f, 4997.386230469f, 5011.211914062f, 5026.242187500f,
+ 5023.573730469f, 5012.373046875f, 5017.451171875f, 5010.541015625f,
+ 4980.446777344f, 4958.639648438f, 4963.649902344f, 5627.020507812f,
+ 6869.356445312f};
+
+// TODO(bugs.webrtc.org/9076): Move to resource file.
+constexpr std::array<float, 147> kPitchBufferAutoCorrCoeffs = {
+ -423.526794434f, -260.724456787f, -173.558380127f, -71.720344543f,
+ -1.149698257f, 71.451370239f, 71.455848694f, 149.755233765f,
+ 199.401885986f, 243.961334229f, 269.339721680f, 243.776992798f,
+ 294.753814697f, 209.465484619f, 139.224700928f, 131.474136353f,
+ 42.872886658f, -32.431114197f, -90.191261292f, -94.912338257f,
+ -172.627227783f, -138.089843750f, -89.236648560f, -69.348426819f,
+ 25.044368744f, 44.184486389f, 61.602676392f, 150.157394409f,
+ 185.254760742f, 233.352676392f, 296.255371094f, 292.464141846f,
+ 256.903472900f, 250.926574707f, 174.207122803f, 130.214172363f,
+ 65.655899048f, -68.448402405f, -147.239669800f, -230.553405762f,
+ -311.217895508f, -447.173889160f, -509.306060791f, -551.155822754f,
+ -580.678405762f, -658.902709961f, -697.141967773f, -751.233032227f,
+ -690.860351562f, -571.689575195f, -521.124572754f, -429.477294922f,
+ -375.685913086f, -277.387329102f, -154.100753784f, -105.723197937f,
+ 117.502632141f, 219.290512085f, 255.376770020f, 444.264831543f,
+ 470.727416992f, 460.139129639f, 494.179931641f, 389.801116943f,
+ 357.082763672f, 222.748138428f, 179.100601196f, -26.893497467f,
+ -85.033767700f, -223.577529907f, -247.136367798f, -223.011428833f,
+ -292.724914551f, -246.538131714f, -247.388458252f, -228.452484131f,
+ -30.476575851f, 4.652336121f, 64.730491638f, 156.081161499f,
+ 177.569305420f, 261.671569824f, 336.274414062f, 424.203369141f,
+ 564.190734863f, 608.841796875f, 671.252136230f, 712.249877930f,
+ 623.135498047f, 564.775695801f, 576.405639648f, 380.181854248f,
+ 306.687164307f, 180.344757080f, -41.317466736f, -183.548736572f,
+ -223.835021973f, -273.299652100f, -235.727813721f, -276.899627686f,
+ -302.224975586f, -349.227142334f, -370.935058594f, -364.022613525f,
+ -287.682952881f, -273.828704834f, -156.869720459f, -88.654510498f,
+ 14.299798012f, 137.048034668f, 260.182342529f, 423.380767822f,
+ 591.277282715f, 581.151306152f, 643.898864746f, 547.919006348f,
+ 355.534271240f, 238.222915649f, 4.463035583f, -193.763305664f,
+ -281.212432861f, -546.399353027f, -615.602600098f, -574.225891113f,
+ -726.701843262f, -564.840942383f, -588.488037109f, -651.052551270f,
+ -453.769104004f, -502.886627197f, -463.373016357f, -291.709564209f,
+ -288.857421875f, -152.114242554f, 105.401855469f, 211.479980469f,
+ 468.501983643f, 796.984985352f, 880.254089355f, 1114.614379883f,
+ 1219.664794922f, 1093.687377930f, 1125.042602539f, 1020.942382812f,
+ 794.315246582f, 772.126831055f, 447.410736084f};
+
+constexpr std::array<size_t, 2> kTestPitchPeriods = {
+ 3 * kMinPitch48kHz / 2, (3 * kMinPitch48kHz + kMaxPitch48kHz) / 2,
+};
+constexpr std::array<float, 2> kTestPitchGains = {0.35f, 0.75f};
+
+} // namespace
+
+class ComputePitchGainThresholdTest
+ : public testing::Test,
+ public ::testing::WithParamInterface<
+ std::tuple<size_t, size_t, size_t, float, size_t, float, float>> {};
+
+TEST_P(ComputePitchGainThresholdTest, BitExactness) {
+ const auto params = GetParam();
+ const size_t candidate_pitch_period = std::get<0>(params);
+ const size_t pitch_period_ratio = std::get<1>(params);
+ const size_t initial_pitch_period = std::get<2>(params);
+ const float initial_pitch_gain = std::get<3>(params);
+ const size_t prev_pitch_period = std::get<4>(params);
+ const size_t prev_pitch_gain = std::get<5>(params);
+ const float threshold = std::get<6>(params);
+
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ EXPECT_NEAR(
+ threshold,
+ ComputePitchGainThreshold(candidate_pitch_period, pitch_period_ratio,
+ initial_pitch_period, initial_pitch_gain,
+ prev_pitch_period, prev_pitch_gain),
+ 3e-6f);
+ }
+}
+
+INSTANTIATE_TEST_CASE_P(
+ RnnVadTest,
+ ComputePitchGainThresholdTest,
+ ::testing::Values(
+ std::make_tuple(31, 7, 219, 0.45649201f, 199, 0.604747f, 0.40000001f),
+ std::make_tuple(113,
+ 2,
+ 226,
+ 0.20967799f,
+ 219,
+ 0.40392199f,
+ 0.30000001f),
+ std::make_tuple(63, 2, 126, 0.210788f, 364, 0.098519f, 0.40000001f),
+ std::make_tuple(30, 5, 152, 0.82356697f, 149, 0.55535901f, 0.700032f),
+ std::make_tuple(76, 2, 151, 0.79522997f, 151, 0.82356697f, 0.675946f),
+ std::make_tuple(31, 5, 153, 0.85069299f, 150, 0.79073799f, 0.72308898f),
+ std::make_tuple(78, 2, 156, 0.72750503f, 153, 0.85069299f, 0.618379f)));
+
+TEST(RnnVadTest, ComputeSlidingFrameSquareEnergiesBitExactness) {
+ std::array<float, kPitchBufferFrameSquareEnergies.size()> computed_output;
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ ComputeSlidingFrameSquareEnergies(
+ {kPitchBufferData.data(), kPitchBufferData.size()},
+ {computed_output.data(), computed_output.size()});
+ }
+ ExpectNearAbsolute({kPitchBufferFrameSquareEnergies.data(),
+ kPitchBufferFrameSquareEnergies.size()},
+ {computed_output.data(), computed_output.size()}, 3e-2f);
+}
+
+TEST(RnnVadTest, ComputePitchAutoCorrelationBitExactness) {
+ std::array<float, kBufSize12kHz> pitch_buf_decimated;
+ Decimate2x({kPitchBufferData.data(), kPitchBufferData.size()},
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()});
+ std::array<float, kPitchBufferAutoCorrCoeffs.size()> computed_output;
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ ComputePitchAutoCorrelation(
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()},
+ kMaxPitch12kHz, {computed_output.data(), computed_output.size()});
+ }
+ ExpectNearAbsolute(
+ {kPitchBufferAutoCorrCoeffs.data(), kPitchBufferAutoCorrCoeffs.size()},
+ {computed_output.data(), computed_output.size()}, 3e-3f);
+}
+
+TEST(RnnVadTest, FindBestPitchPeriodsBitExactness) {
+ std::array<float, kBufSize12kHz> pitch_buf_decimated;
+ Decimate2x({kPitchBufferData.data(), kPitchBufferData.size()},
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()});
+ std::array<size_t, 2> pitch_candidates_inv_lags;
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ pitch_candidates_inv_lags = FindBestPitchPeriods(
+ {kPitchBufferAutoCorrCoeffs}, {pitch_buf_decimated}, kMaxPitch12kHz);
+ }
+ const std::array<size_t, 2> expected_output = {140, 142};
+ EXPECT_EQ(expected_output, pitch_candidates_inv_lags);
+}
+
+TEST(RnnVadTest, RefinePitchPeriod48kHzBitExactness) {
+ std::array<float, kBufSize12kHz> pitch_buf_decimated;
+ Decimate2x({kPitchBufferData.data(), kPitchBufferData.size()},
+ {pitch_buf_decimated.data(), pitch_buf_decimated.size()});
+ size_t pitch_inv_lag;
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ const std::array<size_t, 2> pitch_candidates_inv_lags = {280, 284};
+ pitch_inv_lag = RefinePitchPeriod48kHz(
+ {kPitchBufferData.data(), kPitchBufferData.size()},
+ {pitch_candidates_inv_lags.data(), pitch_candidates_inv_lags.size()});
+ }
+ EXPECT_EQ(560u, pitch_inv_lag);
+}
+
+class CheckLowerPitchPeriodsAndComputePitchGainTest
+ : public testing::Test,
+ public ::testing::WithParamInterface<
+ std::tuple<size_t, size_t, float, size_t, float>> {};
+
+TEST_P(CheckLowerPitchPeriodsAndComputePitchGainTest, BitExactness) {
+ const auto params = GetParam();
+ const size_t initial_pitch_period = std::get<0>(params);
+ const size_t prev_pitch_period = std::get<1>(params);
+ const float prev_pitch_gain = std::get<2>(params);
+ const size_t expected_pitch_period = std::get<3>(params);
+ const float expected_pitch_gain = std::get<4>(params);
+
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ const auto computed_output = CheckLowerPitchPeriodsAndComputePitchGain(
+ {kPitchBufferData.data(), kPitchBufferData.size()},
+ initial_pitch_period, {prev_pitch_period, prev_pitch_gain});
+ EXPECT_EQ(expected_pitch_period, computed_output.period);
+ EXPECT_NEAR(expected_pitch_gain, computed_output.gain, 1e-6f);
+ }
+}
+
+INSTANTIATE_TEST_CASE_P(RnnVadTest,
+ CheckLowerPitchPeriodsAndComputePitchGainTest,
+ ::testing::Values(std::make_tuple(kTestPitchPeriods[0],
+ kTestPitchPeriods[0],
+ kTestPitchGains[0],
+ 91,
+ -0.0188608f),
+ std::make_tuple(kTestPitchPeriods[0],
+ kTestPitchPeriods[0],
+ kTestPitchGains[1],
+ 91,
+ -0.0188608f),
+ std::make_tuple(kTestPitchPeriods[0],
+ kTestPitchPeriods[1],
+ kTestPitchGains[0],
+ 91,
+ -0.0188608f),
+ std::make_tuple(kTestPitchPeriods[0],
+ kTestPitchPeriods[1],
+ kTestPitchGains[1],
+ 91,
+ -0.0188608f),
+ std::make_tuple(kTestPitchPeriods[1],
+ kTestPitchPeriods[0],
+ kTestPitchGains[0],
+ 475,
+ -0.0904344f),
+ std::make_tuple(kTestPitchPeriods[1],
+ kTestPitchPeriods[0],
+ kTestPitchGains[1],
+ 475,
+ -0.0904344f),
+ std::make_tuple(kTestPitchPeriods[1],
+ kTestPitchPeriods[1],
+ kTestPitchGains[0],
+ 475,
+ -0.0904344f),
+ std::make_tuple(kTestPitchPeriods[1],
+ kTestPitchPeriods[1],
+ kTestPitchGains[1],
+ 475,
+ -0.0904344f)));
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/pitch_search_unittest.cc b/modules/audio_processing/agc2/rnn_vad/pitch_search_unittest.cc
new file mode 100644
index 0000000..4417764
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/pitch_search_unittest.cc
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/pitch_search.h"
+
+#include <array>
+
+#include "modules/audio_processing/agc2/rnn_vad/test_utils.h"
+// TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+// #include "test/fpe_observer.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+
+// TODO(bugs.webrtc.org/9076): Remove when the issue is fixed.
+TEST(RnnVadTest, PitchSearchBitExactness) {
+ auto lp_residual_reader = CreateLpResidualAndPitchPeriodGainReader();
+ const size_t num_frames = lp_residual_reader.second;
+ std::array<float, 864> lp_residual;
+ float expected_pitch_period, expected_pitch_gain;
+ PitchInfo last_pitch;
+ {
+ // TODO(bugs.webrtc.org/8948): Add when the issue is fixed.
+ // FloatingPointExceptionObserver fpe_observer;
+
+ for (size_t i = 0; i < num_frames; ++i) {
+ SCOPED_TRACE(i);
+ lp_residual_reader.first->ReadChunk(
+ {lp_residual.data(), lp_residual.size()});
+ lp_residual_reader.first->ReadValue(&expected_pitch_period);
+ lp_residual_reader.first->ReadValue(&expected_pitch_gain);
+ last_pitch =
+ PitchSearch({lp_residual.data(), lp_residual.size()}, last_pitch);
+ EXPECT_EQ(static_cast<size_t>(expected_pitch_period), last_pitch.period);
+ EXPECT_NEAR(expected_pitch_gain, last_pitch.gain, 1e-5f);
+ }
+ }
+}
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/ring_buffer.h b/modules/audio_processing/agc2/rnn_vad/ring_buffer.h
new file mode 100644
index 0000000..294b0c0
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/ring_buffer.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RING_BUFFER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RING_BUFFER_H_
+
+#include <array>
+#include <cstring>
+#include <type_traits>
+
+#include "api/array_view.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+// Ring buffer for N arrays of type T each one with size S.
+template <typename T, size_t S, size_t N>
+class RingBuffer {
+ static_assert(S > 0, "");
+ static_assert(N > 0, "");
+ static_assert(std::is_arithmetic<T>::value,
+ "Integral or floating point required.");
+
+ public:
+ RingBuffer() : tail_(0) {}
+ RingBuffer(const RingBuffer&) = delete;
+ RingBuffer& operator=(const RingBuffer&) = delete;
+ ~RingBuffer() = default;
+ // Set the ring buffer values to zero.
+ void Reset() { buffer_.fill(0); }
+ // Replace the least recently pushed array in the buffer with |new_values|.
+ void Push(rtc::ArrayView<const T, S> new_values) {
+ std::memcpy(buffer_.data() + S * tail_, new_values.data(), S * sizeof(T));
+ tail_ += 1;
+ if (tail_ == N)
+ tail_ = 0;
+ }
+ // Return an array view onto the array with a given delay. A view on the last
+ // and least recently push array is returned when |delay| is 0 and N - 1
+ // respectively.
+ rtc::ArrayView<const T, S> GetArrayView(size_t delay) const {
+ const int delay_int = static_cast<int>(delay);
+ RTC_DCHECK_LE(0, delay_int);
+ RTC_DCHECK_LT(delay_int, N);
+ int offset = tail_ - 1 - delay_int;
+ if (offset < 0)
+ offset += N;
+ return {buffer_.data() + S * offset, S};
+ }
+
+ private:
+ int tail_; // Index of the least recently pushed sub-array.
+ std::array<T, S * N> buffer_{};
+};
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_RING_BUFFER_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/ring_buffer_unittest.cc b/modules/audio_processing/agc2/rnn_vad/ring_buffer_unittest.cc
new file mode 100644
index 0000000..91383d1
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/ring_buffer_unittest.cc
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/ring_buffer.h"
+
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+namespace {
+
+// Compare the elements of two given array views.
+template <typename T, std::ptrdiff_t S>
+void ExpectEq(rtc::ArrayView<const T, S> a, rtc::ArrayView<const T, S> b) {
+ for (size_t i = 0; i < S; ++i) {
+ SCOPED_TRACE(i);
+ EXPECT_EQ(a[i], b[i]);
+ }
+}
+
+// Test push/read sequences.
+template <typename T, size_t S, size_t N>
+void TestRingBuffer() {
+ SCOPED_TRACE(N);
+ SCOPED_TRACE(S);
+ std::array<T, S> prev_pushed_array;
+ std::array<T, S> pushed_array;
+ rtc::ArrayView<const T, S> pushed_array_view(pushed_array.data(), S);
+
+ // Init.
+ RingBuffer<T, S, N> ring_buf;
+ ring_buf.GetArrayView(0);
+ pushed_array.fill(0);
+ ring_buf.Push(pushed_array_view);
+ ExpectEq(pushed_array_view, ring_buf.GetArrayView(0));
+
+ // Push N times and check most recent and second most recent.
+ for (T v = 1; v <= static_cast<T>(N); ++v) {
+ SCOPED_TRACE(v);
+ prev_pushed_array = pushed_array;
+ pushed_array.fill(v);
+ ring_buf.Push(pushed_array_view);
+ ExpectEq(pushed_array_view, ring_buf.GetArrayView(0));
+ if (N > 1) {
+ pushed_array.fill(v - 1);
+ ExpectEq(pushed_array_view, ring_buf.GetArrayView(1));
+ }
+ }
+
+ // Check buffer.
+ for (size_t delay = 2; delay < N; ++delay) {
+ SCOPED_TRACE(delay);
+ T expected_value = N - static_cast<T>(delay);
+ pushed_array.fill(expected_value);
+ ExpectEq(pushed_array_view, ring_buf.GetArrayView(delay));
+ }
+}
+
+} // namespace
+
+// Check that for different delays, different views are returned.
+TEST(RnnVadTest, RingBufferArrayViews) {
+ constexpr size_t s = 3;
+ constexpr size_t n = 4;
+ RingBuffer<int, s, n> ring_buf;
+ std::array<int, s> pushed_array;
+ pushed_array.fill(1);
+ for (size_t k = 0; k <= n; ++k) { // Push data n + 1 times.
+ SCOPED_TRACE(k);
+ // Check array views.
+ for (size_t i = 0; i < n; ++i) {
+ SCOPED_TRACE(i);
+ auto view_i = ring_buf.GetArrayView(i);
+ for (size_t j = i + 1; j < n; ++j) {
+ SCOPED_TRACE(j);
+ auto view_j = ring_buf.GetArrayView(j);
+ EXPECT_NE(view_i, view_j);
+ }
+ }
+ ring_buf.Push({pushed_array.data(), pushed_array.size()});
+ }
+}
+
+TEST(RnnVadTest, RingBufferUnsigned) {
+ TestRingBuffer<uint8_t, 1, 1>();
+ TestRingBuffer<uint8_t, 2, 5>();
+ TestRingBuffer<uint8_t, 5, 2>();
+ TestRingBuffer<uint8_t, 5, 5>();
+}
+
+TEST(RnnVadTest, RingBufferSigned) {
+ TestRingBuffer<int, 1, 1>();
+ TestRingBuffer<int, 2, 5>();
+ TestRingBuffer<int, 5, 2>();
+ TestRingBuffer<int, 5, 5>();
+}
+
+TEST(RnnVadTest, RingBufferFloating) {
+ TestRingBuffer<float, 1, 1>();
+ TestRingBuffer<float, 2, 5>();
+ TestRingBuffer<float, 5, 2>();
+ TestRingBuffer<float, 5, 5>();
+}
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/rnn_vad_tool.cc b/modules/audio_processing/agc2/rnn_vad/rnn_vad_tool.cc
new file mode 100644
index 0000000..6ab932c
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/rnn_vad_tool.cc
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <array>
+#include <string>
+#include <vector>
+
+#include "common_audio/resampler/push_sinc_resampler.h"
+#include "common_audio/wav_file.h"
+#include "modules/audio_processing/agc2/rnn_vad/common.h"
+#include "rtc_base/flags.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using rnn_vad::kFrameSize10ms24kHz;
+
+DEFINE_string(i, "", "Path to the input wav file");
+std::string InputWavFile() {
+ return static_cast<std::string>(FLAG_i);
+}
+
+DEFINE_string(f, "", "Path to the output features file");
+std::string OutputFeaturesFile() {
+ return static_cast<std::string>(FLAG_f);
+}
+
+DEFINE_string(o, "", "Path to the output VAD probabilities file");
+std::string OutputVadProbsFile() {
+ return static_cast<std::string>(FLAG_o);
+}
+
+DEFINE_bool(help, false, "Prints this message");
+
+} // namespace
+
+int main(int argc, char* argv[]) {
+ rtc::LogMessage::LogToDebug(rtc::LS_INFO);
+ rtc::FlagList::SetFlagsFromCommandLine(&argc, argv, true);
+ if (FLAG_help) {
+ rtc::FlagList::Print(nullptr, false);
+ return 0;
+ }
+
+ // Open wav input file and check properties.
+ WavReader wav_reader(InputWavFile());
+ if (wav_reader.num_channels() != 1) {
+ RTC_LOG(LS_ERROR) << "Only mono wav files are supported";
+ return 1;
+ }
+ if (wav_reader.sample_rate() % 100 != 0) {
+ RTC_LOG(LS_ERROR) << "The sample rate rate must allow 10 ms frames.";
+ return 1;
+ }
+ RTC_LOG(LS_INFO) << "Input sample rate: " << wav_reader.sample_rate();
+
+ // Init output files.
+ FILE* vad_probs_file = fopen(OutputVadProbsFile().c_str(), "wb");
+ FILE* features_file = nullptr;
+ const std::string output_feature_file = OutputFeaturesFile();
+ if (!output_feature_file.empty()) {
+ features_file = fopen(output_feature_file.c_str(), "wb");
+ }
+
+ // Init resampling.
+ const size_t frame_size_10ms =
+ rtc::CheckedDivExact(wav_reader.sample_rate(), 100);
+ std::vector<float> samples_10ms;
+ samples_10ms.resize(frame_size_10ms);
+ std::array<float, kFrameSize10ms24kHz> samples_10ms_24kHz;
+ PushSincResampler resampler(frame_size_10ms, kFrameSize10ms24kHz);
+
+ // TODO(bugs.webrtc.org/9076): Init feature extractor and RNN-based VAD.
+
+ // Compute VAD probabilities.
+ while (true) {
+ // Read frame at the input sample rate.
+ const auto read_samples =
+ wav_reader.ReadSamples(frame_size_10ms, samples_10ms.data());
+ if (read_samples < frame_size_10ms) {
+ break; // EOF.
+ }
+ // Resample input.
+ resampler.Resample(samples_10ms.data(), samples_10ms.size(),
+ samples_10ms_24kHz.data(), samples_10ms_24kHz.size());
+
+ // TODO(bugs.webrtc.org/9076): Extract features.
+ float vad_probability;
+ bool is_silence = true;
+
+ // Write features.
+ if (features_file) {
+ const float float_is_silence = is_silence ? 1.f : 0.f;
+ fwrite(&float_is_silence, sizeof(float), 1, features_file);
+ // TODO(bugs.webrtc.org/9076): Write feature vector.
+ }
+
+ // Compute VAD probability.
+ if (is_silence) {
+ vad_probability = 0.f;
+ // TODO(bugs.webrtc.org/9076): Reset VAD.
+ } else {
+ // TODO(bugs.webrtc.org/9076): Compute VAD probability.
+ }
+ RTC_DCHECK_GE(vad_probability, 0.f);
+ RTC_DCHECK_GE(1.f, vad_probability);
+ fwrite(&vad_probability, sizeof(float), 1, vad_probs_file);
+ }
+ // Close output file(s).
+ fclose(vad_probs_file);
+ RTC_LOG(LS_INFO) << "VAD probabilities written to " << FLAG_o;
+ if (features_file) {
+ fclose(features_file);
+ RTC_LOG(LS_INFO) << "features written to " << FLAG_f;
+ }
+
+ return 0;
+}
+
+} // namespace test
+} // namespace webrtc
+
+int main(int argc, char* argv[]) {
+ return webrtc::test::main(argc, argv);
+}
diff --git a/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h b/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h
new file mode 100644
index 0000000..7ae2f95
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/sequence_buffer.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_SEQUENCE_BUFFER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_SEQUENCE_BUFFER_H_
+
+#include <array>
+#include <cstring>
+#include <type_traits>
+
+#include "api/array_view.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+// Linear buffer implementation to (i) push fixed size chunks of sequential data
+// and (ii) view contiguous parts of the buffer. The buffer and the pushed
+// chunks have size S and N respectively. For instance, when S = 2N the first
+// half of the sequence buffer is replaced with its second half, and the new N
+// values are written at the end of the buffer.
+template <typename T, size_t S, size_t N>
+class SequenceBuffer {
+ static_assert(S >= N,
+ "The new chunk size is larger than the sequence buffer size.");
+ static_assert(std::is_arithmetic<T>::value,
+ "Integral or floating point required.");
+
+ public:
+ SequenceBuffer() { buffer_.fill(0); }
+ SequenceBuffer(const SequenceBuffer&) = delete;
+ SequenceBuffer& operator=(const SequenceBuffer&) = delete;
+ ~SequenceBuffer() = default;
+ size_t size() const { return S; }
+ size_t chunks_size() const { return N; }
+ // Sets the sequence buffer values to zero.
+ void Reset() { buffer_.fill(0); }
+ // Returns a view on the whole buffer.
+ rtc::ArrayView<const T, S> GetBufferView() const {
+ return {buffer_.data(), S};
+ }
+ // Returns a view on part of the buffer; the first element starts at the given
+ // offset and the last one is the last one in the buffer.
+ rtc::ArrayView<const T> GetBufferView(int offset) const {
+ RTC_DCHECK_LE(0, offset);
+ RTC_DCHECK_LT(offset, S);
+ return {buffer_.data() + offset, S - offset};
+ }
+ // Returns a view on part of the buffer; the first element starts at the given
+ // offset and the size of the view is |size|.
+ rtc::ArrayView<const T> GetBufferView(int offset, size_t size) const {
+ RTC_DCHECK_LE(0, offset);
+ RTC_DCHECK_LT(offset, S);
+ RTC_DCHECK_LT(0, size);
+ RTC_DCHECK_LE(size, S - offset);
+ return {buffer_.data() + offset, size};
+ }
+ // Shifts left the buffer by N items and add new N items at the end.
+ void Push(rtc::ArrayView<const T, N> new_values) {
+ // Make space for the new values.
+ if (S > N)
+ std::memmove(buffer_.data(), buffer_.data() + N, (S - N) * sizeof(T));
+ // Copy the new values at the end of the buffer.
+ std::memcpy(buffer_.data() + S - N, new_values.data(), N * sizeof(T));
+ }
+
+ private:
+ std::array<T, S> buffer_;
+};
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_SEQUENCE_BUFFER_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/sequence_buffer_unittest.cc b/modules/audio_processing/agc2/rnn_vad/sequence_buffer_unittest.cc
new file mode 100644
index 0000000..7628c17
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/sequence_buffer_unittest.cc
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/sequence_buffer.h"
+
+#include <algorithm>
+
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+namespace {
+
+template <typename T, size_t S, size_t N>
+void TestSequenceBufferPushOp() {
+ SCOPED_TRACE(S);
+ SCOPED_TRACE(N);
+ SequenceBuffer<T, S, N> seq_buf;
+ auto seq_buf_view = seq_buf.GetBufferView();
+ std::array<T, N> chunk;
+ rtc::ArrayView<T, N> chunk_view(chunk.data(), chunk.size());
+
+ // Check that a chunk is fully gone after ceil(S / N) push ops.
+ chunk.fill(1);
+ seq_buf.Push(chunk_view);
+ chunk.fill(0);
+ constexpr size_t required_push_ops = (S % N) ? S / N + 1 : S / N;
+ for (size_t i = 0; i < required_push_ops - 1; ++i) {
+ SCOPED_TRACE(i);
+ seq_buf.Push(chunk_view);
+ // Still in the buffer.
+ const auto* m = std::max_element(seq_buf_view.begin(), seq_buf_view.end());
+ EXPECT_EQ(1, *m);
+ }
+ // Gone after another push.
+ seq_buf.Push(chunk_view);
+ const auto* m = std::max_element(seq_buf_view.begin(), seq_buf_view.end());
+ EXPECT_EQ(0, *m);
+
+ // Check that the last item moves left by N positions after a push op.
+ if (S > N) {
+ // Fill in with non-zero values.
+ for (size_t i = 0; i < N; ++i)
+ chunk[i] = static_cast<T>(i + 1);
+ seq_buf.Push(chunk_view);
+ // With the next Push(), |last| will be moved left by N positions.
+ const T last = chunk[N - 1];
+ for (size_t i = 0; i < N; ++i)
+ chunk[i] = static_cast<T>(last + i + 1);
+ seq_buf.Push(chunk_view);
+ EXPECT_EQ(last, seq_buf_view[S - N - 1]);
+ }
+}
+
+} // namespace
+
+TEST(RnnVadTest, SequenceBufferGetters) {
+ constexpr size_t buffer_size = 8;
+ constexpr size_t chunk_size = 8;
+ SequenceBuffer<int, buffer_size, chunk_size> seq_buf;
+ EXPECT_EQ(buffer_size, seq_buf.size());
+ EXPECT_EQ(chunk_size, seq_buf.chunks_size());
+ // Test view.
+ auto seq_buf_view = seq_buf.GetBufferView();
+ EXPECT_EQ(0, seq_buf_view[0]);
+ EXPECT_EQ(0, seq_buf_view[seq_buf_view.size() - 1]);
+ constexpr std::array<int, chunk_size> chunk = {10, 20, 30, 40,
+ 50, 60, 70, 80};
+ seq_buf.Push({chunk.data(), chunk_size});
+ EXPECT_EQ(10, *seq_buf_view.begin());
+ EXPECT_EQ(80, *(seq_buf_view.end() - 1));
+}
+
+TEST(RnnVadTest, SequenceBufferPushOpsUnsigned) {
+ TestSequenceBufferPushOp<uint8_t, 32, 8>(); // Chunk size: 25%.
+ TestSequenceBufferPushOp<uint8_t, 32, 16>(); // Chunk size: 50%.
+ TestSequenceBufferPushOp<uint8_t, 32, 32>(); // Chunk size: 100%.
+ TestSequenceBufferPushOp<uint8_t, 23, 7>(); // Non-integer ratio.
+}
+
+TEST(RnnVadTest, SequenceBufferPushOpsSigned) {
+ TestSequenceBufferPushOp<int, 32, 8>(); // Chunk size: 25%.
+ TestSequenceBufferPushOp<int, 32, 16>(); // Chunk size: 50%.
+ TestSequenceBufferPushOp<int, 32, 32>(); // Chunk size: 100%.
+ TestSequenceBufferPushOp<int, 23, 7>(); // Non-integer ratio.
+}
+
+TEST(RnnVadTest, SequenceBufferPushOpsFloating) {
+ TestSequenceBufferPushOp<float, 32, 8>(); // Chunk size: 25%.
+ TestSequenceBufferPushOp<float, 32, 16>(); // Chunk size: 50%.
+ TestSequenceBufferPushOp<float, 32, 32>(); // Chunk size: 100%.
+ TestSequenceBufferPushOp<float, 23, 7>(); // Non-integer ratio.
+}
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h b/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h
new file mode 100644
index 0000000..f0282aa
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_SYMMETRIC_MATRIX_BUFFER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_SYMMETRIC_MATRIX_BUFFER_H_
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <utility>
+
+#include "api/array_view.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace rnn_vad {
+
+// Data structure to buffer the results of pair-wise comparisons between items
+// stored in a ring buffer. Every time that the oldest item is replaced in the
+// ring buffer, the new one is compared to the remaining items in the ring
+// buffer. The results of such comparisons need to be buffered and automatically
+// removed when one of the two corresponding items that have been compared is
+// removed from the ring buffer. It is assumed that the comparison is symmetric
+// and that comparing an item with itself is not needed.
+template <typename T, size_t S>
+class SymmetricMatrixBuffer {
+ static_assert(S > 2, "");
+
+ public:
+ SymmetricMatrixBuffer() = default;
+ SymmetricMatrixBuffer(const SymmetricMatrixBuffer&) = delete;
+ SymmetricMatrixBuffer& operator=(const SymmetricMatrixBuffer&) = delete;
+ ~SymmetricMatrixBuffer() = default;
+ // Sets the buffer values to zero.
+ void Reset() {
+ static_assert(std::is_arithmetic<T>::value,
+ "Integral or floating point required.");
+ buf_.fill(0);
+ }
+ // Pushes the results from the comparison between the most recent item and
+ // those that are still in the ring buffer. The first element in |values| must
+ // correspond to the comparison between the most recent item and the second
+ // most recent one in the ring buffer, whereas the last element in |values|
+ // must correspond to the comparison between the most recent item and the
+ // oldest one in the ring buffer.
+ void Push(rtc::ArrayView<T, S - 1> values) {
+ // Move the lower-right sub-matrix of size (S-2) x (S-2) one row up and one
+ // column left.
+ std::memmove(buf_.data(), buf_.data() + S, (buf_.size() - S) * sizeof(T));
+ // Copy new values in the last column in the right order.
+ for (size_t i = 0; i < values.size(); ++i) {
+ const size_t index = (S - 1 - i) * (S - 1) - 1;
+ RTC_DCHECK_LE(static_cast<size_t>(0), index);
+ RTC_DCHECK_LT(index, buf_.size());
+ buf_[index] = values[i];
+ }
+ }
+ // Reads the value that corresponds to comparison of two items in the ring
+ // buffer having delay |delay1| and |delay2|. The two arguments must not be
+ // equal and both must be in {0, ..., S - 1}.
+ T GetValue(size_t delay1, size_t delay2) const {
+ int row = S - 1 - static_cast<int>(delay1);
+ int col = S - 1 - static_cast<int>(delay2);
+ RTC_DCHECK_NE(row, col) << "The diagonal cannot be accessed.";
+ if (row > col)
+ std::swap(row, col); // Swap to access the upper-right triangular part.
+ RTC_DCHECK_LE(0, row);
+ RTC_DCHECK_LT(row, S - 1) << "Not enforcing row < col and row != col.";
+ RTC_DCHECK_LE(1, col) << "Not enforcing row < col and row != col.";
+ RTC_DCHECK_LT(col, S);
+ const int index = row * (S - 1) + (col - 1);
+ RTC_DCHECK_LE(0, index);
+ RTC_DCHECK_LT(index, buf_.size());
+ return buf_[index];
+ }
+
+ private:
+ // Encode an upper-right triangular matrix (excluding its diagonal) using a
+ // square matrix. This allows to move the data in Push() with one single
+ // operation.
+ std::array<T, (S - 1) * (S - 1)> buf_{};
+};
+
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_SYMMETRIC_MATRIX_BUFFER_H_
diff --git a/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer_unittest.cc b/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer_unittest.cc
new file mode 100644
index 0000000..a1b8007
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer_unittest.cc
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/symmetric_matrix_buffer.h"
+
+#include "modules/audio_processing/agc2/rnn_vad/ring_buffer.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+namespace {
+
+template <typename T, size_t S>
+void CheckSymmetry(const SymmetricMatrixBuffer<T, S>* sym_matrix_buf) {
+ for (size_t row = 0; row < S - 1; ++row)
+ for (size_t col = row + 1; col < S; ++col)
+ EXPECT_EQ(sym_matrix_buf->GetValue(row, col),
+ sym_matrix_buf->GetValue(col, row));
+}
+
+using PairType = std::pair<int, int>;
+
+// Checks that the symmetric matrix buffer contains any pair with a value equal
+// to the given one.
+template <size_t S>
+bool CheckPairsWithValueExist(
+ const SymmetricMatrixBuffer<PairType, S>* sym_matrix_buf,
+ const int value) {
+ for (size_t row = 0; row < S - 1; ++row) {
+ for (size_t col = row + 1; col < S; ++col) {
+ auto p = sym_matrix_buf->GetValue(row, col);
+ if (p.first == value || p.second == value)
+ return true;
+ }
+ }
+ return false;
+}
+
+} // namespace
+
+// Test that shows how to combine RingBuffer and SymmetricMatrixBuffer to
+// efficiently compute pair-wise scores. This test verifies that the evolution
+// of a SymmetricMatrixBuffer instance follows that of RingBuffer.
+TEST(RnnVadTest, SymmetricMatrixBufferUseCase) {
+ // Instance a ring buffer which will be fed with a series of integer values.
+ constexpr int kRingBufSize = 10;
+ RingBuffer<int, 1, static_cast<size_t>(kRingBufSize)> ring_buf;
+ // Instance a symmetric matrix buffer for the ring buffer above. It stores
+ // pairs of integers with which this test can easily check that the evolution
+ // of RingBuffer and SymmetricMatrixBuffer match.
+ SymmetricMatrixBuffer<PairType, kRingBufSize> sym_matrix_buf;
+ for (int t = 1; t <= 100; ++t) { // Evolution steps.
+ SCOPED_TRACE(t);
+ const int t_removed = ring_buf.GetArrayView(kRingBufSize - 1)[0];
+ ring_buf.Push({&t, 1});
+ // The head of the ring buffer is |t|.
+ ASSERT_EQ(t, ring_buf.GetArrayView(0)[0]);
+ // Create the comparisons between |t| and the older elements in the ring
+ // buffer.
+ std::array<PairType, kRingBufSize - 1> new_comparions;
+ for (int i = 0; i < kRingBufSize - 1; ++i) {
+ // Start comparing |t| to the second newest element in the ring buffer.
+ const int delay = i + 1;
+ const auto t_prev = ring_buf.GetArrayView(delay)[0];
+ ASSERT_EQ(std::max(0, t - delay), t_prev);
+ // Compare the last element |t| with |t_prev|.
+ new_comparions[i].first = t_prev;
+ new_comparions[i].second = t;
+ }
+ // Push the new comparisons in the symmetric matrix buffer.
+ sym_matrix_buf.Push({new_comparions.data(), new_comparions.size()});
+ // Tests.
+ CheckSymmetry(&sym_matrix_buf);
+ // Check that the pairs resulting from the content in the ring buffer are
+ // in the right position.
+ for (size_t delay1 = 0; delay1 < kRingBufSize - 1; ++delay1) {
+ for (size_t delay2 = delay1 + 1; delay2 < kRingBufSize; ++delay2) {
+ const auto t1 = ring_buf.GetArrayView(delay1)[0];
+ const auto t2 = ring_buf.GetArrayView(delay2)[0];
+ ASSERT_LE(t2, t1);
+ const auto p = sym_matrix_buf.GetValue(delay1, delay2);
+ EXPECT_EQ(p.first, t2);
+ EXPECT_EQ(p.second, t1);
+ }
+ }
+ // Check that every older element in the ring buffer still has a
+ // corresponding pair in the symmetric matrix buffer.
+ for (size_t delay = 1; delay < kRingBufSize; ++delay) {
+ const auto t_prev = ring_buf.GetArrayView(delay)[0];
+ EXPECT_TRUE(CheckPairsWithValueExist(&sym_matrix_buf, t_prev));
+ }
+ // Check that the element removed from the ring buffer has no corresponding
+ // pairs in the symmetric matrix buffer.
+ if (t > kRingBufSize - 1) {
+ EXPECT_FALSE(CheckPairsWithValueExist(&sym_matrix_buf, t_removed));
+ }
+ }
+}
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/test_utils.cc b/modules/audio_processing/agc2/rnn_vad/test_utils.cc
new file mode 100644
index 0000000..c6cf21e
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/test_utils.cc
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/rnn_vad/test_utils.h"
+
+#include "rtc_base/checks.h"
+#include "rtc_base/ptr_util.h"
+#include "test/gtest.h"
+#include "test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+namespace {
+
+using ReaderPairType =
+ std::pair<std::unique_ptr<BinaryFileReader<float>>, const size_t>;
+
+} // namespace
+
+using webrtc::test::ResourcePath;
+
+void ExpectNearAbsolute(rtc::ArrayView<const float> expected,
+ rtc::ArrayView<const float> computed,
+ float tolerance) {
+ ASSERT_EQ(expected.size(), computed.size());
+ for (size_t i = 0; i < expected.size(); ++i) {
+ SCOPED_TRACE(i);
+ EXPECT_NEAR(expected[i], computed[i], tolerance);
+ }
+}
+
+ReaderPairType CreatePitchBuffer24kHzReader() {
+ auto ptr = rtc::MakeUnique<BinaryFileReader<float>>(
+ ResourcePath("audio_processing/agc2/rnn_vad/pitch_buf_24k", "dat"), 864);
+ return {std::move(ptr),
+ rtc::CheckedDivExact(ptr->data_length(), static_cast<size_t>(864))};
+}
+
+ReaderPairType CreateLpResidualAndPitchPeriodGainReader() {
+ constexpr size_t num_lp_residual_coeffs = 864;
+ auto ptr = rtc::MakeUnique<BinaryFileReader<float>>(
+ ResourcePath("audio_processing/agc2/rnn_vad/pitch_lp_res", "dat"),
+ num_lp_residual_coeffs);
+ return {std::move(ptr),
+ rtc::CheckedDivExact(ptr->data_length(), 2 + num_lp_residual_coeffs)};
+}
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/rnn_vad/test_utils.h b/modules/audio_processing/agc2/rnn_vad/test_utils.h
new file mode 100644
index 0000000..3f580ab
--- /dev/null
+++ b/modules/audio_processing/agc2/rnn_vad/test_utils.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_TEST_UTILS_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_TEST_UTILS_H_
+
+#include <algorithm>
+#include <fstream>
+#include <limits>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "api/array_view.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace rnn_vad {
+namespace test {
+
+constexpr float kFloatMin = std::numeric_limits<float>::min();
+
+// Fail for every pair from two equally sized rtc::ArrayView<float> views such
+// that their absolute error is above a given threshold.
+void ExpectNearAbsolute(rtc::ArrayView<const float> expected,
+ rtc::ArrayView<const float> computed,
+ float tolerance);
+
+// Reader for binary files consisting of an arbitrary long sequence of elements
+// having type T. It is possible to read and cast to another type D at once.
+template <typename T, typename D = T>
+class BinaryFileReader {
+ public:
+ explicit BinaryFileReader(const std::string& file_path, size_t chunk_size = 1)
+ : is_(file_path, std::ios::binary | std::ios::ate),
+ data_length_(is_.tellg() / sizeof(T)),
+ chunk_size_(chunk_size) {
+ RTC_CHECK_LT(0, chunk_size_);
+ RTC_CHECK(is_);
+ SeekBeginning();
+ buf_.resize(chunk_size_);
+ }
+ BinaryFileReader(const BinaryFileReader&) = delete;
+ BinaryFileReader& operator=(const BinaryFileReader&) = delete;
+ ~BinaryFileReader() = default;
+ size_t data_length() const { return data_length_; }
+ bool ReadValue(D* dst) {
+ if (std::is_same<T, D>::value) {
+ is_.read(reinterpret_cast<char*>(dst), sizeof(T));
+ } else {
+ T v;
+ is_.read(reinterpret_cast<char*>(&v), sizeof(T));
+ *dst = static_cast<D>(v);
+ }
+ return is_.gcount() == sizeof(T);
+ }
+ bool ReadChunk(rtc::ArrayView<D> dst) {
+ RTC_DCHECK_EQ(chunk_size_, dst.size());
+ const std::streamsize bytes_to_read = chunk_size_ * sizeof(T);
+ if (std::is_same<T, D>::value) {
+ is_.read(reinterpret_cast<char*>(dst.data()), bytes_to_read);
+ } else {
+ is_.read(reinterpret_cast<char*>(buf_.data()), bytes_to_read);
+ std::transform(buf_.begin(), buf_.end(), dst.begin(),
+ [](const T& v) -> D { return static_cast<D>(v); });
+ }
+ return is_.gcount() == bytes_to_read;
+ }
+ void SeekForward(size_t items) { is_.seekg(items * sizeof(T), is_.cur); }
+ void SeekBeginning() { is_.seekg(0, is_.beg); }
+
+ private:
+ std::ifstream is_;
+ const size_t data_length_;
+ const size_t chunk_size_;
+ std::vector<T> buf_;
+};
+
+// Factories for resource file readers; the functions below return a pair where
+// the first item is a reader unique pointer and the second the number of chunks
+// that can be read from the file.
+
+// Creates a reader for the pitch buffer content at 24 kHz.
+std::pair<std::unique_ptr<BinaryFileReader<float>>, const size_t>
+CreatePitchBuffer24kHzReader();
+// Creates a reader for the the LP residual coefficients and the pitch period
+// and gain values.
+std::pair<std::unique_ptr<BinaryFileReader<float>>, const size_t>
+CreateLpResidualAndPitchPeriodGainReader();
+
+} // namespace test
+} // namespace rnn_vad
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_RNN_VAD_TEST_UTILS_H_
diff --git a/modules/audio_processing/agc2/saturation_protector.cc b/modules/audio_processing/agc2/saturation_protector.cc
new file mode 100644
index 0000000..216e1b6
--- /dev/null
+++ b/modules/audio_processing/agc2/saturation_protector.cc
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/saturation_protector.h"
+
+#include <algorithm>
+
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/numerics/safe_minmax.h"
+
+namespace webrtc {
+
+namespace {
+void ShiftBuffer(std::array<float, kPeakEnveloperBufferSize>* buffer_) {
+ // Move everything one element back.
+ std::copy(buffer_->begin() + 1, buffer_->end(), buffer_->begin());
+}
+} // namespace
+
+SaturationProtector::PeakEnveloper::PeakEnveloper() = default;
+
+void SaturationProtector::PeakEnveloper::Process(float frame_peak_dbfs) {
+ // Update the delayed buffer and the current superframe peak.
+ current_superframe_peak_dbfs_ =
+ std::max(current_superframe_peak_dbfs_, frame_peak_dbfs);
+ speech_time_in_estimate_ms_ += kFrameDurationMs;
+ if (speech_time_in_estimate_ms_ > kPeakEnveloperSuperFrameLengthMs) {
+ speech_time_in_estimate_ms_ = 0;
+ const bool buffer_full = elements_in_buffer_ == kPeakEnveloperBufferSize;
+ if (buffer_full) {
+ ShiftBuffer(&peak_delay_buffer_);
+ *peak_delay_buffer_.rbegin() = current_superframe_peak_dbfs_;
+ } else {
+ peak_delay_buffer_[elements_in_buffer_] = current_superframe_peak_dbfs_;
+ elements_in_buffer_++;
+ }
+ current_superframe_peak_dbfs_ = -90.f;
+ }
+}
+
+float SaturationProtector::PeakEnveloper::Query() const {
+ float result;
+ if (elements_in_buffer_ > 0) {
+ result = peak_delay_buffer_[0];
+ } else {
+ result = current_superframe_peak_dbfs_;
+ }
+ return result;
+}
+
+SaturationProtector::SaturationProtector(ApmDataDumper* apm_data_dumper)
+ : apm_data_dumper_(apm_data_dumper) {}
+
+void SaturationProtector::UpdateMargin(
+ const VadWithLevel::LevelAndProbability& vad_data,
+ float last_speech_level_estimate) {
+ peak_enveloper_.Process(vad_data.speech_peak_dbfs);
+ const float delayed_peak_dbfs = peak_enveloper_.Query();
+ const float difference_db = delayed_peak_dbfs - last_speech_level_estimate;
+
+ if (last_margin_ < difference_db) {
+ last_margin_ = last_margin_ * kSaturationProtectorAttackConstant +
+ difference_db * (1.f - kSaturationProtectorAttackConstant);
+ } else {
+ last_margin_ = last_margin_ * kSaturationProtectorDecayConstant +
+ difference_db * (1.f - kSaturationProtectorDecayConstant);
+ }
+
+ last_margin_ = rtc::SafeClamp<float>(last_margin_, 12.f, 25.f);
+}
+
+float SaturationProtector::LastMargin() const {
+ return last_margin_;
+}
+
+void SaturationProtector::DebugDumpEstimate() const {
+ apm_data_dumper_->DumpRaw(
+ "agc2_adaptive_saturation_protector_delayed_peak_dbfs",
+ peak_enveloper_.Query());
+ apm_data_dumper_->DumpRaw("agc2_adaptive_saturation_margin_db", last_margin_);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/saturation_protector.h b/modules/audio_processing/agc2/saturation_protector.h
new file mode 100644
index 0000000..d330c15
--- /dev/null
+++ b/modules/audio_processing/agc2/saturation_protector.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_H_
+
+#include <array>
+
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/vad/vad_with_level.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+
+class SaturationProtector {
+ public:
+ explicit SaturationProtector(ApmDataDumper* apm_data_dumper);
+
+ // Update and return margin estimate. This method should be called
+ // whenever a frame is reliably classified as 'speech'.
+ //
+ // Returned value is in DB scale.
+ void UpdateMargin(const VadWithLevel::LevelAndProbability& vad_data,
+ float last_speech_level_estimate_dbfs);
+
+ // Returns latest computed margin. Used in cases when speech is not
+ // detected.
+ float LastMargin() const;
+
+ void DebugDumpEstimate() const;
+
+ private:
+ // Computes a delayed envelope of peaks.
+ class PeakEnveloper {
+ public:
+ PeakEnveloper();
+ void Process(float frame_peak_dbfs);
+
+ float Query() const;
+
+ private:
+ size_t speech_time_in_estimate_ms_ = 0;
+ float current_superframe_peak_dbfs_ = -90.f;
+ size_t elements_in_buffer_ = 0;
+ std::array<float, kPeakEnveloperBufferSize> peak_delay_buffer_ = {};
+ };
+
+ ApmDataDumper* apm_data_dumper_;
+
+ float last_margin_ = kInitialSaturationMarginDb;
+ PeakEnveloper peak_enveloper_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_H_
diff --git a/modules/audio_processing/agc2/saturation_protector_unittest.cc b/modules/audio_processing/agc2/saturation_protector_unittest.cc
new file mode 100644
index 0000000..88da2a2
--- /dev/null
+++ b/modules/audio_processing/agc2/saturation_protector_unittest.cc
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/saturation_protector.h"
+
+#include <algorithm>
+
+#include "modules/audio_processing/agc2/agc2_common.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/gunit.h"
+
+namespace webrtc {
+namespace {
+float RunOnConstantLevel(int num_iterations,
+ VadWithLevel::LevelAndProbability vad_data,
+ float estimated_level_dbfs,
+ SaturationProtector* saturation_protector) {
+ float last_margin = saturation_protector->LastMargin();
+ float max_difference = 0.f;
+ for (int i = 0; i < num_iterations; ++i) {
+ saturation_protector->UpdateMargin(vad_data, estimated_level_dbfs);
+ const float new_margin = saturation_protector->LastMargin();
+ max_difference =
+ std::max(max_difference, std::abs(new_margin - last_margin));
+ last_margin = new_margin;
+ }
+ return max_difference;
+}
+} // namespace
+
+TEST(AutomaticGainController2SaturationProtector, ProtectorShouldNotCrash) {
+ ApmDataDumper apm_data_dumper(0);
+ SaturationProtector saturation_protector(&apm_data_dumper);
+ VadWithLevel::LevelAndProbability vad_data(1.f, -20.f, -10.f);
+
+ saturation_protector.UpdateMargin(vad_data, -20.f);
+ static_cast<void>(saturation_protector.LastMargin());
+ saturation_protector.DebugDumpEstimate();
+}
+
+// Check that the estimate converges to the ratio between peaks and
+// level estimator values after a while.
+TEST(AutomaticGainController2SaturationProtector,
+ ProtectorEstimatesCrestRatio) {
+ ApmDataDumper apm_data_dumper(0);
+ SaturationProtector saturation_protector(&apm_data_dumper);
+
+ constexpr float kPeakLevel = -20.f;
+ constexpr float kCrestFactor = kInitialSaturationMarginDb + 1.f;
+ constexpr float kSpeechLevel = kPeakLevel - kCrestFactor;
+ const float kMaxDifference =
+ 0.5 * std::abs(kInitialSaturationMarginDb - kCrestFactor);
+
+ static_cast<void>(RunOnConstantLevel(
+ 2000, VadWithLevel::LevelAndProbability(1.f, -90.f, kPeakLevel),
+ kSpeechLevel, &saturation_protector));
+
+ EXPECT_NEAR(saturation_protector.LastMargin(), kCrestFactor, kMaxDifference);
+}
+
+TEST(AutomaticGainController2SaturationProtector, ProtectorChangesSlowly) {
+ ApmDataDumper apm_data_dumper(0);
+ SaturationProtector saturation_protector(&apm_data_dumper);
+
+ constexpr float kPeakLevel = -20.f;
+ constexpr float kCrestFactor = kInitialSaturationMarginDb - 5.f;
+ constexpr float kOtherCrestFactor = kInitialSaturationMarginDb;
+ constexpr float kSpeechLevel = kPeakLevel - kCrestFactor;
+ constexpr float kOtherSpeechLevel = kPeakLevel - kOtherCrestFactor;
+
+ constexpr int kNumIterations = 1000;
+ float max_difference = RunOnConstantLevel(
+ kNumIterations, VadWithLevel::LevelAndProbability(1.f, -90.f, kPeakLevel),
+ kSpeechLevel, &saturation_protector);
+
+ max_difference =
+ std::max(RunOnConstantLevel(
+ kNumIterations,
+ VadWithLevel::LevelAndProbability(1.f, -90.f, kPeakLevel),
+ kOtherSpeechLevel, &saturation_protector),
+ max_difference);
+
+ constexpr float kMaxChangeSpeedDbPerSecond = 0.5; // 1 db / 2 seconds.
+
+ EXPECT_LE(max_difference,
+ kMaxChangeSpeedDbPerSecond / 1000 * kFrameDurationMs);
+}
+
+TEST(AutomaticGainController2SaturationProtector,
+ ProtectorAdaptsToDelayedChanges) {
+ ApmDataDumper apm_data_dumper(0);
+ SaturationProtector saturation_protector(&apm_data_dumper);
+
+ constexpr int kDelayIterations = kFullBufferSizeMs / kFrameDurationMs;
+ constexpr float kInitialSpeechLevelDbfs = -30;
+ constexpr float kLaterSpeechLevelDbfs = -15;
+
+ // First run on initial level.
+ float max_difference = RunOnConstantLevel(
+ kDelayIterations,
+ VadWithLevel::LevelAndProbability(
+ 1.f, -90.f, kInitialSpeechLevelDbfs + kInitialSaturationMarginDb),
+ kInitialSpeechLevelDbfs, &saturation_protector);
+
+ // Then peak changes, but not RMS.
+ max_difference = std::max(
+ RunOnConstantLevel(
+ kDelayIterations,
+ VadWithLevel::LevelAndProbability(
+ 1.f, -90.f, kLaterSpeechLevelDbfs + kInitialSaturationMarginDb),
+ kInitialSpeechLevelDbfs, &saturation_protector),
+ max_difference);
+
+ // Then both change.
+ max_difference = std::max(
+ RunOnConstantLevel(
+ kDelayIterations,
+ VadWithLevel::LevelAndProbability(
+ 1.f, -90.f, kLaterSpeechLevelDbfs + kInitialSaturationMarginDb),
+ kLaterSpeechLevelDbfs, &saturation_protector),
+ max_difference);
+
+ const float total_difference =
+ std::abs(saturation_protector.LastMargin() - kInitialSaturationMarginDb);
+
+ EXPECT_LE(total_difference, 0.05f);
+ EXPECT_LE(max_difference, 0.01f);
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/signal_classifier.cc b/modules/audio_processing/agc2/signal_classifier.cc
new file mode 100644
index 0000000..0ec3414
--- /dev/null
+++ b/modules/audio_processing/agc2/signal_classifier.cc
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/signal_classifier.h"
+
+#include <algorithm>
+#include <numeric>
+#include <vector>
+
+#include "api/array_view.h"
+#include "modules/audio_processing/agc2/down_sampler.h"
+#include "modules/audio_processing/agc2/noise_spectrum_estimator.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+namespace {
+
+void RemoveDcLevel(rtc::ArrayView<float> x) {
+ RTC_DCHECK_LT(0, x.size());
+ float mean = std::accumulate(x.data(), x.data() + x.size(), 0.f);
+ mean /= x.size();
+
+ for (float& v : x) {
+ v -= mean;
+ }
+}
+
+void PowerSpectrum(const OouraFft* ooura_fft,
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<float> spectrum) {
+ RTC_DCHECK_EQ(65, spectrum.size());
+ RTC_DCHECK_EQ(128, x.size());
+ float X[128];
+ std::copy(x.data(), x.data() + x.size(), X);
+ ooura_fft->Fft(X);
+
+ float* X_p = X;
+ RTC_DCHECK_EQ(X_p, &X[0]);
+ spectrum[0] = (*X_p) * (*X_p);
+ ++X_p;
+ RTC_DCHECK_EQ(X_p, &X[1]);
+ spectrum[64] = (*X_p) * (*X_p);
+ for (int k = 1; k < 64; ++k) {
+ ++X_p;
+ RTC_DCHECK_EQ(X_p, &X[2 * k]);
+ spectrum[k] = (*X_p) * (*X_p);
+ ++X_p;
+ RTC_DCHECK_EQ(X_p, &X[2 * k + 1]);
+ spectrum[k] += (*X_p) * (*X_p);
+ }
+}
+
+webrtc::SignalClassifier::SignalType ClassifySignal(
+ rtc::ArrayView<const float> signal_spectrum,
+ rtc::ArrayView<const float> noise_spectrum,
+ ApmDataDumper* data_dumper) {
+ int num_stationary_bands = 0;
+ int num_highly_nonstationary_bands = 0;
+
+ // Detect stationary and highly nonstationary bands.
+ for (size_t k = 1; k < 40; k++) {
+ if (signal_spectrum[k] < 3 * noise_spectrum[k] &&
+ signal_spectrum[k] * 3 > noise_spectrum[k]) {
+ ++num_stationary_bands;
+ } else if (signal_spectrum[k] > 9 * noise_spectrum[k]) {
+ ++num_highly_nonstationary_bands;
+ }
+ }
+
+ data_dumper->DumpRaw("lc_num_stationary_bands", 1, &num_stationary_bands);
+ data_dumper->DumpRaw("lc_num_highly_nonstationary_bands", 1,
+ &num_highly_nonstationary_bands);
+
+ // Use the detected number of bands to classify the overall signal
+ // stationarity.
+ if (num_stationary_bands > 15) {
+ return SignalClassifier::SignalType::kStationary;
+ } else {
+ return SignalClassifier::SignalType::kNonStationary;
+ }
+}
+
+} // namespace
+
+SignalClassifier::FrameExtender::FrameExtender(size_t frame_size,
+ size_t extended_frame_size)
+ : x_old_(extended_frame_size - frame_size, 0.f) {}
+
+SignalClassifier::FrameExtender::~FrameExtender() = default;
+
+void SignalClassifier::FrameExtender::ExtendFrame(
+ rtc::ArrayView<const float> x,
+ rtc::ArrayView<float> x_extended) {
+ RTC_DCHECK_EQ(x_old_.size() + x.size(), x_extended.size());
+ std::copy(x_old_.data(), x_old_.data() + x_old_.size(), x_extended.data());
+ std::copy(x.data(), x.data() + x.size(), x_extended.data() + x_old_.size());
+ std::copy(x_extended.data() + x_extended.size() - x_old_.size(),
+ x_extended.data() + x_extended.size(), x_old_.data());
+}
+
+SignalClassifier::SignalClassifier(ApmDataDumper* data_dumper)
+ : data_dumper_(data_dumper),
+ down_sampler_(data_dumper_),
+ noise_spectrum_estimator_(data_dumper_) {
+ Initialize(48000);
+}
+SignalClassifier::~SignalClassifier() {}
+
+void SignalClassifier::Initialize(int sample_rate_hz) {
+ down_sampler_.Initialize(sample_rate_hz);
+ noise_spectrum_estimator_.Initialize();
+ frame_extender_.reset(new FrameExtender(80, 128));
+ sample_rate_hz_ = sample_rate_hz;
+ initialization_frames_left_ = 2;
+ consistent_classification_counter_ = 3;
+ last_signal_type_ = SignalClassifier::SignalType::kNonStationary;
+}
+
+SignalClassifier::SignalType SignalClassifier::Analyze(
+ rtc::ArrayView<const float> signal) {
+ RTC_DCHECK_EQ(signal.size(), sample_rate_hz_ / 100);
+
+ // Compute the signal power spectrum.
+ float downsampled_frame[80];
+ down_sampler_.DownSample(signal, downsampled_frame);
+ float extended_frame[128];
+ frame_extender_->ExtendFrame(downsampled_frame, extended_frame);
+ RemoveDcLevel(extended_frame);
+ float signal_spectrum[65];
+ PowerSpectrum(&ooura_fft_, extended_frame, signal_spectrum);
+
+ // Classify the signal based on the estimate of the noise spectrum and the
+ // signal spectrum estimate.
+ const SignalType signal_type = ClassifySignal(
+ signal_spectrum, noise_spectrum_estimator_.GetNoiseSpectrum(),
+ data_dumper_);
+
+ // Update the noise spectrum based on the signal spectrum.
+ noise_spectrum_estimator_.Update(signal_spectrum,
+ initialization_frames_left_ > 0);
+
+ // Update the number of frames until a reliable signal spectrum is achieved.
+ initialization_frames_left_ = std::max(0, initialization_frames_left_ - 1);
+
+ if (last_signal_type_ == signal_type) {
+ consistent_classification_counter_ =
+ std::max(0, consistent_classification_counter_ - 1);
+ } else {
+ last_signal_type_ = signal_type;
+ consistent_classification_counter_ = 3;
+ }
+
+ if (consistent_classification_counter_ > 0) {
+ return SignalClassifier::SignalType::kNonStationary;
+ }
+ return signal_type;
+}
+
+} // namespace webrtc
diff --git a/modules/audio_processing/agc2/signal_classifier.h b/modules/audio_processing/agc2/signal_classifier.h
new file mode 100644
index 0000000..23fe315
--- /dev/null
+++ b/modules/audio_processing/agc2/signal_classifier.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_AGC2_SIGNAL_CLASSIFIER_H_
+#define MODULES_AUDIO_PROCESSING_AGC2_SIGNAL_CLASSIFIER_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/array_view.h"
+#include "modules/audio_processing/agc2/down_sampler.h"
+#include "modules/audio_processing/agc2/noise_spectrum_estimator.h"
+#include "modules/audio_processing/utility/ooura_fft.h"
+#include "rtc_base/constructormagic.h"
+
+namespace webrtc {
+
+class ApmDataDumper;
+class AudioBuffer;
+
+class SignalClassifier {
+ public:
+ enum class SignalType { kNonStationary, kStationary };
+
+ explicit SignalClassifier(ApmDataDumper* data_dumper);
+ ~SignalClassifier();
+
+ void Initialize(int sample_rate_hz);
+ SignalType Analyze(rtc::ArrayView<const float> signal);
+
+ private:
+ class FrameExtender {
+ public:
+ FrameExtender(size_t frame_size, size_t extended_frame_size);
+ ~FrameExtender();
+
+ void ExtendFrame(rtc::ArrayView<const float> x,
+ rtc::ArrayView<float> x_extended);
+
+ private:
+ std::vector<float> x_old_;
+
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameExtender);
+ };
+
+ ApmDataDumper* const data_dumper_;
+ DownSampler down_sampler_;
+ std::unique_ptr<FrameExtender> frame_extender_;
+ NoiseSpectrumEstimator noise_spectrum_estimator_;
+ int sample_rate_hz_;
+ int initialization_frames_left_;
+ int consistent_classification_counter_;
+ SignalType last_signal_type_;
+ const OouraFft ooura_fft_;
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(SignalClassifier);
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_AGC2_SIGNAL_CLASSIFIER_H_
diff --git a/modules/audio_processing/agc2/signal_classifier_unittest.cc b/modules/audio_processing/agc2/signal_classifier_unittest.cc
new file mode 100644
index 0000000..62171b3
--- /dev/null
+++ b/modules/audio_processing/agc2/signal_classifier_unittest.cc
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/audio_processing/agc2/signal_classifier.h"
+
+#include <array>
+#include <functional>
+#include <limits>
+
+#include "modules/audio_processing/agc2/agc2_testing_common.h"
+#include "modules/audio_processing/logging/apm_data_dumper.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/random.h"
+
+namespace webrtc {
+
+namespace {
+Random rand_gen(42);
+ApmDataDumper data_dumper(0);
+constexpr int kNumIterations = 100;
+
+// Runs the signal classifier on audio generated by 'sample_generator'
+// for kNumIterations. Returns the number of frames classified as noise.
+int RunClassifier(std::function<float()> sample_generator, int rate) {
+ SignalClassifier classifier(&data_dumper);
+ std::array<float, 480> signal;
+ classifier.Initialize(rate);
+ const size_t samples_per_channel = rtc::CheckedDivExact(rate, 100);
+ int number_of_noise_frames = 0;
+ for (int i = 0; i < kNumIterations; ++i) {
+ for (size_t j = 0; j < samples_per_channel; ++j) {
+ signal[j] = sample_generator();
+ }
+ number_of_noise_frames +=
+ classifier.Analyze({&signal[0], samples_per_channel}) ==
+ SignalClassifier::SignalType::kStationary;
+ }
+ return number_of_noise_frames;
+}
+
+float WhiteNoiseGenerator() {
+ return static_cast<float>(rand_gen.Rand(std::numeric_limits<int16_t>::min(),
+ std::numeric_limits<int16_t>::max()));
+}
+} // namespace
+
+// White random noise is stationary, but does not trigger the detector
+// every frame due to the randomness.
+TEST(AutomaticGainController2SignalClassifier, WhiteNoise) {
+ for (const auto rate : {8000, 16000, 32000, 48000}) {
+ const int number_of_noise_frames = RunClassifier(WhiteNoiseGenerator, rate);
+ EXPECT_GT(number_of_noise_frames, kNumIterations / 2);
+ }
+}
+
+// Sine curves are (very) stationary. They trigger the detector all
+// the time. Except for a few initial frames.
+TEST(AutomaticGainController2SignalClassifier, SineTone) {
+ for (const auto rate : {8000, 16000, 32000, 48000}) {
+ test::SineGenerator gen(600.f, rate);
+ const int number_of_noise_frames = RunClassifier(gen, rate);
+ EXPECT_GE(number_of_noise_frames, kNumIterations - 5);
+ }
+}
+
+// Pulses are transient if they are far enough apart. They shouldn't
+// trigger the noise detector.
+TEST(AutomaticGainController2SignalClassifier, PulseTone) {
+ for (const auto rate : {8000, 16000, 32000, 48000}) {
+ test::PulseGenerator gen(30.f, rate);
+ const int number_of_noise_frames = RunClassifier(gen, rate);
+ EXPECT_EQ(number_of_noise_frames, 0);
+ }
+}
+} // namespace webrtc
diff --git a/modules/audio_processing/audio_buffer.h b/modules/audio_processing/audio_buffer.h
index 8451bde..508f96f 100644
--- a/modules/audio_processing/audio_buffer.h
+++ b/modules/audio_processing/audio_buffer.h
@@ -14,10 +14,10 @@
#include <memory>
#include <vector>
+#include "api/audio/audio_frame.h"
#include "common_audio/channel_buffer.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/splitting_filter.h"
-#include "modules/include/module_common_types.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc
index 554dead..8d1ccee 100644
--- a/modules/audio_processing/audio_processing_impl.cc
+++ b/modules/audio_processing/audio_processing_impl.cc
@@ -20,6 +20,7 @@
#include "common_audio/signal_processing/include/signal_processing_library.h"
#include "modules/audio_processing/aec/aec_core.h"
#include "modules/audio_processing/agc/agc_manager_direct.h"
+#include "modules/audio_processing/agc2/gain_applier.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/beamformer/nonlinear_beamformer.h"
#include "modules/audio_processing/common.h"
@@ -43,9 +44,7 @@
#include "modules/audio_processing/residual_echo_detector.h"
#include "modules/audio_processing/transient/transient_suppressor.h"
#include "modules/audio_processing/voice_detection_impl.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/atomicops.h"
-#include "system_wrappers/include/file_wrapper.h"
#include "system_wrappers/include/metrics.h"
// Check to verify that the define for the intelligibility enhancer is properly
@@ -187,6 +186,7 @@
bool beamformer_enabled,
bool adaptive_gain_controller_enabled,
bool gain_controller2_enabled,
+ bool pre_amplifier_enabled,
bool echo_controller_enabled,
bool voice_activity_detector_enabled,
bool level_estimator_enabled,
@@ -206,6 +206,7 @@
(adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
changed |=
(gain_controller2_enabled != gain_controller2_enabled_);
+ changed |= (pre_amplifier_enabled_ != pre_amplifier_enabled);
changed |= (echo_controller_enabled != echo_controller_enabled_);
changed |= (level_estimator_enabled != level_estimator_enabled_);
changed |=
@@ -221,6 +222,7 @@
beamformer_enabled_ = beamformer_enabled;
adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
gain_controller2_enabled_ = gain_controller2_enabled;
+ pre_amplifier_enabled_ = pre_amplifier_enabled;
echo_controller_enabled_ = echo_controller_enabled;
level_estimator_enabled_ = level_estimator_enabled;
voice_activity_detector_enabled_ = voice_activity_detector_enabled;
@@ -252,7 +254,8 @@
bool AudioProcessingImpl::ApmSubmoduleStates::CaptureFullBandProcessingActive()
const {
- return gain_controller2_enabled_ || capture_post_processor_enabled_;
+ return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
+ pre_amplifier_enabled_;
}
bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive()
@@ -313,6 +316,7 @@
std::unique_ptr<EchoControl> echo_controller;
std::unique_ptr<CustomProcessing> capture_post_processor;
std::unique_ptr<CustomProcessing> render_pre_processor;
+ std::unique_ptr<GainApplier> pre_amplifier;
};
AudioProcessingBuilder::AudioProcessingBuilder() = default;
@@ -380,6 +384,8 @@
NonlinearBeamformer* beamformer)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
+ runtime_settings_(100),
+ runtime_settings_enqueuer_(&runtime_settings_),
high_pass_filter_impl_(new HighPassFilterImpl(this)),
echo_control_factory_(std::move(echo_control_factory)),
submodule_states_(!!capture_post_processor, !!render_pre_processor),
@@ -713,9 +719,12 @@
config_.gain_controller2 = AudioProcessing::Config::GainController2();
}
InitializeGainController2();
+ InitializePreAmplifier();
private_submodules_->gain_controller2->ApplyConfig(config_.gain_controller2);
RTC_LOG(LS_INFO) << "Gain Controller 2 activated: "
<< config_.gain_controller2.enabled;
+ RTC_LOG(LS_INFO) << "Pre-amplifier activated: "
+ << config_.pre_amplifier.enabled;
}
void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {
@@ -796,6 +805,32 @@
}
}
+void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
+ RTC_DCHECK(setting.type() != RuntimeSetting::Type::kNotSpecified);
+ runtime_settings_enqueuer_.Enqueue(setting);
+}
+
+AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
+ SwapQueue<RuntimeSetting>* runtime_settings)
+ : runtime_settings_(*runtime_settings) {
+ RTC_DCHECK(runtime_settings);
+}
+
+AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
+ default;
+
+void AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
+ RuntimeSetting setting) {
+ size_t remaining_attempts = 10;
+ while (!runtime_settings_.Insert(&setting) && remaining_attempts-- > 0) {
+ RuntimeSetting setting_to_discard;
+ if (runtime_settings_.Remove(&setting_to_discard))
+ RTC_LOG(LS_ERROR)
+ << "The runtime settings queue is full. Oldest setting discarded.";
+ }
+ if (remaining_attempts == 0)
+ RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
+}
int AudioProcessingImpl::ProcessStream(const float* const* src,
size_t samples_per_channel,
@@ -878,6 +913,25 @@
return kNoError;
}
+void AudioProcessingImpl::HandleRuntimeSettings() {
+ RuntimeSetting setting;
+ while (runtime_settings_.Remove(&setting)) {
+ switch (setting.type()) {
+ case RuntimeSetting::Type::kCapturePreGain:
+ if (config_.pre_amplifier.enabled) {
+ float value;
+ setting.GetFloat(&value);
+ private_submodules_->pre_amplifier->SetGainFactor(value);
+ }
+ // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
+ break;
+ case RuntimeSetting::Type::kNotSpecified:
+ RTC_NOTREACHED();
+ break;
+ }
+ }
+}
+
void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
@@ -1132,6 +1186,8 @@
}
int AudioProcessingImpl::ProcessCaptureStreamLocked() {
+ HandleRuntimeSettings();
+
// Ensure that not both the AEC and AECM are active at the same time.
// TODO(peah): Simplify once the public API Enable functions for these
// are moved to APM.
@@ -1142,6 +1198,12 @@
AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
+ if (private_submodules_->pre_amplifier) {
+ private_submodules_->pre_amplifier->ApplyGain(AudioFrameView<float>(
+ capture_buffer->channels_f(), capture_buffer->num_channels(),
+ capture_buffer->num_frames()));
+ }
+
capture_input_rms_.Analyze(rtc::ArrayView<const int16_t>(
capture_buffer->channels_const()[0],
capture_nonlocked_.capture_processing_format.num_frames()));
@@ -1209,6 +1271,11 @@
if (private_submodules_->echo_controller) {
data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
+ if (was_stream_delay_set()) {
+ private_submodules_->echo_controller->SetAudioBufferDelay(
+ stream_delay_ms());
+ }
+
private_submodules_->echo_controller->ProcessCapture(
capture_buffer, capture_.echo_path_gain_change);
} else {
@@ -1375,8 +1442,8 @@
processing_config.reverse_output_stream() = output_config;
RETURN_ON_ERR(MaybeInitializeRender(processing_config));
- assert(input_config.num_frames() ==
- formats_.api_format.reverse_input_stream().num_frames());
+ RTC_DCHECK_EQ(input_config.num_frames(),
+ formats_.api_format.reverse_input_stream().num_frames());
if (aec_dump_) {
const size_t channel_size =
@@ -1724,7 +1791,7 @@
capture_nonlocked_.intelligibility_enabled,
capture_nonlocked_.beamformer_enabled,
public_submodules_->gain_control->is_enabled(),
- config_.gain_controller2.enabled,
+ config_.gain_controller2.enabled, config_.pre_amplifier.enabled,
capture_nonlocked_.echo_controller_enabled,
public_submodules_->voice_detection->is_enabled(),
public_submodules_->level_estimator->is_enabled(),
@@ -1790,12 +1857,20 @@
}
}
+void AudioProcessingImpl::InitializePreAmplifier() {
+ if (config_.pre_amplifier.enabled) {
+ private_submodules_->pre_amplifier.reset(
+ new GainApplier(true, config_.pre_amplifier.fixed_gain_factor));
+ } else {
+ private_submodules_->pre_amplifier.reset();
+ }
+}
+
void AudioProcessingImpl::InitializeResidualEchoDetector() {
RTC_DCHECK(private_submodules_->echo_detector);
private_submodules_->echo_detector->Initialize(
- proc_sample_rate_hz(), num_proc_channels(),
- formats_.render_processing_format.sample_rate_hz(),
- formats_.render_processing_format.num_channels());
+ proc_sample_rate_hz(), 1,
+ formats_.render_processing_format.sample_rate_hz(), 1);
}
void AudioProcessingImpl::InitializePostProcessor() {
@@ -1941,6 +2016,9 @@
apm_config.intelligibility_enhancer_enabled =
capture_nonlocked_.intelligibility_enabled;
apm_config.experiments_description = experiments_description;
+ apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
+ apm_config.pre_amplifier_fixed_gain_factor =
+ config_.pre_amplifier.fixed_gain_factor;
if (!forced && apm_config == apm_config_for_aec_dump_) {
return;
diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h
index 55c47ac..a49924d 100644
--- a/modules/audio_processing/audio_processing_impl.h
+++ b/modules/audio_processing/audio_processing_impl.h
@@ -24,10 +24,8 @@
#include "rtc_base/function_view.h"
#include "rtc_base/gtest_prod_util.h"
#include "rtc_base/ignore_wundef.h"
-#include "rtc_base/protobuf_utils.h"
#include "rtc_base/swap_queue.h"
#include "rtc_base/thread_annotations.h"
-#include "system_wrappers/include/file_wrapper.h"
namespace webrtc {
@@ -66,6 +64,8 @@
std::unique_ptr<AudioGenerator> audio_generator) override;
void DetachPlayoutAudioGenerator() override;
+ void SetRuntimeSetting(RuntimeSetting setting) override;
+
// Capture-side exclusive methods possibly running APM in a
// multi-threaded manner. Acquire the capture lock.
int ProcessStream(AudioFrame* frame) override;
@@ -149,6 +149,21 @@
std::unique_ptr<ApmDataDumper> data_dumper_;
static int instance_count_;
+ SwapQueue<RuntimeSetting> runtime_settings_;
+
+ // Class providing thread-safe message pipe functionality for
+ // |runtime_settings_|.
+ class RuntimeSettingEnqueuer {
+ public:
+ explicit RuntimeSettingEnqueuer(
+ SwapQueue<RuntimeSetting>* runtime_settings);
+ ~RuntimeSettingEnqueuer();
+ void Enqueue(RuntimeSetting setting);
+
+ private:
+ SwapQueue<RuntimeSetting>& runtime_settings_;
+ } runtime_settings_enqueuer_;
+
// Submodule interface implementations.
std::unique_ptr<HighPassFilter> high_pass_filter_impl_;
@@ -169,6 +184,7 @@
bool beamformer_enabled,
bool adaptive_gain_controller_enabled,
bool gain_controller2_enabled,
+ bool pre_amplifier_enabled,
bool echo_controller_enabled,
bool voice_activity_detector_enabled,
bool level_estimator_enabled,
@@ -192,6 +208,7 @@
bool beamformer_enabled_ = false;
bool adaptive_gain_controller_enabled_ = false;
bool gain_controller2_enabled_ = false;
+ bool pre_amplifier_enabled_ = false;
bool echo_controller_enabled_ = false;
bool level_estimator_enabled_ = false;
bool voice_activity_detector_enabled_ = false;
@@ -236,9 +253,13 @@
void InitializeLowCutFilter() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+ void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
+ // Handle all the runtime settings in the queue.
+ void HandleRuntimeSettings() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+
void EmptyQueuedRenderAudio();
void AllocateRenderQueue()
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
diff --git a/modules/audio_processing/audio_processing_impl_locking_unittest.cc b/modules/audio_processing/audio_processing_impl_locking_unittest.cc
index d4cff45..39f8b8b 100644
--- a/modules/audio_processing/audio_processing_impl_locking_unittest.cc
+++ b/modules/audio_processing/audio_processing_impl_locking_unittest.cc
@@ -16,7 +16,6 @@
#include "api/array_view.h"
#include "modules/audio_processing/test/test_utils.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/event.h"
#include "rtc_base/platform_thread.h"
diff --git a/modules/audio_processing/audio_processing_impl_unittest.cc b/modules/audio_processing/audio_processing_impl_unittest.cc
index e152bef..2ea0159 100644
--- a/modules/audio_processing/audio_processing_impl_unittest.cc
+++ b/modules/audio_processing/audio_processing_impl_unittest.cc
@@ -11,7 +11,6 @@
#include "modules/audio_processing/audio_processing_impl.h"
#include "modules/audio_processing/test/test_utils.h"
-#include "modules/include/module_common_types.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -34,6 +33,24 @@
MOCK_CONST_METHOD0(Release, rtc::RefCountReleaseStatus());
};
+void GenerateFixedFrame(int16_t audio_level,
+ size_t input_rate,
+ size_t num_channels,
+ AudioFrame* fixed_frame) {
+ const size_t samples_per_input_channel = rtc::CheckedDivExact(
+ input_rate, static_cast<size_t>(rtc::CheckedDivExact(
+ 1000, AudioProcessing::kChunkSizeMs)));
+ fixed_frame->samples_per_channel_ = samples_per_input_channel;
+ fixed_frame->sample_rate_hz_ = input_rate;
+ fixed_frame->num_channels_ = num_channels;
+
+ RTC_DCHECK_LE(samples_per_input_channel * num_channels,
+ AudioFrame::kMaxDataSizeSamples);
+ for (size_t i = 0; i < samples_per_input_channel * num_channels; ++i) {
+ fixed_frame->mutable_data()[i] = audio_level;
+ }
+}
+
} // namespace
TEST(AudioProcessingImplTest, AudioParameterChangeTriggersInit) {
@@ -75,4 +92,34 @@
EXPECT_NOERR(mock.ProcessReverseStream(&frame));
}
+TEST(AudioProcessingImplTest, UpdateCapturePreGainRuntimeSetting) {
+ std::unique_ptr<AudioProcessing> apm(AudioProcessingBuilder().Create());
+ webrtc::AudioProcessing::Config apm_config;
+ apm_config.pre_amplifier.enabled = true;
+ apm_config.pre_amplifier.fixed_gain_factor = 1.f;
+ apm->ApplyConfig(apm_config);
+
+ AudioFrame frame;
+ constexpr int16_t audio_level = 10000;
+ constexpr size_t input_rate = 48000;
+ constexpr size_t num_channels = 2;
+
+ GenerateFixedFrame(audio_level, input_rate, num_channels, &frame);
+ apm->ProcessStream(&frame);
+ EXPECT_EQ(frame.data()[100], audio_level)
+ << "With factor 1, frame shouldn't be modified.";
+
+ constexpr float gain_factor = 2.f;
+ apm->SetRuntimeSetting(
+ AudioProcessing::RuntimeSetting::CreateCapturePreGain(gain_factor));
+
+ // Process for two frames to have time to ramp up gain.
+ for (int i = 0; i < 2; ++i) {
+ GenerateFixedFrame(audio_level, input_rate, num_channels, &frame);
+ apm->ProcessStream(&frame);
+ }
+ EXPECT_EQ(frame.data()[100], gain_factor * audio_level)
+ << "Frame should be amplified.";
+}
+
} // namespace webrtc
diff --git a/modules/audio_processing/audio_processing_performance_unittest.cc b/modules/audio_processing/audio_processing_performance_unittest.cc
index 8dd81b2..d312137 100644
--- a/modules/audio_processing/audio_processing_performance_unittest.cc
+++ b/modules/audio_processing/audio_processing_performance_unittest.cc
@@ -17,7 +17,6 @@
#include "api/array_view.h"
#include "modules/audio_processing/test/test_utils.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/atomicops.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/platform_thread.h"
diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc
index 89d6cb9..fbd81a1 100644
--- a/modules/audio_processing/audio_processing_unittest.cc
+++ b/modules/audio_processing/audio_processing_unittest.cc
@@ -27,7 +27,6 @@
#include "modules/audio_processing/include/mock_audio_processing.h"
#include "modules/audio_processing/test/protobuf_utils.h"
#include "modules/audio_processing/test/test_utils.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/gtest_prod_util.h"
@@ -36,6 +35,7 @@
#include "rtc_base/numerics/safe_minmax.h"
#include "rtc_base/protobuf_utils.h"
#include "rtc_base/refcountedobject.h"
+#include "rtc_base/swap_queue.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/thread.h"
#include "system_wrappers/include/event_wrapper.h"
@@ -2820,6 +2820,34 @@
} // namespace
+TEST(RuntimeSettingTest, TestDefaultCtor) {
+ auto s = AudioProcessing::RuntimeSetting();
+ EXPECT_EQ(AudioProcessing::RuntimeSetting::Type::kNotSpecified, s.type());
+}
+
+TEST(RuntimeSettingTest, TestCapturePreGain) {
+ using Type = AudioProcessing::RuntimeSetting::Type;
+ {
+ auto s = AudioProcessing::RuntimeSetting::CreateCapturePreGain(1.25f);
+ EXPECT_EQ(Type::kCapturePreGain, s.type());
+ float v;
+ s.GetFloat(&v);
+ EXPECT_EQ(1.25f, v);
+ }
+
+#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
+ EXPECT_DEATH(AudioProcessing::RuntimeSetting::CreateCapturePreGain(0.1f), "");
+#endif
+}
+
+TEST(RuntimeSettingTest, TestUsageWithSwapQueue) {
+ SwapQueue<AudioProcessing::RuntimeSetting> q(1);
+ auto s = AudioProcessing::RuntimeSetting();
+ ASSERT_TRUE(q.Insert(&s));
+ ASSERT_TRUE(q.Remove(&s));
+ EXPECT_EQ(AudioProcessing::RuntimeSetting::Type::kNotSpecified, s.type());
+}
+
TEST(ApmConfiguration, EnablePostProcessing) {
// Verify that apm uses a capture post processing module if one is provided.
webrtc::Config webrtc_config;
diff --git a/modules/audio_processing/debug.proto b/modules/audio_processing/debug.proto
index 4417773..f7f8d10 100644
--- a/modules/audio_processing/debug.proto
+++ b/modules/audio_processing/debug.proto
@@ -72,8 +72,11 @@
// Semicolon-separated string containing experimental feature
// descriptions.
optional string experiments_description = 17;
- // Intelligibility Enhancer
+ // Intelligibility Enhancer.
optional bool intelligibility_enhancer_enabled = 18;
+ // Pre amplifier.
+ optional bool pre_amplifier_enabled = 19;
+ optional float pre_amplifier_fixed_gain_factor = 20;
}
message Event {
diff --git a/modules/audio_processing/gain_controller2.cc b/modules/audio_processing/gain_controller2.cc
index aa866c1..6c0149a 100644
--- a/modules/audio_processing/gain_controller2.cc
+++ b/modules/audio_processing/gain_controller2.cc
@@ -23,7 +23,8 @@
GainController2::GainController2()
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
- gain_controller_(data_dumper_.get()) {}
+ fixed_gain_controller_(data_dumper_.get()),
+ adaptive_agc_(data_dumper_.get()) {}
GainController2::~GainController2() = default;
@@ -32,7 +33,7 @@
sample_rate_hz == AudioProcessing::kSampleRate16kHz ||
sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
sample_rate_hz == AudioProcessing::kSampleRate48kHz);
- gain_controller_.SetSampleRate(sample_rate_hz);
+ fixed_gain_controller_.SetSampleRate(sample_rate_hz);
data_dumper_->InitiateNewSetOfRecordings();
data_dumper_->DumpRaw("sample_rate_hz", sample_rate_hz);
}
@@ -40,15 +41,15 @@
void GainController2::Process(AudioBuffer* audio) {
AudioFrameView<float> float_frame(audio->channels_f(), audio->num_channels(),
audio->num_frames());
- gain_controller_.Process(float_frame);
+ adaptive_agc_.Process(float_frame);
+ fixed_gain_controller_.Process(float_frame);
}
void GainController2::ApplyConfig(
const AudioProcessing::Config::GainController2& config) {
RTC_DCHECK(Validate(config));
config_ = config;
- gain_controller_.SetGain(config_.fixed_gain_db);
- gain_controller_.EnableLimiter(config_.enable_limiter);
+ fixed_gain_controller_.SetGain(config_.fixed_gain_db);
}
bool GainController2::Validate(
@@ -60,8 +61,7 @@
const AudioProcessing::Config::GainController2& config) {
std::stringstream ss;
ss << "{enabled: " << (config.enabled ? "true" : "false") << ", "
- << "fixed_gain_dB: " << config.fixed_gain_db << ", "
- << "enable_limiter: " << (config.enable_limiter ? "true" : "false") << "}";
+ << "fixed_gain_dB: " << config.fixed_gain_db << "}";
return ss.str();
}
diff --git a/modules/audio_processing/gain_controller2.h b/modules/audio_processing/gain_controller2.h
index 6de4564..e2ca05e 100644
--- a/modules/audio_processing/gain_controller2.h
+++ b/modules/audio_processing/gain_controller2.h
@@ -14,6 +14,7 @@
#include <memory>
#include <string>
+#include "modules/audio_processing/agc2/adaptive_agc.h"
#include "modules/audio_processing/agc2/fixed_gain_controller.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "rtc_base/constructormagic.h"
@@ -41,8 +42,9 @@
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
- FixedGainController gain_controller_;
+ FixedGainController fixed_gain_controller_;
AudioProcessing::Config::GainController2 config_;
+ AdaptiveAgc adaptive_agc_;
RTC_DISALLOW_COPY_AND_ASSIGN(GainController2);
};
diff --git a/modules/audio_processing/gain_controller2_unittest.cc b/modules/audio_processing/gain_controller2_unittest.cc
index 0c3b383..5bedccd 100644
--- a/modules/audio_processing/gain_controller2_unittest.cc
+++ b/modules/audio_processing/gain_controller2_unittest.cc
@@ -61,12 +61,11 @@
config.fixed_gain_db = 5.f;
config.enabled = false;
- config.enable_limiter = true;
- EXPECT_EQ("{enabled: false, fixed_gain_dB: 5, enable_limiter: true}",
+ EXPECT_EQ("{enabled: false, fixed_gain_dB: 5}",
GainController2::ToString(config));
config.enabled = true;
- EXPECT_EQ("{enabled: true, fixed_gain_dB: 5, enable_limiter: true}",
+ EXPECT_EQ("{enabled: true, fixed_gain_dB: 5}",
GainController2::ToString(config));
}
@@ -81,12 +80,6 @@
SetAudioBufferSamples(sample_value, &ab);
AudioProcessing::Config::GainController2 config;
- // Check that samples are not modified when the fixed gain is 0 dB.
- config.fixed_gain_db = 0.f;
- gain_controller2->ApplyConfig(config);
- gain_controller2->Process(&ab);
- EXPECT_EQ(ab.channels_f()[0][0], sample_value);
-
// Check that samples are amplified when the fixed gain is greater than 0 dB.
config.fixed_gain_db = 5.f;
gain_controller2->ApplyConfig(config);
diff --git a/modules/audio_processing/include/aec_dump.cc b/modules/audio_processing/include/aec_dump.cc
index 365d015..c243b52 100644
--- a/modules/audio_processing/include/aec_dump.cc
+++ b/modules/audio_processing/include/aec_dump.cc
@@ -35,6 +35,9 @@
intelligibility_enhancer_enabled ==
other.intelligibility_enhancer_enabled &&
noise_robust_agc_enabled == other.noise_robust_agc_enabled &&
+ pre_amplifier_enabled == other.pre_amplifier_enabled &&
+ pre_amplifier_fixed_gain_factor ==
+ other.pre_amplifier_fixed_gain_factor &&
experiments_description == other.experiments_description;
}
} // namespace webrtc
diff --git a/modules/audio_processing/include/aec_dump.h b/modules/audio_processing/include/aec_dump.h
index 2035bf4..e824892 100644
--- a/modules/audio_processing/include/aec_dump.h
+++ b/modules/audio_processing/include/aec_dump.h
@@ -16,12 +16,11 @@
#include <vector>
#include "api/array_view.h"
+#include "api/audio/audio_frame.h"
#include "modules/audio_processing/include/audio_frame_view.h"
namespace webrtc {
-class AudioFrame;
-
// Struct for passing current config from APM without having to
// include protobuf headers.
struct InternalAPMConfig {
@@ -51,6 +50,8 @@
bool transient_suppression_enabled = false;
bool intelligibility_enhancer_enabled = false;
bool noise_robust_agc_enabled = false;
+ bool pre_amplifier_enabled = false;
+ float pre_amplifier_fixed_gain_factor = 1.f;
std::string experiments_description = "";
};
diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h
index d31174a..379a664 100644
--- a/modules/audio_processing/include/audio_processing.h
+++ b/modules/audio_processing/include/audio_processing.h
@@ -270,14 +270,20 @@
bool enabled = false;
} high_pass_filter;
- // Enables the next generation AGC functionality. This feature replaces the
- // standard methods of gain control in the previous AGC.
- // The functionality is not yet activated in the code and turning this on
- // does not yet have the desired behavior.
+ // Enabled the pre-amplifier. It amplifies the capture signal
+ // before any other processing is done.
+ struct PreAmplifier {
+ bool enabled = false;
+ float fixed_gain_factor = 1.f;
+ } pre_amplifier;
+
+ // Enables the next generation AGC functionality. This feature
+ // replaces the standard methods of gain control in the previous
+ // AGC. This functionality is currently only partially
+ // implemented.
struct GainController2 {
bool enabled = false;
float fixed_gain_db = 0.f;
- bool enable_limiter = true;
} gain_controller2;
// Explicit copy assignment implementation to avoid issues with memory
@@ -303,6 +309,32 @@
kStereoAndKeyboard
};
+ // Specifies the properties of a setting to be passed to AudioProcessing at
+ // runtime.
+ class RuntimeSetting {
+ public:
+ enum class Type { kNotSpecified, kCapturePreGain };
+
+ RuntimeSetting() : type_(Type::kNotSpecified), value_(0.f) {}
+ ~RuntimeSetting() = default;
+
+ static RuntimeSetting CreateCapturePreGain(float gain) {
+ RTC_DCHECK_GE(gain, 1.f) << "Attenuation is not allowed.";
+ return {Type::kCapturePreGain, gain};
+ }
+
+ Type type() const { return type_; }
+ void GetFloat(float* value) const {
+ RTC_DCHECK(value);
+ *value = value_;
+ }
+
+ private:
+ RuntimeSetting(Type id, float value) : type_(id), value_(value) {}
+ Type type_;
+ float value_;
+ };
+
~AudioProcessing() override {}
// Initializes internal states, while retaining all user settings. This
@@ -360,6 +392,9 @@
// Default false.
virtual void set_output_will_be_muted(bool muted) = 0;
+ // Enqueue a runtime setting.
+ virtual void SetRuntimeSetting(RuntimeSetting setting) = 0;
+
// Processes a 10 ms |frame| of the primary audio stream. On the client-side,
// this is the near-end (or captured) audio.
//
diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h
index 96a04ef..5fa3b1b 100644
--- a/modules/audio_processing/include/mock_audio_processing.h
+++ b/modules/audio_processing/include/mock_audio_processing.h
@@ -121,6 +121,7 @@
MOCK_METHOD2(ProcessCapture,
void(AudioBuffer* capture, bool echo_path_change));
MOCK_CONST_METHOD0(GetMetrics, Metrics());
+ MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms));
};
class MockVoiceDetection : public VoiceDetection {
@@ -167,6 +168,7 @@
MOCK_CONST_METHOD0(num_output_channels, size_t());
MOCK_CONST_METHOD0(num_reverse_channels, size_t());
MOCK_METHOD1(set_output_will_be_muted, void(bool muted));
+ MOCK_METHOD1(SetRuntimeSetting, void(RuntimeSetting setting));
MOCK_METHOD1(ProcessStream, int(AudioFrame* frame));
MOCK_METHOD7(ProcessStream, int(const float* const* src,
size_t samples_per_channel,
diff --git a/modules/audio_processing/logging/apm_data_dumper.cc b/modules/audio_processing/logging/apm_data_dumper.cc
index 2f6a6d6..e2e8602 100644
--- a/modules/audio_processing/logging/apm_data_dumper.cc
+++ b/modules/audio_processing/logging/apm_data_dumper.cc
@@ -10,8 +10,7 @@
#include "modules/audio_processing/logging/apm_data_dumper.h"
-#include <sstream>
-
+#include "rtc_base/strings/string_builder.h"
#include "rtc_base/stringutils.h"
// Check to verify that the define is properly set.
@@ -29,7 +28,8 @@
int instance_index,
int reinit_index,
const std::string& suffix) {
- std::stringstream ss;
+ char buf[1024];
+ rtc::SimpleStringBuilder ss(buf);
ss << name << "_" << instance_index << "-" << reinit_index << suffix;
return ss.str();
}
diff --git a/modules/audio_processing/module.mk b/modules/audio_processing/module.mk
index 7f39204..a6c8dad 100644
--- a/modules/audio_processing/module.mk
+++ b/modules/audio_processing/module.mk
@@ -97,9 +97,11 @@
modules/audio_processing/aec3/block_processor.o \
modules/audio_processing/aec3/block_processor_metrics.o \
modules/audio_processing/aec3/cascaded_biquad_filter.o \
+ modules/audio_processing/aec3/coherence_gain.o \
modules/audio_processing/aec3/comfort_noise_generator.o \
modules/audio_processing/aec3/decimator.o \
modules/audio_processing/aec3/downsampled_render_buffer.o \
+ modules/audio_processing/aec3/echo_audibility.o \
modules/audio_processing/aec3/echo_canceller3.o \
modules/audio_processing/aec3/echo_path_delay_estimator.o \
modules/audio_processing/aec3/echo_path_variability.o \
@@ -108,12 +110,12 @@
modules/audio_processing/aec3/erl_estimator.o \
modules/audio_processing/aec3/erle_estimator.o \
modules/audio_processing/aec3/fft_buffer.o \
+ modules/audio_processing/aec3/filter_analyzer.o \
modules/audio_processing/aec3/frame_blocker.o \
modules/audio_processing/aec3/main_filter_update_gain.o \
modules/audio_processing/aec3/matched_filter.o \
modules/audio_processing/aec3/matched_filter_lag_aggregator.o \
modules/audio_processing/aec3/matrix_buffer.o \
- modules/audio_processing/aec3/output_selector.o \
modules/audio_processing/aec3/render_buffer.o \
modules/audio_processing/aec3/render_delay_buffer.o \
modules/audio_processing/aec3/render_delay_controller.o \
@@ -122,6 +124,7 @@
modules/audio_processing/aec3/residual_echo_estimator.o \
modules/audio_processing/aec3/shadow_filter_update_gain.o \
modules/audio_processing/aec3/skew_estimator.o \
+ modules/audio_processing/aec3/stationarity_estimator.o \
modules/audio_processing/aec3/subtractor.o \
modules/audio_processing/aec3/suppression_filter.o \
modules/audio_processing/aec3/suppression_gain.o \
@@ -135,10 +138,23 @@
api/audio/echo_canceller3_config.o
agc2_CXX_OBJECTS = \
+ modules/audio_processing/agc2/adaptive_agc.o \
+ modules/audio_processing/agc2/adaptive_digital_gain_applier.o \
+ modules/audio_processing/agc2/adaptive_mode_level_estimator.o \
+ modules/audio_processing/agc2/biquad_filter.o \
+ modules/audio_processing/agc2/compute_interpolated_gain_curve.o \
+ modules/audio_processing/agc2/down_sampler.o \
modules/audio_processing/agc2/fixed_digital_level_estimator.o \
modules/audio_processing/agc2/fixed_gain_controller.o \
+ modules/audio_processing/agc2/gain_applier.o \
modules/audio_processing/agc2/gain_curve_applier.o \
- modules/audio_processing/agc2/interpolated_gain_curve.o
+ modules/audio_processing/agc2/interpolated_gain_curve.o \
+ modules/audio_processing/agc2/limiter.o \
+ modules/audio_processing/agc2/noise_level_estimator.o \
+ modules/audio_processing/agc2/noise_spectrum_estimator.o \
+ modules/audio_processing/agc2/saturation_protector.o \
+ modules/audio_processing/agc2/signal_classifier.o \
+ modules/audio_processing/agc2/vector_float_frame.o
# Dependency of agc/agc.o
vad_CXX_OBJECTS = \
diff --git a/modules/audio_processing/ns/noise_suppression.h b/modules/audio_processing/ns/noise_suppression.h
index a167142..fa5da70 100644
--- a/modules/audio_processing/ns/noise_suppression.h
+++ b/modules/audio_processing/ns/noise_suppression.h
@@ -24,7 +24,7 @@
/*
* This function creates an instance of the floating point Noise Suppression.
*/
-NsHandle* WebRtcNs_Create();
+NsHandle* WebRtcNs_Create(void);
/*
* This function frees the dynamic memory of a specified noise suppression
@@ -126,7 +126,7 @@
*
* Return value : Number of frequency bins.
*/
-size_t WebRtcNs_num_freq();
+size_t WebRtcNs_num_freq(void);
#ifdef __cplusplus
}
diff --git a/modules/audio_processing/ns/noise_suppression_x.h b/modules/audio_processing/ns/noise_suppression_x.h
index 838861d..f25fb7a 100644
--- a/modules/audio_processing/ns/noise_suppression_x.h
+++ b/modules/audio_processing/ns/noise_suppression_x.h
@@ -24,7 +24,7 @@
/*
* This function creates an instance of the fixed point Noise Suppression.
*/
-NsxHandle* WebRtcNsx_Create();
+NsxHandle* WebRtcNsx_Create(void);
/*
* This function frees the dynamic memory of a specified Noise Suppression
@@ -104,7 +104,7 @@
*
* Return value : Number of frequency bins.
*/
-size_t WebRtcNsx_num_freq();
+size_t WebRtcNsx_num_freq(void);
#ifdef __cplusplus
}
diff --git a/modules/audio_processing/transient/click_annotate.cc b/modules/audio_processing/transient/click_annotate.cc
index a8b4a30..1f01d01 100644
--- a/modules/audio_processing/transient/click_annotate.cc
+++ b/modules/audio_processing/transient/click_annotate.cc
@@ -14,9 +14,9 @@
#include <memory>
#include <vector>
-#include "modules/audio_processing/transient/transient_detector.h"
#include "modules/audio_processing/transient/file_utils.h"
-#include "system_wrappers/include/file_wrapper.h"
+#include "modules/audio_processing/transient/transient_detector.h"
+#include "rtc_base/system/file_wrapper.h"
using webrtc::FileWrapper;
using webrtc::TransientDetector;
diff --git a/modules/audio_processing/transient/file_utils.cc b/modules/audio_processing/transient/file_utils.cc
index 7bf2e08..40732b9 100644
--- a/modules/audio_processing/transient/file_utils.cc
+++ b/modules/audio_processing/transient/file_utils.cc
@@ -12,7 +12,7 @@
#include <memory>
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/modules/audio_processing/transient/file_utils.h b/modules/audio_processing/transient/file_utils.h
index 3f05c1d..4b04fac 100644
--- a/modules/audio_processing/transient/file_utils.h
+++ b/modules/audio_processing/transient/file_utils.h
@@ -13,7 +13,7 @@
#include <string.h>
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/modules/audio_processing/transient/file_utils_unittest.cc b/modules/audio_processing/transient/file_utils_unittest.cc
index c5e0399..d9880f9 100644
--- a/modules/audio_processing/transient/file_utils_unittest.cc
+++ b/modules/audio_processing/transient/file_utils_unittest.cc
@@ -15,7 +15,7 @@
#include <memory>
#include <vector>
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_processing/transient/transient_detector_unittest.cc b/modules/audio_processing/transient/transient_detector_unittest.cc
index 96af179..8f60954 100644
--- a/modules/audio_processing/transient/transient_detector_unittest.cc
+++ b/modules/audio_processing/transient/transient_detector_unittest.cc
@@ -16,7 +16,7 @@
#include "modules/audio_processing/transient/common.h"
#include "modules/audio_processing/transient/file_utils.h"
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_processing/transient/transient_suppression_test.cc b/modules/audio_processing/transient/transient_suppression_test.cc
index 14fe4f8..8512e01 100644
--- a/modules/audio_processing/transient/transient_suppression_test.cc
+++ b/modules/audio_processing/transient/transient_suppression_test.cc
@@ -19,7 +19,6 @@
#include "common_audio/include/audio_util.h"
#include "modules/audio_processing/agc/agc.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/flags.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_processing/transient/wpd_tree_unittest.cc b/modules/audio_processing/transient/wpd_tree_unittest.cc
index a90af77..02c7f51 100644
--- a/modules/audio_processing/transient/wpd_tree_unittest.cc
+++ b/modules/audio_processing/transient/wpd_tree_unittest.cc
@@ -16,7 +16,7 @@
#include "modules/audio_processing/transient/daubechies_8_wavelet_coeffs.h"
#include "modules/audio_processing/transient/file_utils.h"
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_processing/typing_detection.h b/modules/audio_processing/typing_detection.h
index fe74a59..14dfe1d 100644
--- a/modules/audio_processing/typing_detection.h
+++ b/modules/audio_processing/typing_detection.h
@@ -11,7 +11,6 @@
#ifndef MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
#define MODULES_AUDIO_PROCESSING_TYPING_DETECTION_H_
-#include "modules/include/module_common_types.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/modules/audio_processing/vad/BUILD.gn b/modules/audio_processing/vad/BUILD.gn
index 9976b78..f9f5974 100644
--- a/modules/audio_processing/vad/BUILD.gn
+++ b/modules/audio_processing/vad/BUILD.gn
@@ -35,7 +35,6 @@
"voice_gmm_tables.h",
]
deps = [
- "../..:module_api",
"../../..:typedefs",
"../../../audio/utility:audio_frame_operations",
"../../../common_audio",
@@ -45,6 +44,16 @@
]
}
+rtc_source_set("vad_with_level") {
+ sources = [
+ "vad_with_level.h",
+ ]
+ deps = [
+ "..:audio_frame_view",
+ "../../../api:array_view",
+ ]
+}
+
if (rtc_include_tests) {
rtc_static_library("vad_unittests") {
testonly = true
@@ -60,7 +69,6 @@
]
deps = [
":vad",
- "../..:module_api",
"../../../common_audio",
"../../../test:fileutils",
"../../../test:test_support",
diff --git a/modules/audio_processing/vad/pitch_based_vad.cc b/modules/audio_processing/vad/pitch_based_vad.cc
index bca2552..240ec63 100644
--- a/modules/audio_processing/vad/pitch_based_vad.cc
+++ b/modules/audio_processing/vad/pitch_based_vad.cc
@@ -17,7 +17,6 @@
#include "modules/audio_processing/vad/common.h"
#include "modules/audio_processing/vad/noise_gmm_tables.h"
#include "modules/audio_processing/vad/voice_gmm_tables.h"
-#include "modules/include/module_common_types.h"
namespace webrtc {
diff --git a/modules/audio_processing/vad/standalone_vad.cc b/modules/audio_processing/vad/standalone_vad.cc
index 004cefe..2640892 100644
--- a/modules/audio_processing/vad/standalone_vad.cc
+++ b/modules/audio_processing/vad/standalone_vad.cc
@@ -10,8 +10,9 @@
#include "modules/audio_processing/vad/standalone_vad.h"
+#include <string.h>
+
#include "audio/utility/audio_frame_operations.h"
-#include "modules/include/module_common_types.h"
#include "rtc_base/checks.h"
#include "typedefs.h" // NOLINT(build/include)
diff --git a/modules/audio_processing/vad/standalone_vad_unittest.cc b/modules/audio_processing/vad/standalone_vad_unittest.cc
index 28d1349..512276e 100644
--- a/modules/audio_processing/vad/standalone_vad_unittest.cc
+++ b/modules/audio_processing/vad/standalone_vad_unittest.cc
@@ -14,7 +14,6 @@
#include <memory>
-#include "modules/include/module_common_types.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_processing/vad/vad_audio_proc.cc b/modules/audio_processing/vad/vad_audio_proc.cc
index b1841d0..6b559d1 100644
--- a/modules/audio_processing/vad/vad_audio_proc.cc
+++ b/modules/audio_processing/vad/vad_audio_proc.cc
@@ -12,6 +12,7 @@
#include <math.h>
#include <stdio.h>
+#include <string.h>
#include "common_audio/fft4g.h"
#include "modules/audio_processing/vad/pitch_internal.h"
@@ -24,7 +25,6 @@
#include "modules/audio_coding/codecs/isac/main/source/pitch_estimator.h"
#include "modules/audio_coding/codecs/isac/main/source/structs.h"
}
-#include "modules/include/module_common_types.h"
namespace webrtc {
diff --git a/modules/audio_processing/vad/vad_audio_proc_unittest.cc b/modules/audio_processing/vad/vad_audio_proc_unittest.cc
index c520257..5b96be6 100644
--- a/modules/audio_processing/vad/vad_audio_proc_unittest.cc
+++ b/modules/audio_processing/vad/vad_audio_proc_unittest.cc
@@ -20,7 +20,6 @@
#include <string>
#include "modules/audio_processing/vad/common.h"
-#include "modules/include/module_common_types.h"
#include "test/gtest.h"
#include "test/testsupport/fileutils.h"
diff --git a/modules/audio_processing/vad/vad_with_level.h b/modules/audio_processing/vad/vad_with_level.h
new file mode 100644
index 0000000..9ad4d17
--- /dev/null
+++ b/modules/audio_processing/vad/vad_with_level.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_AUDIO_PROCESSING_VAD_VAD_WITH_LEVEL_H_
+#define MODULES_AUDIO_PROCESSING_VAD_VAD_WITH_LEVEL_H_
+
+#include "api/array_view.h"
+#include "modules/audio_processing/include/audio_frame_view.h"
+
+namespace webrtc {
+class VadWithLevel {
+ public:
+ struct LevelAndProbability {
+ constexpr LevelAndProbability(float prob, float rms, float peak)
+ : speech_probability(prob),
+ speech_rms_dbfs(rms),
+ speech_peak_dbfs(peak) {}
+ LevelAndProbability() = default;
+ float speech_probability = 0;
+ float speech_rms_dbfs = 0; // Root mean square in decibels to full-scale.
+ float speech_peak_dbfs = 0;
+ };
+
+ // TODO(webrtc:7494): This is a stub. Add implementation.
+ rtc::ArrayView<const LevelAndProbability> AnalyzeFrame(
+ AudioFrameView<const float> frame) {
+ return {nullptr, 0};
+ }
+};
+
+} // namespace webrtc
+
+#endif // MODULES_AUDIO_PROCESSING_VAD_VAD_WITH_LEVEL_H_
diff --git a/modules/include/module_common_types.h b/modules/include/module_common_types.h
index 1290075..1c174a7 100644
--- a/modules/include/module_common_types.h
+++ b/modules/include/module_common_types.h
@@ -18,9 +18,6 @@
#include <limits>
#include "api/optional.h"
-// TODO(bugs.webrtc.org/7504): Included here because users of this header expect
-// it to declare AudioFrame. Delete as soon as all known users are updated.
-#include "api/audio/audio_frame.h"
#include "api/rtp_headers.h"
#include "api/video/video_rotation.h"
#include "common_types.h" // NOLINT(build/include)
diff --git a/modules/video_coding/codecs/h264/include/h264_globals.h b/modules/video_coding/codecs/h264/include/h264_globals.h
index cae270c..bfeba67 100644
--- a/modules/video_coding/codecs/h264/include/h264_globals.h
+++ b/modules/video_coding/codecs/h264/include/h264_globals.h
@@ -14,6 +14,8 @@
#ifndef MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_
#define MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_
+#include <string>
+
namespace webrtc {
// The packetization types that we support: single, aggregated, and fragmented.
@@ -40,17 +42,15 @@
// This function is declared inline because it is not clear which
// .cc file it should belong to.
// TODO(hta): Refactor. https://bugs.webrtc.org/6842
-inline std::ostream& operator<<(std::ostream& stream,
- H264PacketizationMode mode) {
- switch (mode) {
- case H264PacketizationMode::NonInterleaved:
- stream << "NonInterleaved";
- break;
- case H264PacketizationMode::SingleNalUnit:
- stream << "SingleNalUnit";
- break;
+// TODO(jonasolsson): Use absl::string_view instead when that's available.
+inline std::string ToString(H264PacketizationMode mode) {
+ if (mode == H264PacketizationMode::NonInterleaved) {
+ return "NonInterleaved";
+ } else if (mode == H264PacketizationMode::SingleNalUnit) {
+ return "SingleNalUnit";
}
- return stream;
+ RTC_NOTREACHED();
+ return "";
}
struct NaluInfo {
diff --git a/modules/video_coding/codecs/vp8/include/vp8.h b/modules/video_coding/codecs/vp8/include/vp8.h
index 00808e2..57be521 100644
--- a/modules/video_coding/codecs/vp8/include/vp8.h
+++ b/modules/video_coding/codecs/vp8/include/vp8.h
@@ -23,14 +23,14 @@
public:
static std::unique_ptr<VP8Encoder> Create();
- virtual ~VP8Encoder() {}
+ ~VP8Encoder() override {}
}; // end of VP8Encoder class
class VP8Decoder : public VideoDecoder {
public:
static std::unique_ptr<VP8Decoder> Create();
- virtual ~VP8Decoder() {}
+ ~VP8Decoder() override {}
}; // end of VP8Decoder class
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/include/vp9.h b/modules/video_coding/codecs/vp9/include/vp9.h
index 172e69e..39dc138 100644
--- a/modules/video_coding/codecs/vp9/include/vp9.h
+++ b/modules/video_coding/codecs/vp9/include/vp9.h
@@ -23,7 +23,7 @@
static bool IsSupported();
static std::unique_ptr<VP9Encoder> Create();
- virtual ~VP9Encoder() {}
+ ~VP9Encoder() override {}
};
class VP9Decoder : public VideoDecoder {
@@ -31,7 +31,7 @@
static bool IsSupported();
static std::unique_ptr<VP9Decoder> Create();
- virtual ~VP9Decoder() {}
+ ~VP9Decoder() override {}
};
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/include/vp9_globals.h b/modules/video_coding/codecs/vp9/include/vp9_globals.h
index 945eb0c..918e411 100644
--- a/modules/video_coding/codecs/vp9/include/vp9_globals.h
+++ b/modules/video_coding/codecs/vp9/include/vp9_globals.h
@@ -14,6 +14,8 @@
#ifndef MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_GLOBALS_H_
#define MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_GLOBALS_H_
+#include <assert.h>
+
#include "modules/video_coding/codecs/interface/common_constants.h"
namespace webrtc {
@@ -27,6 +29,9 @@
const size_t kMaxVp9FramesInGof = 0xFF; // 8 bits
const size_t kMaxVp9NumberOfSpatialLayers = 8;
+const size_t kMinVp9SpatialLayerWidth = 320;
+const size_t kMinVp9SpatialLayerHeight = 180;
+
enum TemporalStructureMode {
kTemporalStructureMode1, // 1 temporal layer structure - i.e., IPPP...
kTemporalStructureMode2, // 2 temporal layers 01...
@@ -157,6 +162,7 @@
beginning_of_frame = false;
end_of_frame = false;
ss_data_available = false;
+ non_ref_for_inter_layer_pred = false;
picture_id = kNoPictureId;
max_picture_id = kMaxTwoBytePictureId;
tl0_pic_idx = kNoTl0PicIdx;
@@ -167,6 +173,7 @@
gof_idx = kNoGofIdx;
num_ref_pics = 0;
num_spatial_layers = 1;
+ end_of_picture = true;
}
bool inter_pic_predicted; // This layer frame is dependent on previously
@@ -177,6 +184,8 @@
bool end_of_frame; // True if this packet is the last in a VP9 layer frame.
bool ss_data_available; // True if SS data is available in this payload
// descriptor.
+ bool non_ref_for_inter_layer_pred; // True for frame which is not used as
+ // reference for inter-layer prediction.
int16_t picture_id; // PictureID index, 15 bits;
// kNoPictureId if PictureID does not exist.
int16_t max_picture_id; // Maximum picture ID index; either 0x7F or 0x7FFF;
@@ -203,6 +212,8 @@
uint16_t width[kMaxVp9NumberOfSpatialLayers];
uint16_t height[kMaxVp9NumberOfSpatialLayers];
GofInfoVP9 gof;
+
+ bool end_of_picture; // This frame is the last frame in picture.
};
} // namespace webrtc
diff --git a/rtc_base/BUILD.gn b/rtc_base/BUILD.gn
index 50e0b24..35c9da8 100644
--- a/rtc_base/BUILD.gn
+++ b/rtc_base/BUILD.gn
@@ -14,16 +14,12 @@
import("//build/config/android/config.gni")
import("//build/config/android/rules.gni")
}
-if (is_win) {
- import("//build/config/clang/clang.gni")
-}
group("base") {
public_deps = [
":rtc_base",
":rtc_base_approved",
":rtc_task_queue",
- ":sequenced_task_checker",
":weak_ptr",
]
if (is_android) {
@@ -85,13 +81,171 @@
rtc_source_set("rtc_base_approved") {
visibility = [ "*" ]
public_deps = [
+ ":atomicops",
+ ":criticalsection",
+ ":logging",
+ ":macromagic",
+ ":platform_thread",
+ ":platform_thread_types",
+ ":ptr_util",
+ ":refcount",
":rtc_base_approved_generic",
+ ":rtc_event",
+ ":safe_conversions",
+ ":stringutils",
+ ":thread_checker",
+ ":timeutils",
]
if (is_mac && !build_with_chromium) {
public_deps += [ ":rtc_base_approved_objc" ]
}
}
+rtc_source_set("macromagic") {
+ sources = [
+ "arraysize.h",
+ "basictypes.h",
+ "constructormagic.h",
+ "format_macros.h",
+ "stringize_macros.h",
+ "thread_annotations.h",
+ ]
+}
+
+rtc_source_set("platform_thread_types") {
+ sources = [
+ "platform_thread_types.cc",
+ "platform_thread_types.h",
+ ]
+}
+
+rtc_source_set("ptr_util") {
+ sources = [
+ "ptr_util.h",
+ "scoped_ref_ptr.h",
+ ]
+}
+
+rtc_source_set("refcount") {
+ sources = [
+ "refcount.h",
+ "refcountedobject.h",
+ "refcounter.h",
+ ]
+ deps = [
+ ":atomicops",
+ ":macromagic",
+ ]
+}
+
+rtc_source_set("criticalsection") {
+ sources = [
+ "criticalsection.cc",
+ "criticalsection.h",
+ ]
+ deps = [
+ ":atomicops",
+ ":checks",
+ ":macromagic",
+ ":platform_thread_types",
+ "..:typedefs",
+ ]
+}
+
+rtc_source_set("platform_thread") {
+ visibility = [
+ ":rtc_base_approved",
+ ":rtc_base_approved_generic",
+ ":rtc_task_queue_libevent",
+ ":rtc_task_queue_win",
+ ":sequenced_task_checker",
+ ]
+ sources = [
+ "platform_thread.cc",
+ "platform_thread.h",
+ ]
+ deps = [
+ ":atomicops",
+ ":checks",
+ ":macromagic",
+ ":platform_thread_types",
+ ":rtc_event",
+ ":thread_checker",
+ ":timeutils",
+ "..:typedefs",
+ ]
+}
+
+rtc_source_set("rtc_event") {
+ deps = [
+ ":checks",
+ ":macromagic",
+ ]
+
+ if (build_with_chromium) {
+ # Dependency on chromium's waitable_event (in //base).
+ deps += [ "//base:base" ]
+ sources = [
+ "../../webrtc_overrides/rtc_base/event.cc",
+ "../../webrtc_overrides/rtc_base/event.h",
+ ]
+ } else {
+ sources = [
+ "event.cc",
+ "event.h",
+ ]
+ }
+}
+
+rtc_source_set("logging") {
+ deps = [
+ ":criticalsection",
+ ":macromagic",
+ ":platform_thread_types",
+ ":stringutils",
+ ":timeutils",
+ ]
+
+ if (build_with_chromium) {
+ # Dependency on chromium's logging (in //base).
+ deps += [ "//base:base" ]
+ sources = [
+ "../../webrtc_overrides/rtc_base/logging.cc",
+ "../../webrtc_overrides/rtc_base/logging.h",
+ ]
+ } else {
+ sources = [
+ "logging.cc",
+ "logging.h",
+ ]
+
+ # logging.h needs the deprecation header while downstream projects are
+ # removing code that depends on logging implementation details.
+ deps += [ ":deprecation" ]
+ }
+}
+
+rtc_source_set("thread_checker") {
+ sources = [
+ "thread_checker.h",
+ "thread_checker_impl.cc",
+ "thread_checker_impl.h",
+ ]
+ deps = [
+ ":checks",
+ ":criticalsection",
+ ":macromagic",
+ ":platform_thread_types",
+ "..:typedefs",
+ ]
+}
+
+rtc_source_set("atomicops") {
+ sources = [
+ "atomicops.h",
+ ]
+}
+
rtc_source_set("checks") {
sources = [
"checks.cc",
@@ -140,8 +294,31 @@
]
}
+rtc_source_set("safe_conversions") {
+ sources = [
+ "numerics/safe_conversions.h",
+ "numerics/safe_conversions_impl.h",
+ ]
+ deps = [
+ ":checks",
+ ]
+}
+
+rtc_source_set("timeutils") {
+ sources = [
+ "timeutils.cc",
+ "timeutils.h",
+ ]
+ deps = [
+ ":checks",
+ ":safe_conversions",
+ ]
+}
+
rtc_source_set("stringutils") {
sources = [
+ "stringencode.cc",
+ "stringencode.h",
"strings/string_builder.cc",
"strings/string_builder.h",
"stringutils.cc",
@@ -154,6 +331,17 @@
]
}
+rtc_source_set("audio_format_to_string") {
+ sources = [
+ "strings/audio_format_to_string.cc",
+ "strings/audio_format_to_string.h",
+ ]
+ deps = [
+ ":stringutils",
+ "../api/audio_codecs:audio_codecs_api",
+ ]
+}
+
rtc_source_set("type_traits") {
sources = [
"type_traits.h",
@@ -170,27 +358,37 @@
rtc_source_set("rtc_base_approved_generic") {
visibility = [
":rtc_base_approved",
- ":rtc_base_approved_objc",
":weak_ptr_unittests",
]
cflags = []
defines = []
libs = []
+ data_deps = []
deps = [
+ ":atomicops",
":checks",
+ ":criticalsection",
+ ":logging",
+ ":macromagic",
+ ":platform_thread",
+ ":platform_thread_types",
+ ":ptr_util",
+ ":refcount",
+ ":rtc_event",
+ ":rtc_task_queue",
":safe_compare",
+ ":safe_conversions",
":stringutils",
+ ":thread_checker",
+ ":timeutils",
":type_traits",
"../:typedefs",
]
sources = [
- "arraysize.h",
- "atomicops.h",
"base64.cc",
"base64.h",
- "basictypes.h",
"bind.h",
"bitbuffer.cc",
"bitbuffer.h",
@@ -202,18 +400,14 @@
"bytebuffer.cc",
"bytebuffer.h",
"byteorder.h",
- "constructormagic.h",
"copyonwritebuffer.cc",
"copyonwritebuffer.h",
- "criticalsection.cc",
- "criticalsection.h",
"event_tracer.cc",
"event_tracer.h",
"file.cc",
"file.h",
"flags.cc",
"flags.h",
- "format_macros.h",
"function_view.h",
"ignore_wundef.h",
"location.cc",
@@ -222,18 +416,11 @@
"numerics/histogram_percentile_counter.h",
"numerics/mod_ops.h",
"numerics/moving_max_counter.h",
- "numerics/safe_conversions.h",
- "numerics/safe_conversions_impl.h",
"onetimeevent.h",
"pathutils.cc",
"pathutils.h",
"platform_file.cc",
"platform_file.h",
- "platform_thread.cc",
- "platform_thread.h",
- "platform_thread_types.cc",
- "platform_thread_types.h",
- "ptr_util.h",
"race_checker.cc",
"race_checker.h",
"random.cc",
@@ -242,25 +429,12 @@
"rate_statistics.h",
"ratetracker.cc",
"ratetracker.h",
- "refcount.h",
- "refcountedobject.h",
- "refcounter.h",
- "scoped_ref_ptr.h",
"string_to_number.cc",
"string_to_number.h",
- "stringencode.cc",
- "stringencode.h",
- "stringize_macros.h",
"swap_queue.h",
"template_util.h",
- "thread_annotations.h",
- "thread_checker.h",
- "thread_checker_impl.cc",
- "thread_checker_impl.h",
"timestampaligner.cc",
"timestampaligner.h",
- "timeutils.cc",
- "timeutils.h",
"trace_event.h",
"zero_memory.cc",
"zero_memory.h",
@@ -276,107 +450,15 @@
libs += [ "log" ]
}
- if (is_posix) {
+ if (is_posix || is_fuchsia) {
sources += [ "file_posix.cc" ]
}
if (is_win) {
sources += [ "file_win.cc" ]
+ data_deps += [ "//build/win:runtime_libs" ]
}
- if (build_with_chromium) {
- # Dependency on chromium's logging (in //base).
- deps += [ "//base:base" ]
- sources += [
- "../../webrtc_overrides/rtc_base/event.cc",
- "../../webrtc_overrides/rtc_base/event.h",
- "../../webrtc_overrides/rtc_base/logging.cc",
- "../../webrtc_overrides/rtc_base/logging.h",
- ]
- } else {
- sources += [
- "event.cc",
- "event.h",
- "logging.cc",
- "logging.h",
- ]
-
- # logging.h needs the deprecation header while downstream projects are
- # removing code that depends on logging implementation details.
- deps += [ ":deprecation" ]
- }
- if (is_component_build && is_win) {
- # Copy the VS runtime DLLs into the isolate so that they don't have to be
- # preinstalled on the target machine. The debug runtimes have a "d" at
- # the end.
- # This is a copy of https://codereview.chromium.org/1783973002.
- # TODO(ehmaldonado): We'd like Chromium to make this changes easier to use,
- # so we don't have to copy their changes and risk breakages.
- # See http://crbug.com/653569
- if (is_debug) {
- vcrt_suffix = "d"
- } else {
- vcrt_suffix = ""
- }
-
- # These runtime files are copied to the output directory by the
- # vs_toolchain script that runs as part of toolchain configuration.
- data = [
- "$root_out_dir/msvcp140${vcrt_suffix}.dll",
- "$root_out_dir/vccorlib140${vcrt_suffix}.dll",
- "$root_out_dir/vcruntime140${vcrt_suffix}.dll",
-
- # Universal Windows 10 CRT files
- "$root_out_dir/api-ms-win-core-console-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-datetime-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-debug-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-errorhandling-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-file-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-file-l1-2-0.dll",
- "$root_out_dir/api-ms-win-core-file-l2-1-0.dll",
- "$root_out_dir/api-ms-win-core-handle-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-heap-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-interlocked-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-libraryloader-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-localization-l1-2-0.dll",
- "$root_out_dir/api-ms-win-core-memory-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-namedpipe-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-processenvironment-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-processthreads-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-processthreads-l1-1-1.dll",
- "$root_out_dir/api-ms-win-core-profile-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-rtlsupport-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-string-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-synch-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-synch-l1-2-0.dll",
- "$root_out_dir/api-ms-win-core-sysinfo-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-timezone-l1-1-0.dll",
- "$root_out_dir/api-ms-win-core-util-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-conio-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-convert-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-environment-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-filesystem-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-heap-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-locale-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-math-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-multibyte-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-private-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-process-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-runtime-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-stdio-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-string-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-time-l1-1-0.dll",
- "$root_out_dir/api-ms-win-crt-utility-l1-1-0.dll",
- "$root_out_dir/ucrtbase${vcrt_suffix}.dll",
- ]
- if (is_asan) {
- if (current_cpu == "x64") {
- data += [ "$clang_base_path/lib/clang/$clang_version/lib/windows/clang_rt.asan_dynamic-x86_64.dll" ]
- } else {
- data += [ "$clang_base_path/lib/clang/$clang_version/lib/windows/clang_rt.asan_dynamic-i386.dll" ]
- }
- }
- }
if (is_nacl) {
deps += [ "//native_client_sdk/src/libraries/nacl_io" ]
}
@@ -395,16 +477,14 @@
"logging_mac.mm",
]
deps = [
- ":rtc_base_approved_generic",
+ ":logging",
]
}
}
rtc_source_set("rtc_task_queue") {
visibility = [ "*" ]
- deps = [
- ":rtc_base_approved",
- ]
+ deps = []
public_deps = [
":rtc_task_queue_api",
]
@@ -421,47 +501,109 @@
# provided implementation should be used. An external implementation should
# depend on rtc_task_queue_api.
rtc_source_set("rtc_task_queue_api") {
+ # The visibility list is commented out so that we won't break external
+ # implementations, but left here to manually test as well as for sake of what
+ # targets we expect to depend on rtc_task_queue_api.
+ # visibility = [
+ # ":rtc_task_queue",
+ # ":rtc_task_queue_impl",
+ # ":sequenced_task_checker",
+ # ]
sources = [
"task_queue.h",
]
deps = [
- ":rtc_base_approved",
+ ":macromagic",
+ ":ptr_util",
]
}
-rtc_source_set("rtc_task_queue_impl") {
- visibility = [ "*" ]
- deps = [
- ":checks",
- ":rtc_base_approved",
- ":rtc_task_queue_api",
- ]
- if (rtc_build_libevent) {
- deps += [ "//base/third_party/libevent" ]
- }
- if (rtc_enable_libevent) {
+if (rtc_enable_libevent) {
+ rtc_source_set("rtc_task_queue_libevent") {
+ visibility = [ ":rtc_task_queue_impl" ]
sources = [
"task_queue_libevent.cc",
"task_queue_posix.cc",
"task_queue_posix.h",
]
+ deps = [
+ ":checks",
+ ":criticalsection",
+ ":logging",
+ ":platform_thread",
+ ":ptr_util",
+ ":refcount",
+ ":rtc_task_queue_api",
+ ":safe_conversions",
+ ":timeutils",
+ ]
+ if (rtc_build_libevent) {
+ deps += [ "//base/third_party/libevent" ]
+ }
+ }
+}
+
+if (is_mac || is_ios) {
+ rtc_source_set("rtc_task_queue_gcd") {
+ visibility = [ ":rtc_task_queue_impl" ]
+ sources = [
+ "task_queue_gcd.cc",
+ "task_queue_posix.cc",
+ "task_queue_posix.h",
+ ]
+ deps = [
+ ":checks",
+ ":logging",
+ ":ptr_util",
+ ":refcount",
+ ":rtc_task_queue_api",
+ ]
+ }
+}
+
+if (is_win) {
+ rtc_source_set("rtc_task_queue_win") {
+ visibility = [ ":rtc_task_queue_impl" ]
+ sources = [
+ "task_queue_win.cc",
+ ]
+ deps = [
+ ":checks",
+ ":criticalsection",
+ ":logging",
+ ":macromagic",
+ ":platform_thread",
+ ":ptr_util",
+ ":refcount",
+ ":rtc_event",
+ ":rtc_task_queue_api",
+ ":safe_conversions",
+ ":timeutils",
+ ]
+ }
+}
+
+rtc_source_set("rtc_task_queue_impl") {
+ visibility = [ "*" ]
+ if (rtc_enable_libevent) {
+ deps = [
+ ":rtc_task_queue_libevent",
+ ]
} else {
if (is_mac || is_ios) {
- sources = [
- "task_queue_gcd.cc",
- "task_queue_posix.cc",
- "task_queue_posix.h",
+ deps = [
+ ":rtc_task_queue_gcd",
]
}
if (is_win) {
- sources = [
- "task_queue_win.cc",
+ deps = [
+ ":rtc_task_queue_win",
]
}
}
}
-rtc_static_library("sequenced_task_checker") {
+rtc_source_set("sequenced_task_checker") {
sources = [
"sequenced_task_checker.h",
"sequenced_task_checker_impl.cc",
@@ -469,8 +611,10 @@
]
deps = [
":checks",
- ":rtc_base_approved",
+ ":criticalsection",
+ ":macromagic",
":rtc_task_queue",
+ ":thread_checker",
]
}
@@ -480,7 +624,8 @@
"weak_ptr.h",
]
deps = [
- ":rtc_base_approved",
+ ":ptr_util",
+ ":refcount",
":sequenced_task_checker",
]
}
@@ -637,17 +782,20 @@
"openssl.h",
"openssladapter.cc",
"openssladapter.h",
+ "opensslcommon.cc",
+ "opensslcommon.h",
"openssldigest.cc",
"openssldigest.h",
"opensslidentity.cc",
"opensslidentity.h",
+ "opensslsessioncache.cc",
+ "opensslsessioncache.h",
"opensslstreamadapter.cc",
"opensslstreamadapter.h",
"physicalsocketserver.cc",
"physicalsocketserver.h",
"proxyinfo.cc",
"proxyinfo.h",
- "ratelimiter.h",
"rtccertificate.cc",
"rtccertificate.h",
"rtccertificategenerator.cc",
@@ -657,6 +805,7 @@
"sigslot.cc",
"sigslot.h",
"sigslotrepeater.h",
+ "socket.cc",
"socket.h",
"socketadapters.cc",
"socketadapters.h",
@@ -802,7 +951,7 @@
defines += [ "_CRT_NONSTDC_NO_DEPRECATE" ]
}
- if (is_posix) {
+ if (is_posix || is_fuchsia) {
sources += [
"ifaddrs_converter.cc",
"ifaddrs_converter.h",
@@ -869,7 +1018,6 @@
"nattypes.h",
"proxyserver.cc",
"proxyserver.h",
- "refcount.h",
"sigslottester.h",
"sigslottester.h.pump",
"testbase64.h",
@@ -964,7 +1112,9 @@
rtc_source_set("rtc_base_approved_unittests") {
testonly = true
-
+ if (is_msan) {
+ cflags = [ "-fsanitize=memory" ]
+ }
sources = [
"atomicops_unittest.cc",
"base64_unittest.cc",
@@ -990,12 +1140,14 @@
"numerics/safe_minmax_unittest.cc",
"onetimeevent_unittest.cc",
"pathutils_unittest.cc",
+ "platform_file_unittest.cc",
"platform_thread_unittest.cc",
"random_unittest.cc",
"rate_limiter_unittest.cc",
"rate_statistics_unittest.cc",
"ratetracker_unittest.cc",
"refcountedobject_unittest.cc",
+ "sanitizer_unittest.cc",
"string_to_number_unittest.cc",
"stringencode_unittest.cc",
"stringize_macros_unittest.cc",
@@ -1019,11 +1171,13 @@
":rtc_task_queue",
":safe_compare",
":safe_minmax",
+ ":sanitizer",
":stringutils",
"../api:array_view",
"../system_wrappers:system_wrappers",
"../test:fileutils",
"../test:test_support",
+ "memory:unittests",
]
}
@@ -1070,6 +1224,7 @@
":rtc_base_approved_generic",
":rtc_base_tests_main",
":rtc_base_tests_utils",
+ ":rtc_event",
":rtc_task_queue",
":weak_ptr",
"../test:test_support",
@@ -1134,9 +1289,11 @@
"win32window_unittest.cc",
]
}
- if (is_posix) {
+ if (is_posix || is_fuchsia) {
sources += [
"openssladapter_unittest.cc",
+ "opensslcommon_unittest.cc",
+ "opensslsessioncache_unittest.cc",
"ssladapter_unittest.cc",
"sslidentity_unittest.cc",
"sslstreamadapter_unittest.cc",
diff --git a/rtc_base/OWNERS b/rtc_base/OWNERS
index 9e8ef1f..f46838c 100644
--- a/rtc_base/OWNERS
+++ b/rtc_base/OWNERS
@@ -1,13 +1,12 @@
-henrikg@webrtc.org
+deadbeef@webrtc.org
hta@webrtc.org
juberti@webrtc.org
+kwiberg@webrtc.org
mflodman@webrtc.org
perkj@webrtc.org
pthatcher@webrtc.org
sergeyu@chromium.org
tommi@webrtc.org
-deadbeef@webrtc.org
-kwiberg@webrtc.org
# These are for the common case of adding or renaming files. If you're doing
# structural changes, please get a review from a reviewer in this file.
diff --git a/rtc_base/asyncpacketsocket.cc b/rtc_base/asyncpacketsocket.cc
index d945039..e1b1eae 100644
--- a/rtc_base/asyncpacketsocket.cc
+++ b/rtc_base/asyncpacketsocket.cc
@@ -12,18 +12,31 @@
namespace rtc {
-PacketTimeUpdateParams::PacketTimeUpdateParams()
- : rtp_sendtime_extension_id(-1),
- srtp_auth_tag_len(-1),
- srtp_packet_index(-1) {
-}
+PacketTimeUpdateParams::PacketTimeUpdateParams() = default;
+
+PacketTimeUpdateParams::PacketTimeUpdateParams(
+ const PacketTimeUpdateParams& other) = default;
PacketTimeUpdateParams::~PacketTimeUpdateParams() = default;
-AsyncPacketSocket::AsyncPacketSocket() {
-}
+PacketOptions::PacketOptions() = default;
+PacketOptions::PacketOptions(DiffServCodePoint dscp) : dscp(dscp) {}
+PacketOptions::PacketOptions(const PacketOptions& other) = default;
+PacketOptions::~PacketOptions() = default;
-AsyncPacketSocket::~AsyncPacketSocket() {
+AsyncPacketSocket::AsyncPacketSocket() = default;
+
+AsyncPacketSocket::~AsyncPacketSocket() = default;
+
+void CopySocketInformationToPacketInfo(size_t packet_size_bytes,
+ const AsyncPacketSocket& socket_from,
+ bool is_connectionless,
+ rtc::PacketInfo* info) {
+ info->packet_size_bytes = packet_size_bytes;
+ info->local_socket_address = socket_from.GetLocalAddress();
+ if (!is_connectionless) {
+ info->remote_socket_address = socket_from.GetRemoteAddress();
+ }
}
}; // namespace rtc
diff --git a/rtc_base/asyncpacketsocket.h b/rtc_base/asyncpacketsocket.h
index 16f4de0..ca59afb 100644
--- a/rtc_base/asyncpacketsocket.h
+++ b/rtc_base/asyncpacketsocket.h
@@ -24,23 +24,30 @@
// after changing the value.
struct PacketTimeUpdateParams {
PacketTimeUpdateParams();
+ PacketTimeUpdateParams(const PacketTimeUpdateParams& other);
~PacketTimeUpdateParams();
- int rtp_sendtime_extension_id; // extension header id present in packet.
+ int rtp_sendtime_extension_id = -1; // extension header id present in packet.
std::vector<char> srtp_auth_key; // Authentication key.
- int srtp_auth_tag_len; // Authentication tag length.
- int64_t srtp_packet_index; // Required for Rtp Packet authentication.
+ int srtp_auth_tag_len = -1; // Authentication tag length.
+ int64_t srtp_packet_index = -1; // Required for Rtp Packet authentication.
};
// This structure holds meta information for the packet which is about to send
// over network.
struct PacketOptions {
- PacketOptions() : dscp(DSCP_NO_CHANGE), packet_id(-1) {}
- explicit PacketOptions(DiffServCodePoint dscp) : dscp(dscp), packet_id(-1) {}
+ PacketOptions();
+ explicit PacketOptions(DiffServCodePoint dscp);
+ PacketOptions(const PacketOptions& other);
+ ~PacketOptions();
- DiffServCodePoint dscp;
- int packet_id; // 16 bits, -1 represents "not set".
+ DiffServCodePoint dscp = DSCP_NO_CHANGE;
+ // When used with RTP packets (for example, webrtc::PacketOptions), the value
+ // should be 16 bits. A value of -1 represents "not set".
+ int64_t packet_id = -1;
PacketTimeUpdateParams packet_time_params;
+ // PacketInfo is passed to SentPacket when signaling this packet is sent.
+ PacketInfo info_signaled_after_sent;
};
// This structure will have the information about when packet is actually
@@ -138,6 +145,11 @@
RTC_DISALLOW_COPY_AND_ASSIGN(AsyncPacketSocket);
};
+void CopySocketInformationToPacketInfo(size_t packet_size_bytes,
+ const AsyncPacketSocket& socket_from,
+ bool is_connectionless,
+ rtc::PacketInfo* info);
+
} // namespace rtc
#endif // RTC_BASE_ASYNCPACKETSOCKET_H_
diff --git a/rtc_base/asynctcpsocket.cc b/rtc_base/asynctcpsocket.cc
index 9e0589c..058d945 100644
--- a/rtc_base/asynctcpsocket.cc
+++ b/rtc_base/asynctcpsocket.cc
@@ -297,7 +297,9 @@
return res;
}
- rtc::SentPacket sent_packet(options.packet_id, rtc::TimeMillis());
+ rtc::SentPacket sent_packet(options.packet_id, rtc::TimeMillis(),
+ options.info_signaled_after_sent);
+ CopySocketInformationToPacketInfo(cb, *this, false, &sent_packet.info);
SignalSentPacket(this, sent_packet);
// We claim to have sent the whole thing, even if we only sent partial
diff --git a/rtc_base/asyncudpsocket.cc b/rtc_base/asyncudpsocket.cc
index 5a50ae3..d8e60b9 100644
--- a/rtc_base/asyncudpsocket.cc
+++ b/rtc_base/asyncudpsocket.cc
@@ -60,7 +60,9 @@
int AsyncUDPSocket::Send(const void *pv, size_t cb,
const rtc::PacketOptions& options) {
- rtc::SentPacket sent_packet(options.packet_id, rtc::TimeMillis());
+ rtc::SentPacket sent_packet(options.packet_id, rtc::TimeMillis(),
+ options.info_signaled_after_sent);
+ CopySocketInformationToPacketInfo(cb, *this, false, &sent_packet.info);
int ret = socket_->Send(pv, cb);
SignalSentPacket(this, sent_packet);
return ret;
@@ -69,7 +71,10 @@
int AsyncUDPSocket::SendTo(const void *pv, size_t cb,
const SocketAddress& addr,
const rtc::PacketOptions& options) {
- rtc::SentPacket sent_packet(options.packet_id, rtc::TimeMillis());
+ rtc::SentPacket sent_packet(options.packet_id, rtc::TimeMillis(),
+ options.info_signaled_after_sent);
+ CopySocketInformationToPacketInfo(cb, *this, true, &sent_packet.info);
+ sent_packet.info.remote_socket_address = addr;
int ret = socket_->SendTo(pv, cb, addr);
SignalSentPacket(this, sent_packet);
return ret;
diff --git a/rtc_base/criticalsection.cc b/rtc_base/criticalsection.cc
index f9168d8..b2bbd03 100644
--- a/rtc_base/criticalsection.cc
+++ b/rtc_base/criticalsection.cc
@@ -10,6 +10,7 @@
#include "rtc_base/criticalsection.h"
+#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
#include "rtc_base/platform_thread_types.h"
diff --git a/rtc_base/criticalsection.h b/rtc_base/criticalsection.h
index 1ef9634..569e147 100644
--- a/rtc_base/criticalsection.h
+++ b/rtc_base/criticalsection.h
@@ -11,7 +11,6 @@
#ifndef RTC_BASE_CRITICALSECTION_H_
#define RTC_BASE_CRITICALSECTION_H_
-#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/platform_thread_types.h"
diff --git a/rtc_base/criticalsection_unittest.cc b/rtc_base/criticalsection_unittest.cc
index f5d6957..6792504 100644
--- a/rtc_base/criticalsection_unittest.cc
+++ b/rtc_base/criticalsection_unittest.cc
@@ -13,6 +13,7 @@
#include <vector>
#include "rtc_base/arraysize.h"
+#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/event.h"
diff --git a/rtc_base/event_tracer.cc b/rtc_base/event_tracer.cc
index 9cbda97..3461ec6 100644
--- a/rtc_base/event_tracer.cc
+++ b/rtc_base/event_tracer.cc
@@ -14,6 +14,7 @@
#include <string>
#include <vector>
+#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/event.h"
diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn
index 305ad24..1ba8827 100644
--- a/rtc_base/experiments/BUILD.gn
+++ b/rtc_base/experiments/BUILD.gn
@@ -19,3 +19,47 @@
"../../system_wrappers:field_trial_api",
]
}
+
+rtc_static_library("congestion_controller_experiment") {
+ sources = [
+ "congestion_controller_experiment.cc",
+ "congestion_controller_experiment.h",
+ ]
+ deps = [
+ "../:rtc_base_approved",
+ "../../api:optional",
+ "../../system_wrappers:field_trial_api",
+ ]
+}
+
+rtc_static_library("quality_scaling_experiment") {
+ sources = [
+ "quality_scaling_experiment.cc",
+ "quality_scaling_experiment.h",
+ ]
+ deps = [
+ "../:rtc_base_approved",
+ "../..:webrtc_common",
+ "../../api:optional",
+ "../../api/video_codecs:video_codecs_api",
+ "../../system_wrappers:field_trial_api",
+ ]
+}
+
+if (rtc_include_tests) {
+ rtc_source_set("experiments_unittests") {
+ testonly = true
+
+ sources = [
+ "congestion_controller_experiment_unittest.cc",
+ "quality_scaling_experiment_unittest.cc",
+ ]
+ deps = [
+ ":congestion_controller_experiment",
+ ":quality_scaling_experiment",
+ "../:rtc_base_tests_main",
+ "../:rtc_base_tests_utils",
+ "../../test:field_trial",
+ ]
+ }
+}
diff --git a/rtc_base/experiments/congestion_controller_experiment.cc b/rtc_base/experiments/congestion_controller_experiment.cc
new file mode 100644
index 0000000..5733fea
--- /dev/null
+++ b/rtc_base/experiments/congestion_controller_experiment.cc
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "rtc_base/experiments/congestion_controller_experiment.h"
+
+#include <string>
+
+#include "system_wrappers/include/field_trial.h"
+
+namespace webrtc {
+namespace {
+
+const char kBbrControllerExperiment[] = "WebRTC-BweCongestionController";
+} // namespace
+
+bool CongestionControllerExperiment::BbrControllerEnabled() {
+ std::string trial_string =
+ webrtc::field_trial::FindFullName(kBbrControllerExperiment);
+ return trial_string.find("Enabled,BBR") == 0;
+}
+
+rtc::Optional<CongestionControllerExperiment::BbrExperimentConfig>
+CongestionControllerExperiment::GetBbrExperimentConfig() {
+ if (!BbrControllerEnabled())
+ return rtc::nullopt;
+ std::string trial_string =
+ webrtc::field_trial::FindFullName(kBbrControllerExperiment);
+ BbrExperimentConfig config;
+ if (sscanf(
+ trial_string.c_str(),
+ "Enabled,BBR,"
+ "%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,"
+ "%lf,%lf,%lf,%lf,%lf,%lf",
+ &config.exit_startup_on_loss, &config.exit_startup_rtt_threshold_ms,
+ &config.fully_drain_queue, &config.initial_conservation_in_startup,
+ &config.num_startup_rtts, &config.probe_rtt_based_on_bdp,
+ &config.probe_rtt_disabled_if_app_limited,
+ &config.probe_rtt_skipped_if_similar_rtt, &config.rate_based_recovery,
+ &config.rate_based_startup, &config.slower_startup,
+ &config.encoder_rate_gain, &config.encoder_rate_gain_in_probe_rtt,
+ &config.max_ack_height_window_multiplier,
+ &config.max_aggregation_bytes_multiplier,
+ &config.probe_bw_pacing_gain_offset,
+ &config.probe_rtt_congestion_window_gain) == 17) {
+ return config;
+ } else {
+ return rtc::nullopt;
+ }
+}
+
+} // namespace webrtc
diff --git a/rtc_base/experiments/congestion_controller_experiment.h b/rtc_base/experiments/congestion_controller_experiment.h
new file mode 100644
index 0000000..478d106
--- /dev/null
+++ b/rtc_base/experiments/congestion_controller_experiment.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef RTC_BASE_EXPERIMENTS_CONGESTION_CONTROLLER_EXPERIMENT_H_
+#define RTC_BASE_EXPERIMENTS_CONGESTION_CONTROLLER_EXPERIMENT_H_
+#include <api/optional.h>
+namespace webrtc {
+class CongestionControllerExperiment {
+ public:
+ struct BbrExperimentConfig {
+ int exit_startup_on_loss;
+ int exit_startup_rtt_threshold_ms;
+ int fully_drain_queue;
+ int initial_conservation_in_startup;
+ int num_startup_rtts;
+ int probe_rtt_based_on_bdp;
+ int probe_rtt_disabled_if_app_limited;
+ int probe_rtt_skipped_if_similar_rtt;
+ int rate_based_recovery;
+ int rate_based_startup;
+ int slower_startup;
+ double encoder_rate_gain;
+ double encoder_rate_gain_in_probe_rtt;
+ double max_ack_height_window_multiplier;
+ double max_aggregation_bytes_multiplier;
+ double probe_bw_pacing_gain_offset;
+ double probe_rtt_congestion_window_gain;
+ };
+ static bool BbrControllerEnabled();
+ static rtc::Optional<BbrExperimentConfig> GetBbrExperimentConfig();
+};
+
+} // namespace webrtc
+
+#endif // RTC_BASE_EXPERIMENTS_CONGESTION_CONTROLLER_EXPERIMENT_H_
diff --git a/rtc_base/experiments/congestion_controller_experiment_unittest.cc b/rtc_base/experiments/congestion_controller_experiment_unittest.cc
new file mode 100644
index 0000000..b8e1fff
--- /dev/null
+++ b/rtc_base/experiments/congestion_controller_experiment_unittest.cc
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/experiments/congestion_controller_experiment.h"
+#include "rtc_base/gunit.h"
+#include "test/field_trial.h"
+
+namespace webrtc {
+namespace {
+void ExpectEquals(CongestionControllerExperiment::BbrExperimentConfig a,
+ CongestionControllerExperiment::BbrExperimentConfig b) {
+ EXPECT_EQ(a.exit_startup_on_loss, b.exit_startup_on_loss);
+ EXPECT_EQ(a.exit_startup_rtt_threshold_ms, b.exit_startup_rtt_threshold_ms);
+ EXPECT_EQ(a.fully_drain_queue, b.fully_drain_queue);
+ EXPECT_EQ(a.initial_conservation_in_startup,
+ b.initial_conservation_in_startup);
+ EXPECT_EQ(a.num_startup_rtts, b.num_startup_rtts);
+ EXPECT_EQ(a.probe_rtt_based_on_bdp, b.probe_rtt_based_on_bdp);
+ EXPECT_EQ(a.probe_rtt_disabled_if_app_limited,
+ b.probe_rtt_disabled_if_app_limited);
+ EXPECT_EQ(a.probe_rtt_skipped_if_similar_rtt,
+ b.probe_rtt_skipped_if_similar_rtt);
+ EXPECT_EQ(a.rate_based_recovery, b.rate_based_recovery);
+ EXPECT_EQ(a.rate_based_startup, b.rate_based_startup);
+ EXPECT_EQ(a.slower_startup, b.slower_startup);
+ EXPECT_EQ(a.encoder_rate_gain, b.encoder_rate_gain);
+ EXPECT_EQ(a.encoder_rate_gain_in_probe_rtt, b.encoder_rate_gain_in_probe_rtt);
+ EXPECT_EQ(a.max_ack_height_window_multiplier,
+ b.max_ack_height_window_multiplier);
+ EXPECT_EQ(a.max_aggregation_bytes_multiplier,
+ b.max_aggregation_bytes_multiplier);
+ EXPECT_EQ(a.probe_bw_pacing_gain_offset, b.probe_bw_pacing_gain_offset);
+ EXPECT_EQ(a.probe_rtt_congestion_window_gain,
+ b.probe_rtt_congestion_window_gain);
+}
+} // namespace
+
+TEST(CongestionControllerExperimentTest, BbrDisabledByDefault) {
+ webrtc::test::ScopedFieldTrials field_trials("");
+ EXPECT_FALSE(CongestionControllerExperiment::BbrControllerEnabled());
+}
+
+TEST(CongestionControllerExperimentTest, BbrEnabledByFieldTrial) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-BweCongestionController/Enabled,BBR/");
+ EXPECT_TRUE(CongestionControllerExperiment::BbrControllerEnabled());
+}
+
+TEST(CongestionControllerExperimentTest, BbrBadParametersFails) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-BweCongestionController/Enabled,BBR,"
+ "garbage,here/");
+ EXPECT_FALSE(CongestionControllerExperiment::GetBbrExperimentConfig());
+}
+
+TEST(CongestionControllerExperimentTest, BbrZeroParametersParsed) {
+ CongestionControllerExperiment::BbrExperimentConfig truth = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-BweCongestionController/Enabled,BBR,"
+ "0,0,0,0,0,0,0,0,0,0,0,"
+ "0,0,0,0,0,0/");
+ auto config = CongestionControllerExperiment::GetBbrExperimentConfig();
+ EXPECT_TRUE(config);
+ if (config)
+ ExpectEquals(truth, *config);
+}
+
+TEST(CongestionControllerExperimentTest, BbrNonZeroParametersParsed) {
+ CongestionControllerExperiment::BbrExperimentConfig truth = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25};
+
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-BweCongestionController/Enabled,BBR,"
+ "1,1,1,1,1,1,1,1,1,1,1,"
+ "0.25,0.25,0.25,0.25,0.25,0.25/");
+ auto config = CongestionControllerExperiment::GetBbrExperimentConfig();
+ EXPECT_TRUE(config);
+ if (config)
+ ExpectEquals(truth, *config);
+}
+} // namespace webrtc
diff --git a/rtc_base/experiments/quality_scaling_experiment.cc b/rtc_base/experiments/quality_scaling_experiment.cc
new file mode 100644
index 0000000..8c99954
--- /dev/null
+++ b/rtc_base/experiments/quality_scaling_experiment.cc
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "rtc_base/experiments/quality_scaling_experiment.h"
+
+#include <string>
+
+#include "rtc_base/logging.h"
+#include "system_wrappers/include/field_trial.h"
+
+namespace webrtc {
+namespace {
+constexpr char kFieldTrial[] = "WebRTC-Video-QualityScaling";
+constexpr int kMinQp = 1;
+constexpr int kMaxVp8Qp = 127;
+constexpr int kMaxVp9Qp = 255;
+constexpr int kMaxH264Qp = 51;
+constexpr int kMaxGenericQp = 255;
+
+rtc::Optional<VideoEncoder::QpThresholds> GetThresholds(int low,
+ int high,
+ int max) {
+ if (low < kMinQp || high > max || high < low)
+ return rtc::nullopt;
+
+ RTC_LOG(LS_INFO) << "QP thresholds: low: " << low << ", high: " << high;
+ return rtc::Optional<VideoEncoder::QpThresholds>(
+ VideoEncoder::QpThresholds(low, high));
+}
+} // namespace
+
+bool QualityScalingExperiment::Enabled() {
+ return webrtc::field_trial::IsEnabled(kFieldTrial);
+}
+
+rtc::Optional<QualityScalingExperiment::Settings>
+QualityScalingExperiment::ParseSettings() {
+ const std::string group = webrtc::field_trial::FindFullName(kFieldTrial);
+ if (group.empty())
+ return rtc::nullopt;
+
+ Settings s;
+ if (sscanf(group.c_str(), "Enabled-%d,%d,%d,%d,%d,%d,%d,%d,%f,%f,%d",
+ &s.vp8_low, &s.vp8_high, &s.vp9_low, &s.vp9_high, &s.h264_low,
+ &s.h264_high, &s.generic_low, &s.generic_high, &s.alpha_high,
+ &s.alpha_low, &s.drop) != 11) {
+ RTC_LOG(LS_WARNING) << "Invalid number of parameters provided.";
+ return rtc::nullopt;
+ }
+ return s;
+}
+
+rtc::Optional<VideoEncoder::QpThresholds>
+QualityScalingExperiment::GetQpThresholds(VideoCodecType codec_type) {
+ const auto settings = ParseSettings();
+ if (!settings)
+ return rtc::nullopt;
+
+ switch (codec_type) {
+ case kVideoCodecVP8:
+ return GetThresholds(settings->vp8_low, settings->vp8_high, kMaxVp8Qp);
+ case kVideoCodecVP9:
+ return GetThresholds(settings->vp9_low, settings->vp9_high, kMaxVp9Qp);
+ case kVideoCodecH264:
+ return GetThresholds(settings->h264_low, settings->h264_high, kMaxH264Qp);
+ case kVideoCodecGeneric:
+ return GetThresholds(settings->generic_low, settings->generic_high,
+ kMaxGenericQp);
+ default:
+ return rtc::nullopt;
+ }
+}
+
+QualityScalingExperiment::Config QualityScalingExperiment::GetConfig() {
+ const auto settings = ParseSettings();
+ if (!settings)
+ return Config();
+
+ Config config;
+ config.use_all_drop_reasons = settings->drop > 0;
+
+ if (settings->alpha_high < 0 || settings->alpha_low < settings->alpha_high) {
+ RTC_LOG(LS_WARNING) << "Invalid alpha value provided, using default.";
+ return config;
+ }
+ config.alpha_high = settings->alpha_high;
+ config.alpha_low = settings->alpha_low;
+ return config;
+}
+
+} // namespace webrtc
diff --git a/rtc_base/experiments/quality_scaling_experiment.h b/rtc_base/experiments/quality_scaling_experiment.h
new file mode 100644
index 0000000..6f24d6b
--- /dev/null
+++ b/rtc_base/experiments/quality_scaling_experiment.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef RTC_BASE_EXPERIMENTS_QUALITY_SCALING_EXPERIMENT_H_
+#define RTC_BASE_EXPERIMENTS_QUALITY_SCALING_EXPERIMENT_H_
+
+#include "api/optional.h"
+#include "api/video_codecs/video_encoder.h"
+#include "common_types.h" // NOLINT(build/include)
+
+namespace webrtc {
+class QualityScalingExperiment {
+ public:
+ struct Settings {
+ int vp8_low; // VP8: low QP threshold.
+ int vp8_high; // VP8: high QP threshold.
+ int vp9_low; // VP9: low QP threshold.
+ int vp9_high; // VP9: high QP threshold.
+ int h264_low; // H264: low QP threshold.
+ int h264_high; // H264: high QP threshold.
+ int generic_low; // Generic: low QP threshold.
+ int generic_high; // Generic: high QP threshold.
+ float alpha_high; // |alpha_| for ExpFilter used when checking high QP.
+ float alpha_low; // |alpha_| for ExpFilter used when checking low QP.
+ int drop; // >0 sets |use_all_drop_reasons| to true.
+ };
+
+ // Used by QualityScaler.
+ struct Config {
+ float alpha_high = 0.9995f;
+ float alpha_low = 0.9999f;
+ // If set, all type of dropped frames are used.
+ // Otherwise only dropped frames by MediaOptimization are used.
+ bool use_all_drop_reasons = false;
+ };
+
+ // Returns true if the experiment is enabled.
+ static bool Enabled();
+
+ // Returns settings from field trial.
+ static rtc::Optional<Settings> ParseSettings();
+
+ // Returns QpThresholds for the |codec_type|.
+ static rtc::Optional<VideoEncoder::QpThresholds> GetQpThresholds(
+ VideoCodecType codec_type);
+
+ // Returns parsed values. If the parsing fails, default values are returned.
+ static Config GetConfig();
+};
+
+} // namespace webrtc
+
+#endif // RTC_BASE_EXPERIMENTS_QUALITY_SCALING_EXPERIMENT_H_
diff --git a/rtc_base/experiments/quality_scaling_experiment_unittest.cc b/rtc_base/experiments/quality_scaling_experiment_unittest.cc
new file mode 100644
index 0000000..15aadd6
--- /dev/null
+++ b/rtc_base/experiments/quality_scaling_experiment_unittest.cc
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/experiments/quality_scaling_experiment.h"
+
+#include "rtc_base/gunit.h"
+#include "test/field_trial.h"
+
+namespace webrtc {
+namespace {
+void ExpectEqualSettings(QualityScalingExperiment::Settings a,
+ QualityScalingExperiment::Settings b) {
+ EXPECT_EQ(a.vp8_low, b.vp8_low);
+ EXPECT_EQ(a.vp8_high, b.vp8_high);
+ EXPECT_EQ(a.vp9_low, b.vp9_low);
+ EXPECT_EQ(a.vp9_high, b.vp9_high);
+ EXPECT_EQ(a.h264_low, b.h264_low);
+ EXPECT_EQ(a.h264_high, b.h264_high);
+ EXPECT_EQ(a.generic_low, b.generic_low);
+ EXPECT_EQ(a.generic_high, b.generic_high);
+ EXPECT_EQ(a.alpha_high, b.alpha_high);
+ EXPECT_EQ(a.alpha_low, b.alpha_low);
+ EXPECT_EQ(a.drop, b.drop);
+}
+
+void ExpectEqualConfig(QualityScalingExperiment::Config a,
+ QualityScalingExperiment::Config b) {
+ EXPECT_EQ(a.alpha_high, b.alpha_high);
+ EXPECT_EQ(a.alpha_low, b.alpha_low);
+ EXPECT_EQ(a.use_all_drop_reasons, b.use_all_drop_reasons);
+}
+} // namespace
+
+TEST(QualityScalingExperimentTest, DisabledWithoutFieldTrial) {
+ webrtc::test::ScopedFieldTrials field_trials("");
+ EXPECT_FALSE(QualityScalingExperiment::Enabled());
+}
+
+TEST(QualityScalingExperimentTest, EnabledWithFieldTrial) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled/");
+ EXPECT_TRUE(QualityScalingExperiment::Enabled());
+}
+
+TEST(QualityScalingExperimentTest, ParseSettings) {
+ const QualityScalingExperiment::Settings kExpected = {1, 2, 3, 4, 5, 6,
+ 7, 8, 0.9f, 0.99f, 1};
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,7,8,0.9,0.99,1/");
+ const auto settings = QualityScalingExperiment::ParseSettings();
+ EXPECT_TRUE(settings);
+ ExpectEqualSettings(kExpected, *settings);
+}
+
+TEST(QualityScalingExperimentTest, ParseSettingsFailsWithoutFieldTrial) {
+ webrtc::test::ScopedFieldTrials field_trials("");
+ EXPECT_FALSE(QualityScalingExperiment::ParseSettings());
+}
+
+TEST(QualityScalingExperimentTest, ParseSettingsFailsWithInvalidFieldTrial) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-invalid/");
+ EXPECT_FALSE(QualityScalingExperiment::ParseSettings());
+}
+
+TEST(QualityScalingExperimentTest, GetConfig) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,7,8,0.9,0.99,0/");
+ const auto config = QualityScalingExperiment::GetConfig();
+ EXPECT_EQ(0.9f, config.alpha_high);
+ EXPECT_EQ(0.99f, config.alpha_low);
+ EXPECT_FALSE(config.use_all_drop_reasons);
+}
+
+TEST(QualityScalingExperimentTest, GetsDefaultConfigForInvalidFieldTrial) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-invalid/");
+ const auto config = QualityScalingExperiment::GetConfig();
+ ExpectEqualConfig(config, QualityScalingExperiment::Config());
+}
+
+TEST(QualityScalingExperimentTest, GetsDefaultAlphaForInvalidValue) {
+ QualityScalingExperiment::Config expected_config;
+ expected_config.use_all_drop_reasons = true;
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,7,8,0.99,0.9,1/");
+ const auto config = QualityScalingExperiment::GetConfig();
+ ExpectEqualConfig(config, expected_config);
+}
+
+TEST(QualityScalingExperimentTest, GetVp8Thresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,0,0,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecVP8);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(1, thresholds->low);
+ EXPECT_EQ(2, thresholds->high);
+}
+
+TEST(QualityScalingExperimentTest, GetThresholdsFailsForInvalidVp8Value) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-0,0,3,4,5,6,7,8,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecVP8);
+ EXPECT_FALSE(thresholds);
+}
+
+TEST(QualityScalingExperimentTest, GetVp9Thresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,0,0,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecVP9);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(3, thresholds->low);
+ EXPECT_EQ(4, thresholds->high);
+}
+
+TEST(QualityScalingExperimentTest, GetThresholdsFailsForInvalidVp9Value) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,0,0,5,6,7,8,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecVP9);
+ EXPECT_FALSE(thresholds);
+}
+
+TEST(QualityScalingExperimentTest, GetH264Thresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,0,0,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecH264);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(5, thresholds->low);
+ EXPECT_EQ(6, thresholds->high);
+}
+
+TEST(QualityScalingExperimentTest, GetThresholdsFailsForInvalidH264Value) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,0,0,7,8,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecH264);
+ EXPECT_FALSE(thresholds);
+}
+
+TEST(QualityScalingExperimentTest, GetGenericThresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,0,0,7,8,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecGeneric);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(7, thresholds->low);
+ EXPECT_EQ(8, thresholds->high);
+}
+
+TEST(QualityScalingExperimentTest, GetThresholdsFailsForInvalidGenericValue) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,0,0,0.9,0.99,1/");
+ const auto thresholds =
+ QualityScalingExperiment::GetQpThresholds(kVideoCodecGeneric);
+ EXPECT_FALSE(thresholds);
+}
+} // namespace webrtc
diff --git a/rtc_base/format_macros.h b/rtc_base/format_macros.h
index d252a94..48127e1 100644
--- a/rtc_base/format_macros.h
+++ b/rtc_base/format_macros.h
@@ -24,8 +24,6 @@
// printf("xyz: %" PRIuS, size);
// The "u" in the macro corresponds to %u, and S is for "size".
-#include "typedefs.h" // NOLINT(build/include)
-
#if defined(WEBRTC_POSIX)
#if (defined(_INTTYPES_H) || defined(_INTTYPES_H_)) && !defined(PRId64)
diff --git a/rtc_base/ifaddrs-android.cc b/rtc_base/ifaddrs-android.cc
index 85a4497..b713c02 100644
--- a/rtc_base/ifaddrs-android.cc
+++ b/rtc_base/ifaddrs-android.cc
@@ -174,24 +174,24 @@
rtattr* rta = IFA_RTA(address_msg);
ssize_t payload_len = IFA_PAYLOAD(header);
while (RTA_OK(rta, payload_len)) {
- if (rta->rta_type == IFA_ADDRESS) {
- int family = address_msg->ifa_family;
- if (family == AF_INET || family == AF_INET6) {
- ifaddrs* newest = new ifaddrs;
- memset(newest, 0, sizeof(ifaddrs));
- if (current) {
- current->ifa_next = newest;
- } else {
- start = newest;
- }
- if (populate_ifaddrs(newest, address_msg, RTA_DATA(rta),
- RTA_PAYLOAD(rta)) != 0) {
- freeifaddrs(start);
- *result = nullptr;
- return -1;
- }
- current = newest;
+ if ((address_msg->ifa_family == AF_INET &&
+ rta->rta_type == IFA_LOCAL) ||
+ (address_msg->ifa_family == AF_INET6 &&
+ rta->rta_type == IFA_ADDRESS)) {
+ ifaddrs* newest = new ifaddrs;
+ memset(newest, 0, sizeof(ifaddrs));
+ if (current) {
+ current->ifa_next = newest;
+ } else {
+ start = newest;
}
+ if (populate_ifaddrs(newest, address_msg, RTA_DATA(rta),
+ RTA_PAYLOAD(rta)) != 0) {
+ freeifaddrs(start);
+ *result = nullptr;
+ return -1;
+ }
+ current = newest;
}
rta = RTA_NEXT(rta, payload_len);
}
diff --git a/rtc_base/ifaddrs_converter.cc b/rtc_base/ifaddrs_converter.cc
index 2db99ef..586b4e9 100644
--- a/rtc_base/ifaddrs_converter.cc
+++ b/rtc_base/ifaddrs_converter.cc
@@ -22,8 +22,8 @@
IPAddress* mask) {
switch (interface->ifa_addr->sa_family) {
case AF_INET: {
- *ip = IPAddress(
- reinterpret_cast<sockaddr_in*>(interface->ifa_addr)->sin_addr);
+ *ip = InterfaceAddress(IPAddress(
+ reinterpret_cast<sockaddr_in*>(interface->ifa_addr)->sin_addr));
*mask = IPAddress(
reinterpret_cast<sockaddr_in*>(interface->ifa_netmask)->sin_addr);
return true;
diff --git a/rtc_base/ipaddress.cc b/rtc_base/ipaddress.cc
index d441f07..e6a02c4 100644
--- a/rtc_base/ipaddress.cc
+++ b/rtc_base/ipaddress.cc
@@ -120,11 +120,6 @@
return false;
}
-std::ostream& operator<<(std::ostream& os, const IPAddress& ip) {
- os << ip.ToString();
- return os;
-}
-
in6_addr IPAddress::ipv6_address() const {
return u_.ip6;
}
@@ -216,13 +211,13 @@
return *this;
}
-std::ostream& operator<<(std::ostream& os, const InterfaceAddress& ip) {
- os << static_cast<const IPAddress&>(ip);
+std::string InterfaceAddress::ToString() const {
+ std::string result = IPAddress::ToString();
- if (ip.family() == AF_INET6)
- os << "|flags:0x" << std::hex << ip.ipv6_flags();
+ if (family() == AF_INET6)
+ result += "|flags:0x" + rtc::ToHex(ipv6_flags());
- return os;
+ return result;
}
static bool IPIsPrivateNetworkV4(const IPAddress& ip) {
diff --git a/rtc_base/ipaddress.h b/rtc_base/ipaddress.h
index 4ef7d08..c965cf1 100644
--- a/rtc_base/ipaddress.h
+++ b/rtc_base/ipaddress.h
@@ -83,7 +83,13 @@
bool operator!=(const IPAddress& other) const;
bool operator <(const IPAddress& other) const;
bool operator >(const IPAddress& other) const;
- friend std::ostream& operator<<(std::ostream& os, const IPAddress& addr);
+
+#ifdef UNIT_TEST
+ inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982)
+ std::ostream& os) { // no-presubmit-check TODO(webrtc:8982)
+ return os << ToString();
+ }
+#endif // UNIT_TEST
int family() const { return family_; }
in_addr ipv4_address() const;
@@ -126,8 +132,8 @@
public:
InterfaceAddress() : ipv6_flags_(IPV6_ADDRESS_FLAG_NONE) {}
- InterfaceAddress(IPAddress ip)
- : IPAddress(ip), ipv6_flags_(IPV6_ADDRESS_FLAG_NONE) {}
+ explicit InterfaceAddress(IPAddress ip)
+ : IPAddress(ip), ipv6_flags_(IPV6_ADDRESS_FLAG_NONE) {}
InterfaceAddress(IPAddress addr, int ipv6_flags)
: IPAddress(addr), ipv6_flags_(ipv6_flags) {}
@@ -141,8 +147,8 @@
bool operator!=(const InterfaceAddress& other) const;
int ipv6_flags() const { return ipv6_flags_; }
- friend std::ostream& operator<<(std::ostream& os,
- const InterfaceAddress& addr);
+
+ std::string ToString() const;
private:
int ipv6_flags_;
diff --git a/rtc_base/ipaddress_unittest.cc b/rtc_base/ipaddress_unittest.cc
index 90c9559..3698d3f 100644
--- a/rtc_base/ipaddress_unittest.cc
+++ b/rtc_base/ipaddress_unittest.cc
@@ -288,39 +288,39 @@
IPAddress addr(v4addr);
IPAddress addr2(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(INADDR_ANY);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(INADDR_LOOPBACK);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(kIPv4PublicAddr);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(kIPv4RFC1918Addr);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(in6addr_any);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(in6addr_loopback);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(kIPv6LinkLocalAddr);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr = IPAddress(kIPv6PublicAddr);
addr2 = IPAddress(addr);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
}
TEST(IPAddressTest, TestEquality) {
@@ -446,46 +446,46 @@
EXPECT_TRUE(IPFromString(kIPv4AnyAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv4AnyAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(INADDR_LOOPBACK);
EXPECT_TRUE(IPFromString(kIPv4LoopbackAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv4LoopbackAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(kIPv4RFC1918Addr);
EXPECT_TRUE(IPFromString(kIPv4RFC1918AddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv4RFC1918AddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(kIPv4PublicAddr);
EXPECT_TRUE(IPFromString(kIPv4PublicAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv4PublicAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(in6addr_any);
EXPECT_TRUE(IPFromString(kIPv6AnyAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv6AnyAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(in6addr_loopback);
EXPECT_TRUE(IPFromString(kIPv6LoopbackAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv6LoopbackAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(kIPv6LinkLocalAddr);
EXPECT_TRUE(IPFromString(kIPv6LinkLocalAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv6LinkLocalAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(kIPv6PublicAddr);
EXPECT_TRUE(IPFromString(kIPv6PublicAddrString, &addr));
EXPECT_EQ(addr.ToString(), kIPv6PublicAddrString);
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
addr2 = IPAddress(kIPv4MappedRFC1918Addr);
EXPECT_TRUE(IPFromString(kIPv4MappedV4StyleAddrString, &addr));
- EXPECT_PRED2(AreEqual, addr, addr2);
+ EXPECT_TRUE(AreEqual(addr, addr2));
// Broken cases, should set addr to AF_UNSPEC.
EXPECT_PRED1(BrokenIPStringFails, kIPv4BrokenString1);
diff --git a/rtc_base/java/src/org/webrtc/ThreadUtils.java b/rtc_base/java/src/org/webrtc/ThreadUtils.java
index 3cc80d3..a403870 100644
--- a/rtc_base/java/src/org/webrtc/ThreadUtils.java
+++ b/rtc_base/java/src/org/webrtc/ThreadUtils.java
@@ -16,13 +16,14 @@
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
public class ThreadUtils {
/**
* Utility class to be used for checking that a method is called on the correct thread.
*/
public static class ThreadChecker {
- private Thread thread = Thread.currentThread();
+ @Nullable private Thread thread = Thread.currentThread();
public void checkIsOnValidThread() {
if (thread == null) {
diff --git a/rtc_base/memory/BUILD.gn b/rtc_base/memory/BUILD.gn
new file mode 100644
index 0000000..e24b8ce
--- /dev/null
+++ b/rtc_base/memory/BUILD.gn
@@ -0,0 +1,47 @@
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../webrtc.gni")
+if (is_android) {
+ import("//build/config/android/config.gni")
+ import("//build/config/android/rules.gni")
+}
+
+rtc_source_set("aligned_array") {
+ sources = [
+ "aligned_array.h",
+ ]
+ deps = [
+ ":aligned_malloc",
+ "..:checks",
+ ]
+}
+
+rtc_source_set("aligned_malloc") {
+ sources = [
+ "aligned_malloc.cc",
+ "aligned_malloc.h",
+ ]
+ deps = [
+ "../..:typedefs",
+ ]
+}
+
+rtc_source_set("unittests") {
+ testonly = true
+ sources = [
+ "aligned_array_unittest.cc",
+ "aligned_malloc_unittest.cc",
+ ]
+ deps = [
+ ":aligned_array",
+ ":aligned_malloc",
+ "../..:typedefs",
+ "../../test:test_support",
+ ]
+}
diff --git a/system_wrappers/include/aligned_array.h b/rtc_base/memory/aligned_array.h
similarity index 88%
rename from system_wrappers/include/aligned_array.h
rename to rtc_base/memory/aligned_array.h
index 793c785..dcdef12 100644
--- a/system_wrappers/include/aligned_array.h
+++ b/rtc_base/memory/aligned_array.h
@@ -8,11 +8,11 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef WEBRTC_SYSTEM_WRAPPERS_INCLUDE_ALIGNED_ARRAY_
-#define WEBRTC_SYSTEM_WRAPPERS_INCLUDE_ALIGNED_ARRAY_
+#ifndef RTC_BASE_MEMORY_ALIGNED_ARRAY_H_
+#define RTC_BASE_MEMORY_ALIGNED_ARRAY_H_
#include "rtc_base/checks.h"
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
namespace webrtc {
@@ -75,4 +75,4 @@
} // namespace webrtc
-#endif // WEBRTC_SYSTEM_WRAPPERS_INCLUDE_ALIGNED_ARRAY_
+#endif // RTC_BASE_MEMORY_ALIGNED_ARRAY_H_
diff --git a/system_wrappers/source/aligned_array_unittest.cc b/rtc_base/memory/aligned_array_unittest.cc
similarity index 96%
rename from system_wrappers/source/aligned_array_unittest.cc
rename to rtc_base/memory/aligned_array_unittest.cc
index e5a3c18..81fd468 100644
--- a/system_wrappers/source/aligned_array_unittest.cc
+++ b/rtc_base/memory/aligned_array_unittest.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/include/aligned_array.h"
+#include "rtc_base/memory/aligned_array.h"
#include <stdint.h>
diff --git a/system_wrappers/source/aligned_malloc.cc b/rtc_base/memory/aligned_malloc.cc
similarity index 97%
rename from system_wrappers/source/aligned_malloc.cc
rename to rtc_base/memory/aligned_malloc.cc
index 43ece9e..4e1e85c 100644
--- a/system_wrappers/source/aligned_malloc.cc
+++ b/rtc_base/memory/aligned_malloc.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
#include <memory.h>
#include <stdlib.h>
diff --git a/system_wrappers/include/aligned_malloc.h b/rtc_base/memory/aligned_malloc.h
similarity index 92%
rename from system_wrappers/include/aligned_malloc.h
rename to rtc_base/memory/aligned_malloc.h
index 33b23d2..42a6daa 100644
--- a/system_wrappers/include/aligned_malloc.h
+++ b/rtc_base/memory/aligned_malloc.h
@@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_INCLUDE_ALIGNED_MALLOC_H_
-#define SYSTEM_WRAPPERS_INCLUDE_ALIGNED_MALLOC_H_
+#ifndef RTC_BASE_MEMORY_ALIGNED_MALLOC_H_
+#define RTC_BASE_MEMORY_ALIGNED_MALLOC_H_
// The functions declared here
// 1) Allocates block of aligned memory.
@@ -54,4 +54,4 @@
} // namespace webrtc
-#endif // SYSTEM_WRAPPERS_INCLUDE_ALIGNED_MALLOC_H_
+#endif // RTC_BASE_MEMORY_ALIGNED_MALLOC_H_
diff --git a/system_wrappers/source/aligned_malloc_unittest.cc b/rtc_base/memory/aligned_malloc_unittest.cc
similarity index 97%
rename from system_wrappers/source/aligned_malloc_unittest.cc
rename to rtc_base/memory/aligned_malloc_unittest.cc
index 7afbf6d..742a772 100644
--- a/system_wrappers/source/aligned_malloc_unittest.cc
+++ b/rtc_base/memory/aligned_malloc_unittest.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/include/aligned_malloc.h"
+#include "rtc_base/memory/aligned_malloc.h"
#include <memory>
diff --git a/rtc_base/memory/module.mk b/rtc_base/memory/module.mk
new file mode 100644
index 0000000..37c23f4
--- /dev/null
+++ b/rtc_base/memory/module.mk
@@ -0,0 +1,5 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+include common.mk
diff --git a/rtc_base/module.mk b/rtc_base/module.mk
index 8bf4804..0f8f8ca 100644
--- a/rtc_base/module.mk
+++ b/rtc_base/module.mk
@@ -42,6 +42,8 @@
rtc_base_CXX_OBJECTS = \
rtc_base/checks.o \
+ rtc_base/memory/aligned_malloc.o \
+ rtc_base/system/file_wrapper.o \
$(rtc_base_approved_generic_CXX_OBJECTS) \
$(task_queue_impl_CXX_OBJECTS)
diff --git a/rtc_base/nat_unittest.cc b/rtc_base/nat_unittest.cc
index d68df1d..68f0f1c 100644
--- a/rtc_base/nat_unittest.cc
+++ b/rtc_base/nat_unittest.cc
@@ -232,11 +232,12 @@
}
}
if (ext_addr2.IsNil()) {
- RTC_LOG(LS_WARNING) << "No available IP of same family as " << int_addr;
+ RTC_LOG(LS_WARNING) << "No available IP of same family as "
+ << int_addr.ToString();
return;
}
- RTC_LOG(LS_INFO) << "selected ip " << ext_addr2.ipaddr();
+ RTC_LOG(LS_INFO) << "selected ip " << ext_addr2.ipaddr().ToString();
SocketAddress ext_addrs[4] = {
SocketAddress(ext_addr1),
diff --git a/rtc_base/network.cc b/rtc_base/network.cc
index 6d59888..6daa7c3 100644
--- a/rtc_base/network.cc
+++ b/rtc_base/network.cc
@@ -24,7 +24,7 @@
#if defined(WEBRTC_WIN)
#include "rtc_base/win32.h"
-#include <Iphlpapi.h>
+#include <iphlpapi.h>
#elif !defined(__native_client__)
#include "rtc_base/ifaddrs_converter.h"
#endif
@@ -111,6 +111,24 @@
}
}
+uint16_t ComputeNetworkCostByType(int type) {
+ switch (type) {
+ case rtc::ADAPTER_TYPE_ETHERNET:
+ case rtc::ADAPTER_TYPE_LOOPBACK:
+ return kNetworkCostMin;
+ case rtc::ADAPTER_TYPE_WIFI:
+ return kNetworkCostLow;
+ case rtc::ADAPTER_TYPE_CELLULAR:
+ return kNetworkCostHigh;
+ case rtc::ADAPTER_TYPE_VPN:
+ // The cost of a VPN should be computed using its underlying network type.
+ RTC_NOTREACHED();
+ return kNetworkCostUnknown;
+ default:
+ return kNetworkCostUnknown;
+ }
+}
+
#if !defined(__native_client__)
bool IsIgnoredIPv6(const InterfaceAddress& ip) {
if (ip.family() != AF_INET6) {
@@ -476,6 +494,7 @@
}
AdapterType adapter_type = ADAPTER_TYPE_UNKNOWN;
+ AdapterType vpn_underlying_adapter_type = ADAPTER_TYPE_UNKNOWN;
if (cursor->ifa_flags & IFF_LOOPBACK) {
adapter_type = ADAPTER_TYPE_LOOPBACK;
} else {
@@ -488,6 +507,11 @@
adapter_type = GetAdapterTypeFromName(cursor->ifa_name);
}
}
+
+ if (adapter_type == ADAPTER_TYPE_VPN && network_monitor_) {
+ vpn_underlying_adapter_type =
+ network_monitor_->GetVpnUnderlyingAdapterType(cursor->ifa_name);
+ }
int prefix_length = CountIPMaskBits(mask);
prefix = TruncateIP(ip, prefix_length);
std::string key = MakeNetworkKey(std::string(cursor->ifa_name),
@@ -502,6 +526,7 @@
network->set_scope_id(scope_id);
network->AddIP(ip);
network->set_ignored(IsIgnoredNetwork(*network));
+ network->set_underlying_type_for_vpn(vpn_underlying_adapter_type);
if (include_ignored || !network->ignored()) {
current_networks[key] = network.get();
networks->push_back(network.release());
@@ -511,6 +536,8 @@
existing_network->AddIP(ip);
if (adapter_type != ADAPTER_TYPE_UNKNOWN) {
existing_network->set_type(adapter_type);
+ existing_network->set_underlying_type_for_vpn(
+ vpn_underlying_adapter_type);
}
}
}
@@ -633,7 +660,7 @@
scope_id = v6_addr->sin6_scope_id;
ip = IPAddress(v6_addr->sin6_addr);
- if (IsIgnoredIPv6(ip)) {
+ if (IsIgnoredIPv6(InterfaceAddress(ip))) {
continue;
}
@@ -971,13 +998,22 @@
return static_cast<IPAddress>(selected_ip);
}
+uint16_t Network::GetCost() const {
+ AdapterType type = IsVpn() ? underlying_type_for_vpn_ : type_;
+ return ComputeNetworkCostByType(type);
+}
+
std::string Network::ToString() const {
std::stringstream ss;
// Print out the first space-terminated token of the network desc, plus
// the IP address.
- ss << "Net[" << description_.substr(0, description_.find(' '))
- << ":" << prefix_.ToSensitiveString() << "/" << prefix_length_
- << ":" << AdapterTypeToString(type_) << "]";
+ ss << "Net[" << description_.substr(0, description_.find(' ')) << ":"
+ << prefix_.ToSensitiveString() << "/" << prefix_length_ << ":"
+ << AdapterTypeToString(type_);
+ if (IsVpn()) {
+ ss << "/" << AdapterTypeToString(underlying_type_for_vpn_);
+ }
+ ss << "]";
return ss.str();
}
diff --git a/rtc_base/network.h b/rtc_base/network.h
index 49934cc..49f500c 100644
--- a/rtc_base/network.h
+++ b/rtc_base/network.h
@@ -293,7 +293,7 @@
AdapterType type);
Network(const Network&);
~Network();
-
+ // This signal is fired whenever type() or underlying_type_for_vpn() changes.
sigslot::signal1<const Network*> SignalTypeChanged;
const DefaultLocalAddressProvider* default_local_address_provider() {
@@ -346,6 +346,7 @@
// Adds an active IP address to this network. Does not check for duplicates.
void AddIP(const InterfaceAddress& ip) { ips_.push_back(ip); }
+ void AddIP(const IPAddress& ip) { ips_.push_back(rtc::InterfaceAddress(ip)); }
// Sets the network's IP address list. Returns true if new IP addresses were
// detected. Passing true to already_changed skips this check.
@@ -366,28 +367,37 @@
void set_ignored(bool ignored) { ignored_ = ignored; }
AdapterType type() const { return type_; }
+ // When type() is ADAPTER_TYPE_VPN, this returns the type of the underlying
+ // network interface used by the VPN, typically the preferred network type
+ // (see for example, the method setUnderlyingNetworks(android.net.Network[])
+ // on https://developer.android.com/reference/android/net/VpnService.html).
+ // When this information is unavailable from the OS, ADAPTER_TYPE_UNKNOWN is
+ // returned.
+ AdapterType underlying_type_for_vpn() const {
+ return underlying_type_for_vpn_;
+ }
void set_type(AdapterType type) {
if (type_ == type) {
return;
}
type_ = type;
+ if (type != ADAPTER_TYPE_VPN) {
+ underlying_type_for_vpn_ = ADAPTER_TYPE_UNKNOWN;
+ }
SignalTypeChanged(this);
}
- uint16_t GetCost() const {
- switch (type_) {
- case rtc::ADAPTER_TYPE_ETHERNET:
- case rtc::ADAPTER_TYPE_LOOPBACK:
- return kNetworkCostMin;
- case rtc::ADAPTER_TYPE_WIFI:
- case rtc::ADAPTER_TYPE_VPN:
- return kNetworkCostLow;
- case rtc::ADAPTER_TYPE_CELLULAR:
- return kNetworkCostHigh;
- default:
- return kNetworkCostUnknown;
+ void set_underlying_type_for_vpn(AdapterType type) {
+ if (underlying_type_for_vpn_ == type) {
+ return;
}
+ underlying_type_for_vpn_ = type;
+ SignalTypeChanged(this);
}
+
+ bool IsVpn() const { return type_ == ADAPTER_TYPE_VPN; }
+
+ uint16_t GetCost() const;
// A unique id assigned by the network manager, which may be signaled
// to the remote side in the candidate.
uint16_t id() const { return id_; }
@@ -420,6 +430,7 @@
int scope_id_;
bool ignored_;
AdapterType type_;
+ AdapterType underlying_type_for_vpn_ = ADAPTER_TYPE_UNKNOWN;
int preference_;
bool active_ = true;
uint16_t id_ = 0;
diff --git a/rtc_base/network_unittest.cc b/rtc_base/network_unittest.cc
index bf09c24..f4dcc6c 100644
--- a/rtc_base/network_unittest.cc
+++ b/rtc_base/network_unittest.cc
@@ -552,19 +552,16 @@
// But with two addresses now.
EXPECT_EQ(2U, (*it)->GetIPs().size());
EXPECT_NE((*it)->GetIPs().end(),
- std::find((*it)->GetIPs().begin(),
- (*it)->GetIPs().end(),
- check_ip));
+ std::find((*it)->GetIPs().begin(), (*it)->GetIPs().end(),
+ InterfaceAddress(check_ip)));
EXPECT_NE((*it)->GetIPs().end(),
- std::find((*it)->GetIPs().begin(),
- (*it)->GetIPs().end(),
- ip));
+ std::find((*it)->GetIPs().begin(), (*it)->GetIPs().end(),
+ InterfaceAddress(ip)));
} else {
// Check the IP didn't get added anywhere it wasn't supposed to.
EXPECT_EQ((*it)->GetIPs().end(),
- std::find((*it)->GetIPs().begin(),
- (*it)->GetIPs().end(),
- ip));
+ std::find((*it)->GetIPs().begin(), (*it)->GetIPs().end(),
+ InterfaceAddress(ip)));
}
}
}
@@ -606,9 +603,8 @@
} else {
// Check the IP didn't get added anywhere it wasn't supposed to.
EXPECT_EQ((*it)->GetIPs().end(),
- std::find((*it)->GetIPs().begin(),
- (*it)->GetIPs().end(),
- ip));
+ std::find((*it)->GetIPs().begin(), (*it)->GetIPs().end(),
+ InterfaceAddress(ip)));
}
}
}
@@ -962,8 +958,8 @@
// IPAddresses.
EXPECT_EQ(list2.size(), 1uL);
EXPECT_EQ(list2[0]->GetIPs().size(), 2uL);
- EXPECT_EQ(list2[0]->GetIPs()[0], ip1);
- EXPECT_EQ(list2[0]->GetIPs()[1], ip2);
+ EXPECT_EQ(list2[0]->GetIPs()[0], InterfaceAddress(ip1));
+ EXPECT_EQ(list2[0]->GetIPs()[1], InterfaceAddress(ip2));
}
// Test that MergeNetworkList successfully detects the change if
diff --git a/rtc_base/networkmonitor.cc b/rtc_base/networkmonitor.cc
index 0272951..ad6805a 100644
--- a/rtc_base/networkmonitor.cc
+++ b/rtc_base/networkmonitor.cc
@@ -39,6 +39,11 @@
SignalNetworksChanged();
}
+AdapterType NetworkMonitorBase::GetVpnUnderlyingAdapterType(
+ const std::string& interface_name) {
+ return ADAPTER_TYPE_UNKNOWN;
+}
+
NetworkMonitorFactory::NetworkMonitorFactory() {}
NetworkMonitorFactory::~NetworkMonitorFactory() {}
diff --git a/rtc_base/networkmonitor.h b/rtc_base/networkmonitor.h
index 254b225..a174473 100644
--- a/rtc_base/networkmonitor.h
+++ b/rtc_base/networkmonitor.h
@@ -74,6 +74,8 @@
virtual void OnNetworksChanged() = 0;
virtual AdapterType GetAdapterType(const std::string& interface_name) = 0;
+ virtual AdapterType GetVpnUnderlyingAdapterType(
+ const std::string& interface_name) = 0;
};
class NetworkMonitorBase : public NetworkMonitorInterface,
@@ -87,6 +89,9 @@
void OnMessage(Message* msg) override;
+ AdapterType GetVpnUnderlyingAdapterType(
+ const std::string& interface_name) override;
+
protected:
Thread* worker_thread() { return worker_thread_; }
diff --git a/rtc_base/openssl.h b/rtc_base/openssl.h
index dbbae05..eeed373 100644
--- a/rtc_base/openssl.h
+++ b/rtc_base/openssl.h
@@ -11,6 +11,11 @@
#ifndef RTC_BASE_OPENSSL_H_
#define RTC_BASE_OPENSSL_H_
+#if defined(WEBRTC_WIN)
+// Must be included first before openssl headers.
+#include "rtc_base/win32.h" // NOLINT
+#endif // WEBRTC_WIN
+
#include <openssl/ssl.h>
#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
diff --git a/rtc_base/openssladapter.cc b/rtc_base/openssladapter.cc
index ce00469..03b3ca8 100644
--- a/rtc_base/openssladapter.cc
+++ b/rtc_base/openssladapter.cc
@@ -14,11 +14,6 @@
#include <unistd.h>
#endif
-#if defined(WEBRTC_WIN)
-// Must be included first before openssl headers.
-#include "rtc_base/win32.h" // NOLINT
-#endif // WEBRTC_WIN
-
#include <openssl/bio.h>
#include <openssl/crypto.h>
#include <openssl/err.h>
@@ -26,12 +21,14 @@
#include <openssl/rand.h>
#include <openssl/x509.h>
#include <openssl/x509v3.h>
+#include "rtc_base/openssl.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
-#include "rtc_base/openssl.h"
+#include "rtc_base/opensslcommon.h"
+#include "rtc_base/ptr_util.h"
#include "rtc_base/sslroots.h"
#include "rtc_base/stringencode.h"
#include "rtc_base/stringutils.h"
@@ -205,9 +202,9 @@
}
OpenSSLAdapter::OpenSSLAdapter(AsyncSocket* socket,
- OpenSSLAdapterFactory* factory)
+ OpenSSLSessionCache* ssl_session_cache)
: SSLAdapter(socket),
- factory_(factory),
+ ssl_session_cache_(ssl_session_cache),
state_(SSL_NONE),
role_(SSL_CLIENT),
ssl_read_needs_write_(false),
@@ -221,8 +218,8 @@
// If a factory is used, take a reference on the factory's SSL_CTX.
// Otherwise, we'll create our own later.
// Either way, we'll release our reference via SSL_CTX_free() in Cleanup().
- if (factory_) {
- ssl_ctx_ = factory_->ssl_ctx();
+ if (ssl_session_cache_ != nullptr) {
+ ssl_ctx_ = ssl_session_cache_->GetSSLContext();
RTC_DCHECK(ssl_ctx_);
// Note: if using OpenSSL, requires version 1.1.0 or later.
SSL_CTX_up_ref(ssl_ctx_);
@@ -306,7 +303,7 @@
// First set up the context. We should either have a factory, with its own
// pre-existing context, or be running standalone, in which case we will
// need to create one, and specify |false| to disable session caching.
- if (!factory_) {
+ if (ssl_session_cache_ == nullptr) {
RTC_DCHECK(!ssl_ctx_);
ssl_ctx_ = CreateContext(ssl_mode_, false);
}
@@ -351,8 +348,8 @@
SSL_set_tlsext_host_name(ssl_, ssl_host_name_.c_str());
// Enable session caching, if configured and a hostname is supplied.
- if (factory_) {
- SSL_SESSION* cached = factory_->LookupSession(ssl_host_name_);
+ if (ssl_session_cache_ != nullptr) {
+ SSL_SESSION* cached = ssl_session_cache_->LookupSession(ssl_host_name_);
if (cached) {
if (SSL_set_session(ssl_, cached) == 0) {
RTC_LOG(LS_WARNING) << "Failed to apply SSL session from cache";
@@ -414,7 +411,7 @@
int code = (role_ == SSL_CLIENT) ? SSL_connect(ssl_) : SSL_accept(ssl_);
switch (SSL_get_error(ssl_, code)) {
case SSL_ERROR_NONE:
- if (!SSLPostConnectionCheck(ssl_, ssl_host_name_.c_str())) {
+ if (!SSLPostConnectionCheck(ssl_, ssl_host_name_)) {
RTC_LOG(LS_ERROR) << "TLS post connection check failed";
// make sure we close the socket
Cleanup();
@@ -614,7 +611,6 @@
int OpenSSLAdapter::Recv(void* pv, size_t cb, int64_t* timestamp) {
switch (state_) {
-
case SSL_NONE:
return AsyncSocketAdapter::Recv(pv, cb, timestamp);
@@ -790,96 +786,18 @@
AsyncSocketAdapter::OnCloseEvent(socket, err);
}
-bool OpenSSLAdapter::VerifyServerName(SSL* ssl, const char* host,
- bool ignore_bad_cert) {
- if (!host)
- return false;
+bool OpenSSLAdapter::SSLPostConnectionCheck(SSL* ssl, const std::string& host) {
+ bool is_valid_cert_name = openssl::VerifyPeerCertMatchesHost(ssl, host) &&
+ (SSL_get_verify_result(ssl) == X509_V_OK ||
+ custom_verification_succeeded_);
- // Checking the return from SSL_get_peer_certificate here is not strictly
- // necessary. With our setup, it is not possible for it to return
- // null. However, it is good form to check the return.
- X509* certificate = SSL_get_peer_certificate(ssl);
- if (!certificate)
- return false;
-
- // Logging certificates is extremely verbose. So it is disabled by default.
-#ifdef LOG_CERTIFICATES
- {
- RTC_DLOG(LS_INFO) << "Certificate from server:";
- BIO* mem = BIO_new(BIO_s_mem());
- X509_print_ex(mem, certificate, XN_FLAG_SEP_CPLUS_SPC, X509_FLAG_NO_HEADER);
- BIO_write(mem, "\0", 1);
- char* buffer;
- BIO_get_mem_data(mem, &buffer);
- RTC_DLOG(LS_INFO) << buffer;
- BIO_free(mem);
-
- char* cipher_description =
- SSL_CIPHER_description(SSL_get_current_cipher(ssl), nullptr, 128);
- RTC_DLOG(LS_INFO) << "Cipher: " << cipher_description;
- OPENSSL_free(cipher_description);
+ if (!is_valid_cert_name && ignore_bad_cert_) {
+ RTC_DLOG(LS_WARNING) << "Other TLS post connection checks failed."
+ "ignore_bad_cert_ set to true. Overriding name "
+ "verification failure!";
+ is_valid_cert_name = true;
}
-#endif
-
- bool ok = false;
- GENERAL_NAMES* names = reinterpret_cast<GENERAL_NAMES*>(
- X509_get_ext_d2i(certificate, NID_subject_alt_name, nullptr, nullptr));
- if (names) {
- for (size_t i = 0; i < static_cast<size_t>(sk_GENERAL_NAME_num(names));
- i++) {
- const GENERAL_NAME* name = sk_GENERAL_NAME_value(names, i);
- if (name->type != GEN_DNS)
- continue;
- std::string value(
- reinterpret_cast<const char*>(ASN1_STRING_data(name->d.dNSName)),
- ASN1_STRING_length(name->d.dNSName));
- // string_match takes NUL-terminated strings, so check for embedded NULs.
- if (value.find('\0') != std::string::npos)
- continue;
- if (string_match(host, value.c_str())) {
- ok = true;
- break;
- }
- }
- GENERAL_NAMES_free(names);
- }
-
- char data[256];
- X509_NAME* subject;
- if (!ok && ((subject = X509_get_subject_name(certificate)) != nullptr) &&
- (X509_NAME_get_text_by_NID(subject, NID_commonName, data, sizeof(data)) >
- 0)) {
- data[sizeof(data)-1] = 0;
- if (_stricmp(data, host) == 0)
- ok = true;
- }
-
- X509_free(certificate);
-
- // This should only ever be turned on for debugging and development.
- if (!ok && ignore_bad_cert) {
- RTC_DLOG(LS_WARNING) << "TLS certificate check FAILED. "
- << "Allowing connection anyway.";
- ok = true;
- }
-
- return ok;
-}
-
-bool OpenSSLAdapter::SSLPostConnectionCheck(SSL* ssl, const char* host) {
- bool ok = VerifyServerName(ssl, host, ignore_bad_cert_);
-
- if (ok) {
- ok = (SSL_get_verify_result(ssl) == X509_V_OK ||
- custom_verification_succeeded_);
- }
-
- if (!ok && ignore_bad_cert_) {
- RTC_DLOG(LS_INFO) << "Other TLS post connection checks failed.";
- ok = true;
- }
-
- return ok;
+ return is_valid_cert_name;
}
#if !defined(NDEBUG)
@@ -960,9 +878,9 @@
int OpenSSLAdapter::NewSSLSessionCallback(SSL* ssl, SSL_SESSION* session) {
OpenSSLAdapter* stream =
reinterpret_cast<OpenSSLAdapter*>(SSL_get_app_data(ssl));
- RTC_DCHECK(stream->factory_);
+ RTC_DCHECK(stream->ssl_session_cache_);
RTC_LOG(LS_INFO) << "Caching SSL session for " << stream->ssl_host_name_;
- stream->factory_->AddSession(stream->ssl_host_name_, session);
+ stream->ssl_session_cache_->AddSession(stream->ssl_host_name_, session);
return 1; // We've taken ownership of the session; OpenSSL shouldn't free it.
}
@@ -1061,43 +979,26 @@
// OpenSSLAdapterFactory
//////////////////////////////////////////////////////////////////////
-OpenSSLAdapterFactory::OpenSSLAdapterFactory()
- : ssl_mode_(SSL_MODE_TLS), ssl_ctx_(nullptr) {}
-
-OpenSSLAdapterFactory::~OpenSSLAdapterFactory() {
- for (auto it : sessions_) {
- SSL_SESSION_free(it.second);
- }
- SSL_CTX_free(ssl_ctx_);
-}
+OpenSSLAdapterFactory::OpenSSLAdapterFactory() = default;
+OpenSSLAdapterFactory::~OpenSSLAdapterFactory() = default;
void OpenSSLAdapterFactory::SetMode(SSLMode mode) {
- RTC_DCHECK(!ssl_ctx_);
+ RTC_DCHECK(!ssl_session_cache_);
ssl_mode_ = mode;
}
OpenSSLAdapter* OpenSSLAdapterFactory::CreateAdapter(AsyncSocket* socket) {
- if (!ssl_ctx_) {
- bool enable_cache = true;
- ssl_ctx_ = OpenSSLAdapter::CreateContext(ssl_mode_, enable_cache);
- if (!ssl_ctx_) {
+ if (ssl_session_cache_ == nullptr) {
+ SSL_CTX* ssl_ctx =
+ OpenSSLAdapter::CreateContext(ssl_mode_, /* enable_cache = */ true);
+ if (ssl_ctx == nullptr) {
return nullptr;
}
+ // The OpenSSLSessionCache will upref the ssl_ctx.
+ ssl_session_cache_ = MakeUnique<OpenSSLSessionCache>(ssl_mode_, ssl_ctx);
+ SSL_CTX_free(ssl_ctx);
}
-
- return new OpenSSLAdapter(socket, this);
+ return new OpenSSLAdapter(socket, ssl_session_cache_.get());
}
-SSL_SESSION* OpenSSLAdapterFactory::LookupSession(const std::string& hostname) {
- auto it = sessions_.find(hostname);
- return (it != sessions_.end()) ? it->second : nullptr;
-}
-
-void OpenSSLAdapterFactory::AddSession(const std::string& hostname,
- SSL_SESSION* new_session) {
- SSL_SESSION* old_session = LookupSession(hostname);
- SSL_SESSION_free(old_session);
- sessions_[hostname] = new_session;
-}
-
-} // namespace rtc
+} // namespace rtc
diff --git a/rtc_base/openssladapter.h b/rtc_base/openssladapter.h
index 2d0474e..5f5eb80 100644
--- a/rtc_base/openssladapter.h
+++ b/rtc_base/openssladapter.h
@@ -11,30 +11,29 @@
#ifndef RTC_BASE_OPENSSLADAPTER_H_
#define RTC_BASE_OPENSSLADAPTER_H_
+#include <openssl/ossl_typ.h>
+
#include <map>
+#include <memory>
#include <string>
+#include <vector>
+
#include "rtc_base/buffer.h"
#include "rtc_base/messagehandler.h"
#include "rtc_base/messagequeue.h"
#include "rtc_base/opensslidentity.h"
+#include "rtc_base/opensslsessioncache.h"
#include "rtc_base/ssladapter.h"
-typedef struct ssl_st SSL;
-typedef struct ssl_ctx_st SSL_CTX;
-typedef struct x509_store_ctx_st X509_STORE_CTX;
-typedef struct ssl_session_st SSL_SESSION;
-
namespace rtc {
-class OpenSSLAdapterFactory;
-
class OpenSSLAdapter : public SSLAdapter, public MessageHandler {
public:
static bool InitializeSSL(VerificationCallback callback);
static bool CleanupSSL();
explicit OpenSSLAdapter(AsyncSocket* socket,
- OpenSSLAdapterFactory* factory = nullptr);
+ OpenSSLSessionCache* ssl_session_cache = nullptr);
~OpenSSLAdapter() override;
void SetIgnoreBadCert(bool ignore) override;
@@ -91,9 +90,8 @@
void OnMessage(Message* msg) override;
- static bool VerifyServerName(SSL* ssl, const char* host,
- bool ignore_bad_cert);
- bool SSLPostConnectionCheck(SSL* ssl, const char* host);
+ bool SSLPostConnectionCheck(SSL* ssl, const std::string& host);
+
#if !defined(NDEBUG)
// In debug builds, logs info about the state of the SSL connection.
static void SSLInfoCallback(const SSL* ssl, int where, int ret);
@@ -111,7 +109,7 @@
// Parent object that maintains shared state.
// Can be null if state sharing is not needed.
- OpenSSLAdapterFactory* factory_;
+ OpenSSLSessionCache* ssl_session_cache_ = nullptr;
SSLState state_;
std::unique_ptr<OpenSSLIdentity> identity_;
@@ -145,32 +143,32 @@
std::string TransformAlpnProtocols(const std::vector<std::string>& protos);
/////////////////////////////////////////////////////////////////////////////
+
+// The OpenSSLAdapterFactory is responsbile for creating multiple new
+// OpenSSLAdapters with a shared SSL_CTX and a shared SSL_SESSION cache. The
+// SSL_SESSION cache allows existing SSL_SESSIONS to be reused instead of
+// recreating them leading to a significant performance improvement.
class OpenSSLAdapterFactory : public SSLAdapterFactory {
public:
OpenSSLAdapterFactory();
~OpenSSLAdapterFactory() override;
-
+ // Set the SSL Mode to use with this factory. This should only be set before
+ // the first adapter is created with the factory. If it is called after it
+ // will DCHECK.
void SetMode(SSLMode mode) override;
+ // Constructs a new socket using the shared OpenSSLSessionCache. This means
+ // existing SSLSessions already in the cache will be reused instead of
+ // re-created for improved performance.
OpenSSLAdapter* CreateAdapter(AsyncSocket* socket) override;
- static OpenSSLAdapterFactory* Create();
-
private:
- SSL_CTX* ssl_ctx() { return ssl_ctx_; }
- // Looks up a session by hostname. The returned SSL_SESSION is not up_refed.
- SSL_SESSION* LookupSession(const std::string& hostname);
- // Adds a session to the cache, and up_refs it. Any existing session with the
- // same hostname is replaced.
- void AddSession(const std::string& hostname, SSL_SESSION* session);
+ // Holds the SSLMode (DTLS,TLS) that will be used to set the session cache.
+ SSLMode ssl_mode_ = SSL_MODE_TLS;
+ // Holds a cache of existing SSL Sessions.
+ std::unique_ptr<OpenSSLSessionCache> ssl_session_cache_;
+ // TODO(benwright): Remove this when context is moved to OpenSSLCommon.
+ // Hold a friend class to the OpenSSLAdapter to retrieve the context.
friend class OpenSSLAdapter;
-
- SSLMode ssl_mode_;
- // Holds the shared SSL_CTX for all created adapters.
- SSL_CTX* ssl_ctx_;
- // Map of hostnames to SSL_SESSIONs; holds references to the SSL_SESSIONs,
- // which are cleaned up when the factory is destroyed.
- // TODO(juberti): Add LRU eviction to keep the cache from growing forever.
- std::map<std::string, SSL_SESSION*> sessions_;
};
} // namespace rtc
diff --git a/rtc_base/opensslcommon.cc b/rtc_base/opensslcommon.cc
new file mode 100644
index 0000000..5213392
--- /dev/null
+++ b/rtc_base/opensslcommon.cc
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/opensslcommon.h"
+
+#if defined(WEBRTC_POSIX)
+#include <unistd.h>
+#endif
+
+#if defined(WEBRTC_WIN)
+// Must be included first before openssl headers.
+#include "rtc_base/win32.h" // NOLINT
+#endif // WEBRTC_WIN
+
+#include <openssl/bio.h>
+#include <openssl/crypto.h>
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/openssl.h"
+
+namespace rtc {
+namespace openssl {
+
+// Holds various helper methods.
+namespace {
+void LogCertificates(SSL* ssl, X509* certificate) {
+// Logging certificates is extremely verbose. So it is disabled by default.
+#ifdef LOG_CERTIFICATES
+ BIO* mem = BIO_new(BIO_s_mem());
+ if (mem == nullptr) {
+ RTC_DLOG(LS_ERROR) << "BIO_new() failed to allocate memory.";
+ return;
+ }
+
+ RTC_DLOG(LS_INFO) << "Certificate from server:";
+ X509_print_ex(mem, certificate, XN_FLAG_SEP_CPLUS_SPC, X509_FLAG_NO_HEADER);
+ BIO_write(mem, "\0", 1);
+
+ char* buffer = nullptr;
+ BIO_get_mem_data(mem, &buffer);
+ if (buffer != nullptr) {
+ RTC_DLOG(LS_INFO) << buffer;
+ } else {
+ RTC_DLOG(LS_ERROR) << "BIO_get_mem_data() failed to get buffer.";
+ }
+ BIO_free(mem);
+
+ const char* cipher_name = SSL_CIPHER_get_name(SSL_get_current_cipher(ssl));
+ if (cipher_name != nullptr) {
+ RTC_DLOG(LS_INFO) << "Cipher: " << cipher_name;
+ } else {
+ RTC_DLOG(LS_ERROR) << "SSL_CIPHER_DESCRIPTION() failed to get cipher_name.";
+ }
+#endif
+}
+} // namespace
+
+bool VerifyPeerCertMatchesHost(SSL* ssl, const std::string& host) {
+ if (host.empty()) {
+ RTC_DLOG(LS_ERROR) << "Hostname is empty. Cannot verify peer certificate.";
+ return false;
+ }
+
+ if (ssl == nullptr) {
+ RTC_DLOG(LS_ERROR) << "SSL is nullptr. Cannot verify peer certificate.";
+ return false;
+ }
+
+ X509* certificate = SSL_get_peer_certificate(ssl);
+ if (certificate == nullptr) {
+ RTC_DLOG(LS_ERROR)
+ << "SSL_get_peer_certificate failed. This should never happen.";
+ return false;
+ }
+
+ LogCertificates(ssl, certificate);
+
+ bool is_valid_cert_name =
+ X509_check_host(certificate, host.c_str(), host.size(), 0, nullptr) == 1;
+ X509_free(certificate);
+ return is_valid_cert_name;
+}
+
+} // namespace openssl
+} // namespace rtc
diff --git a/rtc_base/opensslcommon.h b/rtc_base/opensslcommon.h
new file mode 100644
index 0000000..c05165b
--- /dev/null
+++ b/rtc_base/opensslcommon.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef RTC_BASE_OPENSSLCOMMON_H_
+#define RTC_BASE_OPENSSLCOMMON_H_
+
+#include <string>
+
+typedef struct ssl_st SSL;
+
+namespace rtc {
+// The openssl namespace holds static helper methods. All methods related
+// to OpenSSL that are commonly used and don't require global state should be
+// placed here.
+namespace openssl {
+// Verifies that the hostname provided matches that in the peer certificate
+// attached to this SSL state.
+bool VerifyPeerCertMatchesHost(SSL* ssl, const std::string& host);
+} // namespace openssl
+} // namespace rtc
+
+#endif // RTC_BASE_OPENSSLCOMMON_H_
diff --git a/rtc_base/opensslcommon_unittest.cc b/rtc_base/opensslcommon_unittest.cc
new file mode 100644
index 0000000..322a7b9
--- /dev/null
+++ b/rtc_base/opensslcommon_unittest.cc
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <string>
+#include <vector>
+
+#if defined(WEBRTC_POSIX)
+#include <unistd.h>
+#endif
+
+#if defined(WEBRTC_WIN)
+// Must be included first before openssl headers.
+#include "rtc_base/win32.h" // NOLINT
+#endif // WEBRTC_WIN
+
+#include <openssl/bio.h>
+#include <openssl/crypto.h>
+#include <openssl/x509.h>
+#include <openssl/x509v3.h>
+
+#include "rtc_base/arraysize.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/numerics/safe_conversions.h"
+#include "rtc_base/openssl.h"
+#include "rtc_base/opensslcommon.h"
+#include "rtc_base/sslroots.h"
+#include "test/gmock.h"
+
+namespace rtc {
+namespace {
+// Fake Self-Signed SSL Certifiacte with CN: *.webrtc.org.
+// This is only to be used for testing (it isn't signed by a CA anyway).
+const unsigned char kFakeSSLCertificate[] = {
+ 0x30, 0x82, 0x02, 0x68, 0x30, 0x82, 0x02, 0x12, 0xA0, 0x03, 0x02, 0x01,
+ 0x02, 0x02, 0x09, 0x00, 0xC8, 0x83, 0x59, 0x4D, 0x90, 0xC3, 0x5F, 0xC8,
+ 0x30, 0x0D, 0x06, 0x09, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x01,
+ 0x0B, 0x05, 0x00, 0x30, 0x81, 0x8D, 0x31, 0x0B, 0x30, 0x09, 0x06, 0x03,
+ 0x55, 0x04, 0x06, 0x13, 0x02, 0x55, 0x53, 0x31, 0x0B, 0x30, 0x09, 0x06,
+ 0x03, 0x55, 0x04, 0x08, 0x0C, 0x02, 0x57, 0x41, 0x31, 0x2C, 0x30, 0x2A,
+ 0x06, 0x03, 0x55, 0x04, 0x0A, 0x0C, 0x23, 0x46, 0x61, 0x6B, 0x65, 0x20,
+ 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69,
+ 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x20, 0x46, 0x6F, 0x72, 0x20, 0x54,
+ 0x65, 0x73, 0x74, 0x69, 0x6E, 0x67, 0x31, 0x2C, 0x30, 0x2A, 0x06, 0x03,
+ 0x55, 0x04, 0x0B, 0x0C, 0x23, 0x46, 0x61, 0x6B, 0x65, 0x20, 0x57, 0x65,
+ 0x62, 0x52, 0x54, 0x43, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69,
+ 0x63, 0x61, 0x74, 0x65, 0x20, 0x46, 0x6F, 0x72, 0x20, 0x54, 0x65, 0x73,
+ 0x74, 0x69, 0x6E, 0x67, 0x31, 0x15, 0x30, 0x13, 0x06, 0x03, 0x55, 0x04,
+ 0x03, 0x0C, 0x0C, 0x2A, 0x2E, 0x77, 0x65, 0x62, 0x72, 0x74, 0x63, 0x2E,
+ 0x6F, 0x72, 0x67, 0x30, 0x1E, 0x17, 0x0D, 0x31, 0x38, 0x30, 0x34, 0x30,
+ 0x33, 0x32, 0x31, 0x35, 0x34, 0x30, 0x38, 0x5A, 0x17, 0x0D, 0x31, 0x39,
+ 0x30, 0x34, 0x30, 0x33, 0x32, 0x31, 0x35, 0x34, 0x30, 0x38, 0x5A, 0x30,
+ 0x81, 0x8D, 0x31, 0x0B, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13,
+ 0x02, 0x55, 0x53, 0x31, 0x0B, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x08,
+ 0x0C, 0x02, 0x57, 0x41, 0x31, 0x2C, 0x30, 0x2A, 0x06, 0x03, 0x55, 0x04,
+ 0x0A, 0x0C, 0x23, 0x46, 0x61, 0x6B, 0x65, 0x20, 0x57, 0x65, 0x62, 0x52,
+ 0x54, 0x43, 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61,
+ 0x74, 0x65, 0x20, 0x46, 0x6F, 0x72, 0x20, 0x54, 0x65, 0x73, 0x74, 0x69,
+ 0x6E, 0x67, 0x31, 0x2C, 0x30, 0x2A, 0x06, 0x03, 0x55, 0x04, 0x0B, 0x0C,
+ 0x23, 0x46, 0x61, 0x6B, 0x65, 0x20, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43,
+ 0x20, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65,
+ 0x20, 0x46, 0x6F, 0x72, 0x20, 0x54, 0x65, 0x73, 0x74, 0x69, 0x6E, 0x67,
+ 0x31, 0x15, 0x30, 0x13, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0C, 0x0C, 0x2A,
+ 0x2E, 0x77, 0x65, 0x62, 0x72, 0x74, 0x63, 0x2E, 0x6F, 0x72, 0x67, 0x30,
+ 0x5C, 0x30, 0x0D, 0x06, 0x09, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01,
+ 0x01, 0x01, 0x05, 0x00, 0x03, 0x4B, 0x00, 0x30, 0x48, 0x02, 0x41, 0x00,
+ 0xAE, 0xAE, 0x85, 0x2A, 0x40, 0xD6, 0x99, 0x35, 0x09, 0x34, 0x1B, 0xC5,
+ 0xAC, 0x6C, 0x79, 0xC7, 0xC3, 0xDE, 0x1B, 0xCF, 0x17, 0x8D, 0x6B, 0x84,
+ 0xEC, 0x8B, 0x4E, 0x2B, 0xC1, 0x83, 0x43, 0xDF, 0x76, 0x0F, 0x5F, 0x5A,
+ 0xA9, 0x7D, 0x94, 0xC0, 0x54, 0x5C, 0xFF, 0xBC, 0x7C, 0x86, 0xDC, 0x9A,
+ 0xCE, 0xB9, 0xDF, 0xE6, 0x0B, 0xC4, 0x5B, 0x6E, 0x56, 0x9F, 0xBC, 0x40,
+ 0xF5, 0xA0, 0x52, 0xA7, 0x02, 0x03, 0x01, 0x00, 0x01, 0xA3, 0x53, 0x30,
+ 0x51, 0x30, 0x1D, 0x06, 0x03, 0x55, 0x1D, 0x0E, 0x04, 0x16, 0x04, 0x14,
+ 0xB7, 0xC0, 0x9A, 0xA7, 0x22, 0xAF, 0xF8, 0x7D, 0xFF, 0x68, 0xDB, 0x80,
+ 0xAC, 0x0A, 0xB6, 0xDC, 0x64, 0x89, 0xDB, 0xD4, 0x30, 0x1F, 0x06, 0x03,
+ 0x55, 0x1D, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, 0xB7, 0xC0, 0x9A,
+ 0xA7, 0x22, 0xAF, 0xF8, 0x7D, 0xFF, 0x68, 0xDB, 0x80, 0xAC, 0x0A, 0xB6,
+ 0xDC, 0x64, 0x89, 0xDB, 0xD4, 0x30, 0x0F, 0x06, 0x03, 0x55, 0x1D, 0x13,
+ 0x01, 0x01, 0xFF, 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xFF, 0x30, 0x0D,
+ 0x06, 0x09, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x01, 0x0B, 0x05,
+ 0x00, 0x03, 0x41, 0x00, 0x50, 0x6D, 0xCC, 0x62, 0xAE, 0xD1, 0x7C, 0x4D,
+ 0xEF, 0x90, 0x1E, 0x9B, 0x72, 0x73, 0xE0, 0x56, 0x66, 0x32, 0x6A, 0x78,
+ 0xE8, 0x0F, 0xAD, 0x21, 0x32, 0x54, 0xA5, 0xB3, 0xB8, 0x14, 0x54, 0xBC,
+ 0x50, 0xF7, 0x7F, 0x73, 0xD6, 0x44, 0x1E, 0x82, 0xD9, 0x4B, 0x49, 0x48,
+ 0x9E, 0x02, 0x8B, 0xFE, 0xC3, 0xFD, 0x5D, 0x15, 0x02, 0xE1, 0x78, 0xAC,
+ 0x9A, 0xAE, 0xFC, 0xC7, 0x48, 0xC6, 0x48, 0x6B};
+
+// Simple testing helper method to create a fake SSL_SESSION with a testing
+// peer connection set.
+SSL_SESSION* CreateSSLSessionWithFakePeerCertificate(SSL_CTX* ssl_ctx) {
+ SSL_SESSION* ssl_session = SSL_SESSION_new(ssl_ctx);
+ const unsigned char* cert_buffer = kFakeSSLCertificate;
+ size_t cert_buffer_len = arraysize(kFakeSSLCertificate);
+ X509* ssl_peer_certificate = d2i_X509(
+ nullptr, &cert_buffer, checked_cast<long>(cert_buffer_len)); // NOLINT
+ EXPECT_NE(ssl_peer_certificate, nullptr);
+#ifdef OPENSSL_IS_BORINGSSL
+ ssl_session->x509_peer = ssl_peer_certificate;
+#else
+ ssl_session->peer = ssl_peer_certificate;
+#endif
+ return ssl_session;
+}
+} // namespace
+
+TEST(OpenSSLCommonTest, VerifyPeerCertMatchesHostFailsOnNoPeerCertificate) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+ SSL* ssl = SSL_new(ssl_ctx);
+
+ EXPECT_FALSE(openssl::VerifyPeerCertMatchesHost(ssl, "webrtc.org"));
+
+ SSL_free(ssl);
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLCommonTest, VerifyPeerCertMatchesHostSucceedsOnCorrectHostname) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+ SSL* ssl = SSL_new(ssl_ctx);
+ SSL_SESSION* ssl_session = CreateSSLSessionWithFakePeerCertificate(ssl_ctx);
+ SSL_set_session(ssl, ssl_session);
+
+ EXPECT_TRUE(openssl::VerifyPeerCertMatchesHost(ssl, "www.webrtc.org"));
+ EXPECT_TRUE(openssl::VerifyPeerCertMatchesHost(ssl, "alice.webrtc.org"));
+ EXPECT_TRUE(openssl::VerifyPeerCertMatchesHost(ssl, "bob.webrtc.org"));
+
+ SSL_SESSION_free(ssl_session);
+ SSL_free(ssl);
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLCommonTest, VerifyPeerCertMatchesHostFailsOnInvalidHostname) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+ SSL* ssl = SSL_new(ssl_ctx);
+ SSL_SESSION* ssl_session = CreateSSLSessionWithFakePeerCertificate(ssl_ctx);
+ SSL_set_session(ssl, ssl_session);
+
+ EXPECT_FALSE(openssl::VerifyPeerCertMatchesHost(ssl, "a.b.webrtc.org"));
+ EXPECT_FALSE(openssl::VerifyPeerCertMatchesHost(ssl, "notwebrtc.org"));
+ EXPECT_FALSE(openssl::VerifyPeerCertMatchesHost(ssl, "webrtc.org"));
+
+ SSL_SESSION_free(ssl_session);
+ SSL_free(ssl);
+ SSL_CTX_free(ssl_ctx);
+}
+
+} // namespace rtc
diff --git a/rtc_base/opensslsessioncache.cc b/rtc_base/opensslsessioncache.cc
new file mode 100644
index 0000000..2e37d55
--- /dev/null
+++ b/rtc_base/opensslsessioncache.cc
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/opensslsessioncache.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/openssl.h"
+
+namespace rtc {
+
+OpenSSLSessionCache::OpenSSLSessionCache(SSLMode ssl_mode, SSL_CTX* ssl_ctx)
+ : ssl_mode_(ssl_mode), ssl_ctx_(ssl_ctx) {
+ // It is invalid to pass in a null context.
+ RTC_DCHECK(ssl_ctx != nullptr);
+ SSL_CTX_up_ref(ssl_ctx);
+}
+
+OpenSSLSessionCache::~OpenSSLSessionCache() {
+ for (auto it : sessions_) {
+ SSL_SESSION_free(it.second);
+ }
+ SSL_CTX_free(ssl_ctx_);
+}
+
+SSL_SESSION* OpenSSLSessionCache::LookupSession(
+ const std::string& hostname) const {
+ auto it = sessions_.find(hostname);
+ return (it != sessions_.end()) ? it->second : nullptr;
+}
+
+void OpenSSLSessionCache::AddSession(const std::string& hostname,
+ SSL_SESSION* new_session) {
+ SSL_SESSION* old_session = LookupSession(hostname);
+ SSL_SESSION_free(old_session);
+ sessions_[hostname] = new_session;
+}
+
+SSL_CTX* OpenSSLSessionCache::GetSSLContext() const {
+ return ssl_ctx_;
+}
+
+SSLMode OpenSSLSessionCache::GetSSLMode() const {
+ return ssl_mode_;
+}
+
+} // namespace rtc
diff --git a/rtc_base/opensslsessioncache.h b/rtc_base/opensslsessioncache.h
new file mode 100644
index 0000000..ee5b525
--- /dev/null
+++ b/rtc_base/opensslsessioncache.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef RTC_BASE_OPENSSLSESSIONCACHE_H_
+#define RTC_BASE_OPENSSLSESSIONCACHE_H_
+
+#include <openssl/ossl_typ.h>
+#include <map>
+#include <string>
+
+#include "rtc_base/constructormagic.h"
+#include "rtc_base/sslstreamadapter.h"
+
+namespace rtc {
+
+// The OpenSSLSessionCache maps hostnames to SSL_SESSIONS. This cache is
+// owned by the OpenSSLAdapterFactory and is passed down to each OpenSSLAdapter
+// created with the factory.
+class OpenSSLSessionCache final {
+ public:
+ // Creates a new OpenSSLSessionCache using the provided the SSL_CTX and
+ // the ssl_mode. The SSL_CTX will be up_refed. ssl_ctx cannot be nullptr,
+ // the constructor immediately dchecks this.
+ OpenSSLSessionCache(SSLMode ssl_mode, SSL_CTX* ssl_ctx);
+ // Frees the cached SSL_SESSIONS and then frees the SSL_CTX.
+ ~OpenSSLSessionCache();
+ // Looks up a session by hostname. The returned SSL_SESSION is not up_refed.
+ SSL_SESSION* LookupSession(const std::string& hostname) const;
+ // Adds a session to the cache, and up_refs it. Any existing session with the
+ // same hostname is replaced.
+ void AddSession(const std::string& hostname, SSL_SESSION* session);
+ // Returns the true underlying SSL Context that holds these cached sessions.
+ SSL_CTX* GetSSLContext() const;
+ // The SSL Mode tht the OpenSSLSessionCache was constructed with. This cannot
+ // be changed after launch.
+ SSLMode GetSSLMode() const;
+
+ private:
+ // Holds the SSL Mode that the OpenSSLCache was initialized with. This is
+ // immutable after creation and cannot change.
+ const SSLMode ssl_mode_;
+ /// SSL Context for all shared cached sessions. This SSL_CTX is initialized
+ // with SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_CLIENT); Meaning
+ // all client sessions will be added to the cache internal to the context.
+ SSL_CTX* ssl_ctx_ = nullptr;
+ // Map of hostnames to SSL_SESSIONs; holds references to the SSL_SESSIONs,
+ // which are cleaned up when the factory is destroyed.
+ // TODO(juberti): Add LRU eviction to keep the cache from growing forever.
+ std::map<std::string, SSL_SESSION*> sessions_;
+ // The cache should never be copied or assigned directly.
+ RTC_DISALLOW_COPY_AND_ASSIGN(OpenSSLSessionCache);
+};
+
+} // namespace rtc
+
+#endif // RTC_BASE_OPENSSLSESSIONCACHE_H_
diff --git a/rtc_base/opensslsessioncache_unittest.cc b/rtc_base/opensslsessioncache_unittest.cc
new file mode 100644
index 0000000..6489b2b
--- /dev/null
+++ b/rtc_base/opensslsessioncache_unittest.cc
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <openssl/ssl.h>
+#include <stdlib.h>
+
+#include <map>
+#include <memory>
+
+#include "rtc_base/gunit.h"
+#include "rtc_base/openssl.h"
+#include "rtc_base/opensslsessioncache.h"
+
+namespace rtc {
+
+TEST(OpenSSLSessionCache, DTLSModeSetCorrectly) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+
+ OpenSSLSessionCache session_cache(SSL_MODE_DTLS, ssl_ctx);
+ EXPECT_EQ(session_cache.GetSSLMode(), SSL_MODE_DTLS);
+
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLSessionCache, TLSModeSetCorrectly) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(TLSv1_2_client_method());
+
+ OpenSSLSessionCache session_cache(SSL_MODE_TLS, ssl_ctx);
+ EXPECT_EQ(session_cache.GetSSLMode(), SSL_MODE_TLS);
+
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLSessionCache, SSLContextSetCorrectly) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+
+ OpenSSLSessionCache session_cache(SSL_MODE_DTLS, ssl_ctx);
+ EXPECT_EQ(session_cache.GetSSLContext(), ssl_ctx);
+
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLSessionCache, InvalidLookupReturnsNullptr) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+
+ OpenSSLSessionCache session_cache(SSL_MODE_DTLS, ssl_ctx);
+ EXPECT_EQ(session_cache.LookupSession("Invalid"), nullptr);
+ EXPECT_EQ(session_cache.LookupSession(""), nullptr);
+ EXPECT_EQ(session_cache.LookupSession("."), nullptr);
+
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLSessionCache, SimpleValidSessionLookup) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+ SSL_SESSION* ssl_session = SSL_SESSION_new(ssl_ctx);
+
+ OpenSSLSessionCache session_cache(SSL_MODE_DTLS, ssl_ctx);
+ session_cache.AddSession("webrtc.org", ssl_session);
+ EXPECT_EQ(session_cache.LookupSession("webrtc.org"), ssl_session);
+
+ SSL_CTX_free(ssl_ctx);
+}
+
+TEST(OpenSSLSessionCache, AddToExistingReplacesPrevious) {
+ SSL_CTX* ssl_ctx = SSL_CTX_new(DTLSv1_2_client_method());
+ SSL_SESSION* ssl_session_1 = SSL_SESSION_new(ssl_ctx);
+ SSL_SESSION* ssl_session_2 = SSL_SESSION_new(ssl_ctx);
+
+ OpenSSLSessionCache session_cache(SSL_MODE_DTLS, ssl_ctx);
+ session_cache.AddSession("webrtc.org", ssl_session_1);
+ session_cache.AddSession("webrtc.org", ssl_session_2);
+ EXPECT_EQ(session_cache.LookupSession("webrtc.org"), ssl_session_2);
+
+ SSL_CTX_free(ssl_ctx);
+}
+
+} // namespace rtc
diff --git a/rtc_base/opensslstreamadapter.h b/rtc_base/opensslstreamadapter.h
index 97ab557..7a6e099 100644
--- a/rtc_base/opensslstreamadapter.h
+++ b/rtc_base/opensslstreamadapter.h
@@ -11,6 +11,8 @@
#ifndef RTC_BASE_OPENSSLSTREAMADAPTER_H_
#define RTC_BASE_OPENSSLSTREAMADAPTER_H_
+#include <openssl/ossl_typ.h>
+
#include <string>
#include <memory>
#include <vector>
@@ -19,11 +21,6 @@
#include "rtc_base/opensslidentity.h"
#include "rtc_base/sslstreamadapter.h"
-typedef struct ssl_st SSL;
-typedef struct ssl_ctx_st SSL_CTX;
-typedef struct ssl_cipher_st SSL_CIPHER;
-typedef struct x509_store_ctx_st X509_STORE_CTX;
-
namespace rtc {
// This class was written with OpenSSLAdapter (a socket adapter) as a
diff --git a/rtc_base/physicalsocketserver_unittest.cc b/rtc_base/physicalsocketserver_unittest.cc
index d09385b..81f1c9d 100644
--- a/rtc_base/physicalsocketserver_unittest.cc
+++ b/rtc_base/physicalsocketserver_unittest.cc
@@ -201,13 +201,13 @@
server_->CreateAsyncSocket(loopback.family(), SOCK_STREAM));
sink.Monitor(client1.get());
EXPECT_EQ(AsyncSocket::CS_CLOSED, client1->GetState());
- EXPECT_PRED1(IsUnspecOrEmptyIP, client1->GetLocalAddress().ipaddr());
+ EXPECT_TRUE(IsUnspecOrEmptyIP(client1->GetLocalAddress().ipaddr()));
std::unique_ptr<AsyncSocket> client2(
server_->CreateAsyncSocket(loopback.family(), SOCK_STREAM));
sink.Monitor(client2.get());
EXPECT_EQ(AsyncSocket::CS_CLOSED, client2->GetState());
- EXPECT_PRED1(IsUnspecOrEmptyIP, client2->GetLocalAddress().ipaddr());
+ EXPECT_TRUE(IsUnspecOrEmptyIP(client2->GetLocalAddress().ipaddr()));
// Create server and listen.
std::unique_ptr<AsyncSocket> server(
diff --git a/rtc_base/platform_file.cc b/rtc_base/platform_file.cc
index 35a2622..a4c906a 100644
--- a/rtc_base/platform_file.cc
+++ b/rtc_base/platform_file.cc
@@ -23,17 +23,21 @@
namespace rtc {
+FILE* FdopenPlatformFileForWriting(PlatformFile file) {
+ return FdopenPlatformFile(file, "w");
+}
+
#if defined(WEBRTC_WIN)
const PlatformFile kInvalidPlatformFileValue = INVALID_HANDLE_VALUE;
-FILE* FdopenPlatformFileForWriting(PlatformFile file) {
+FILE* FdopenPlatformFile(PlatformFile file, const char* modes) {
if (file == kInvalidPlatformFileValue)
return nullptr;
int fd = _open_osfhandle(reinterpret_cast<intptr_t>(file), 0);
if (fd < 0)
return nullptr;
- return _fdopen(fd, "w");
+ return _fdopen(fd, modes);
}
bool ClosePlatformFile(PlatformFile file) {
@@ -49,6 +53,11 @@
nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
}
+PlatformFile OpenPlatformFileReadOnly(const std::string& path) {
+ return ::CreateFile(ToUtf16(path).c_str(), GENERIC_READ, FILE_SHARE_READ,
+ nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+}
+
PlatformFile CreatePlatformFile(const std::string& path) {
return ::CreateFile(ToUtf16(path).c_str(), GENERIC_READ | GENERIC_WRITE, 0,
nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
@@ -58,8 +67,8 @@
const PlatformFile kInvalidPlatformFileValue = -1;
-FILE* FdopenPlatformFileForWriting(PlatformFile file) {
- return fdopen(file, "w");
+FILE* FdopenPlatformFile(PlatformFile file, const char* modes) {
+ return fdopen(file, modes);
}
bool ClosePlatformFile(PlatformFile file) {
@@ -74,6 +83,10 @@
return ::open(path.c_str(), O_RDWR);
}
+PlatformFile OpenPlatformFileReadOnly(const std::string& path) {
+ return ::open(path.c_str(), O_RDONLY);
+}
+
PlatformFile CreatePlatformFile(const std::string& path) {
return ::open(path.c_str(), O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
}
diff --git a/rtc_base/platform_file.h b/rtc_base/platform_file.h
index 8e911be..52fbaff 100644
--- a/rtc_base/platform_file.h
+++ b/rtc_base/platform_file.h
@@ -35,6 +35,11 @@
// the PlatformFile should no longer be used.
FILE* FdopenPlatformFileForWriting(PlatformFile file);
+// Associates a standard FILE stream with an existing PlatformFile.
+// Note that after this function has returned a valid FILE stream,
+// the PlatformFile should no longer be used.
+FILE* FdopenPlatformFile(PlatformFile file, const char* modes);
+
// Closes a PlatformFile. Returns true on success, false on failure.
// Don't use ClosePlatformFile to close a file opened with FdopenPlatformFile.
// Use fclose instead.
@@ -47,6 +52,10 @@
// instead.
PlatformFile OpenPlatformFile(const std::string& path);
+// Opens a file for reading only. You might want to use base/file.h
+// instead.
+PlatformFile OpenPlatformFileReadOnly(const std::string& path);
+
// Creates a new file for reading and writing. If the file already exists it
// will be overwritten. You might want to use base/file.h instead.
PlatformFile CreatePlatformFile(const std::string& path);
diff --git a/rtc_base/platform_file_unittest.cc b/rtc_base/platform_file_unittest.cc
new file mode 100644
index 0000000..4e72400
--- /dev/null
+++ b/rtc_base/platform_file_unittest.cc
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/platform_file.h"
+#include "test/gtest.h"
+#include "test/testsupport/fileutils.h"
+
+namespace rtc {
+
+void FillWithDummyDataAndClose(FILE* const file, const std::string& filename) {
+ EXPECT_GT(fprintf(file, "%s", "Dummy data"), 0)
+ << "Failed to write to file: " << filename;
+ fclose(file);
+}
+
+TEST(PlatformFileTest, CreateWriteAndDelete) {
+ const std::string filename = webrtc::test::GenerateTempFilename(
+ webrtc::test::OutputPath(), ".testfile");
+ const PlatformFile fd = rtc::CreatePlatformFile(filename);
+ ASSERT_NE(fd, rtc::kInvalidPlatformFileValue)
+ << "Failed to create file descriptor for file: " << filename;
+ FILE* const file = rtc::FdopenPlatformFile(fd, "w");
+ ASSERT_TRUE(file != nullptr) << "Failed to open file: " << filename;
+ FillWithDummyDataAndClose(file, filename);
+ webrtc::test::RemoveFile(filename);
+}
+
+TEST(PlatformFileTest, OpenExistingWriteAndDelete) {
+ const std::string filename = webrtc::test::GenerateTempFilename(
+ webrtc::test::OutputPath(), ".testfile");
+
+ // Create file with dummy data.
+ FILE* file = fopen(filename.c_str(), "wb");
+ ASSERT_TRUE(file != nullptr) << "Failed to open file: " << filename;
+ FillWithDummyDataAndClose(file, filename);
+
+ // Open it for write, write and delete.
+ const PlatformFile fd = rtc::OpenPlatformFile(filename);
+ ASSERT_NE(fd, rtc::kInvalidPlatformFileValue)
+ << "Failed to open file descriptor for file: " << filename;
+ file = rtc::FdopenPlatformFile(fd, "w");
+ ASSERT_TRUE(file != nullptr) << "Failed to open file: " << filename;
+ FillWithDummyDataAndClose(file, filename);
+ webrtc::test::RemoveFile(filename);
+}
+
+TEST(PlatformFileTest, OpenExistingReadOnlyAndDelete) {
+ const std::string filename = webrtc::test::GenerateTempFilename(
+ webrtc::test::OutputPath(), ".testfile");
+
+ // Create file with dummy data.
+ FILE* file = fopen(filename.c_str(), "wb");
+ ASSERT_TRUE(file != nullptr) << "Failed to open file: " << filename;
+ FillWithDummyDataAndClose(file, filename);
+
+ // Open it for read, read and delete.
+ const PlatformFile fd = rtc::OpenPlatformFileReadOnly(filename);
+ ASSERT_NE(fd, rtc::kInvalidPlatformFileValue)
+ << "Failed to open file descriptor for file: " << filename;
+ file = rtc::FdopenPlatformFile(fd, "r");
+ ASSERT_TRUE(file != nullptr) << "Failed to open file: " << filename;
+
+ int buf[]{0};
+ ASSERT_GT(fread(&buf, 1, 1, file), 0u)
+ << "Failed to read from file: " << filename;
+ fclose(file);
+ webrtc::test::RemoveFile(filename);
+}
+
+} // namespace rtc
diff --git a/rtc_base/platform_thread.cc b/rtc_base/platform_thread.cc
index ca2ce13..79d9d53 100644
--- a/rtc_base/platform_thread.cc
+++ b/rtc_base/platform_thread.cc
@@ -13,7 +13,6 @@
#include "rtc_base/atomicops.h"
#include "rtc_base/checks.h"
#include "rtc_base/timeutils.h"
-#include "rtc_base/trace_event.h"
#if defined(WEBRTC_LINUX)
#include <sys/prctl.h>
@@ -178,8 +177,6 @@
#endif
do {
- TRACE_EVENT1("webrtc", "PlatformThread::Run", "name", name_.c_str());
-
// The interface contract of Start/Stop is that for a successful call to
// Start, there should be at least one call to the run function. So we
// call the function before checking |stop_|.
diff --git a/rtc_base/ratelimiter.h b/rtc_base/ratelimiter.h
deleted file mode 100644
index 8aa84aa..0000000
--- a/rtc_base/ratelimiter.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2018 The WebRTC Project Authors. All rights reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef RTC_BASE_RATELIMITER_H_
-#define RTC_BASE_RATELIMITER_H_
-
-#include "rtc_base/data_rate_limiter.h"
-
-namespace rtc {
-// Deprecated, use DataRateLimiter instead
-class RateLimiter : public DataRateLimiter {
- public:
- using DataRateLimiter::DataRateLimiter;
-};
-} // namespace rtc
-
-#endif // RTC_BASE_RATELIMITER_H_
diff --git a/rtc_base/sanitizer.h b/rtc_base/sanitizer.h
index 1b94e1e..23a748f 100644
--- a/rtc_base/sanitizer.h
+++ b/rtc_base/sanitizer.h
@@ -11,7 +11,11 @@
#ifndef RTC_BASE_SANITIZER_H_
#define RTC_BASE_SANITIZER_H_
-#include <stddef.h> // for size_t
+#include <stddef.h> // For size_t.
+
+#ifdef __cplusplus
+#include <type_traits>
+#endif
#if defined(__has_feature)
#if __has_feature(address_sanitizer)
@@ -90,6 +94,17 @@
#ifdef __cplusplus
namespace rtc {
+namespace sanitizer_impl {
+
+template <typename T>
+constexpr bool IsTriviallyCopyable() {
+ return static_cast<bool>(std::is_trivially_copy_constructible<T>::value &&
+ (std::is_trivially_copy_assignable<T>::value ||
+ !std::is_copy_assignable<T>::value) &&
+ std::is_trivially_destructible<T>::value);
+}
+
+} // namespace sanitizer_impl
template <typename T>
inline void AsanPoison(const T& mem) {
@@ -107,6 +122,15 @@
}
template <typename T>
+inline T MsanUninitialized(T t) {
+ // TODO(bugs.webrtc.org/8762): Switch to std::is_trivially_copyable when it
+ // becomes available in downstream projects.
+ static_assert(sanitizer_impl::IsTriviallyCopyable<T>(), "");
+ rtc_MsanMarkUninitialized(&t, sizeof(T), 1);
+ return t;
+}
+
+template <typename T>
inline void MsanCheckInitialized(const T& mem) {
rtc_MsanCheckInitialized(mem.data(), sizeof(mem.data()[0]), mem.size());
}
diff --git a/rtc_base/sanitizer_unittest.cc b/rtc_base/sanitizer_unittest.cc
new file mode 100644
index 0000000..21ef432
--- /dev/null
+++ b/rtc_base/sanitizer_unittest.cc
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/sanitizer.h"
+
+#include "rtc_base/gunit.h"
+#include "rtc_base/logging.h"
+
+#if RTC_HAS_MSAN
+#include <sanitizer/msan_interface.h>
+#endif
+
+namespace rtc {
+namespace {
+
+// Test sanitizer_impl::IsTriviallyCopyable (at compile time).
+
+// Trivially copyable.
+
+struct TrTrTr {
+ TrTrTr(const TrTrTr&) = default;
+ TrTrTr& operator=(const TrTrTr&) = default;
+ ~TrTrTr() = default;
+};
+static_assert(sanitizer_impl::IsTriviallyCopyable<TrTrTr>(), "");
+
+struct TrDeTr {
+ TrDeTr(const TrDeTr&) = default;
+ TrDeTr& operator=(const TrDeTr&) = delete;
+ ~TrDeTr() = default;
+};
+static_assert(sanitizer_impl::IsTriviallyCopyable<TrDeTr>(), "");
+
+// Non trivially copyable.
+
+struct TrTrNt {
+ TrTrNt(const TrTrNt&) = default;
+ TrTrNt& operator=(const TrTrNt&) = default;
+ ~TrTrNt();
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<TrTrNt>(), "");
+
+struct TrNtTr {
+ TrNtTr(const TrNtTr&) = default;
+ TrNtTr& operator=(const TrNtTr&);
+ ~TrNtTr() = default;
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<TrNtTr>(), "");
+
+struct TrNtNt {
+ TrNtNt(const TrNtNt&) = default;
+ TrNtNt& operator=(const TrNtNt&);
+ ~TrNtNt();
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<TrNtNt>(), "");
+
+struct TrDeNt {
+ TrDeNt(const TrDeNt&) = default;
+ TrDeNt& operator=(const TrDeNt&) = delete;
+ ~TrDeNt();
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<TrDeNt>(), "");
+
+struct NtTrTr {
+ NtTrTr(const NtTrTr&);
+ NtTrTr& operator=(const NtTrTr&) = default;
+ ~NtTrTr() = default;
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<NtTrTr>(), "");
+
+struct NtTrNt {
+ NtTrNt(const NtTrNt&);
+ NtTrNt& operator=(const NtTrNt&) = default;
+ ~NtTrNt();
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<NtTrNt>(), "");
+
+struct NtNtTr {
+ NtNtTr(const NtNtTr&);
+ NtNtTr& operator=(const NtNtTr&);
+ ~NtNtTr() = default;
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<NtNtTr>(), "");
+
+struct NtNtNt {
+ NtNtNt(const NtNtNt&);
+ NtNtNt& operator=(const NtNtNt&);
+ ~NtNtNt();
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<NtNtNt>(), "");
+
+struct NtDeTr {
+ NtDeTr(const NtDeTr&);
+ NtDeTr& operator=(const NtDeTr&) = delete;
+ ~NtDeTr() = default;
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<NtDeTr>(), "");
+
+struct NtDeNt {
+ NtDeNt(const NtDeNt&);
+ NtDeNt& operator=(const NtDeNt&) = delete;
+ ~NtDeNt();
+};
+static_assert(!sanitizer_impl::IsTriviallyCopyable<NtDeNt>(), "");
+
+// Trivially copyable types.
+
+struct Foo {
+ uint32_t field1;
+ uint16_t field2;
+};
+
+struct Bar {
+ uint32_t ID;
+ Foo foo;
+};
+
+// Run the callback, and crash if it *doesn't* make an uninitialized memory
+// read. If MSan isn't on, just run the callback.
+template <typename F>
+void MsanExpectUninitializedRead(F&& f) {
+#if RTC_HAS_MSAN
+ // Allow uninitialized memory reads.
+ RTC_LOG(LS_INFO) << "__msan_set_expect_umr(1)";
+ __msan_set_expect_umr(1);
+#endif
+ f();
+#if RTC_HAS_MSAN
+ // Disallow uninitialized memory reads again, and verify that at least
+ // one uninitialized memory read happened while we weren't looking.
+ RTC_LOG(LS_INFO) << "__msan_set_expect_umr(0)";
+ __msan_set_expect_umr(0);
+#endif
+}
+
+} // namespace
+
+// TODO(b/9116): Enable the test when the bug is fixed.
+TEST(SanitizerTest, DISABLED_MsanUninitialized) {
+ Bar bar = MsanUninitialized<Bar>({});
+ // Check that a read after initialization is OK.
+ bar.ID = 1;
+ EXPECT_EQ(1u, bar.ID);
+ RTC_LOG(LS_INFO) << "read after init passed";
+ // Check that other fields are uninitialized and equal to zero.
+ MsanExpectUninitializedRead([&] { EXPECT_EQ(0u, bar.foo.field1); });
+ MsanExpectUninitializedRead([&] { EXPECT_EQ(0u, bar.foo.field2); });
+ RTC_LOG(LS_INFO) << "read with no init passed";
+}
+
+} // namespace rtc
diff --git a/rtc_base/scoped_ref_ptr.h b/rtc_base/scoped_ref_ptr.h
index 0f4698a..8fefc73 100644
--- a/rtc_base/scoped_ref_ptr.h
+++ b/rtc_base/scoped_ref_ptr.h
@@ -103,11 +103,11 @@
operator T*() const { return ptr_; }
T* operator->() const { return ptr_; }
- // Release a pointer.
- // The return value is the current pointer held by this object.
- // If this object holds a null pointer, the return value is null.
- // After this operation, this object will hold a null pointer,
- // and will not own the object any more.
+ // Returns the (possibly null) raw pointer, and makes the scoped_refptr hold a
+ // null pointer, all without touching the reference count of the underlying
+ // pointed-to object. The object is still reference counted, and the caller of
+ // release() is now the proud owner of one reference, so it is responsible for
+ // calling Release() once on the object when no longer using it.
T* release() {
T* retVal = ptr_;
ptr_ = nullptr;
diff --git a/rtc_base/sequenced_task_checker_impl.cc b/rtc_base/sequenced_task_checker_impl.cc
index d7f46ea..16069c2 100644
--- a/rtc_base/sequenced_task_checker_impl.cc
+++ b/rtc_base/sequenced_task_checker_impl.cc
@@ -14,7 +14,6 @@
#include <dispatch/dispatch.h>
#endif
-#include "rtc_base/platform_thread.h"
#include "rtc_base/sequenced_task_checker.h"
#include "rtc_base/task_queue.h"
diff --git a/rtc_base/sequenced_task_checker_impl.h b/rtc_base/sequenced_task_checker_impl.h
index 86d5ef0..293b1ac 100644
--- a/rtc_base/sequenced_task_checker_impl.h
+++ b/rtc_base/sequenced_task_checker_impl.h
@@ -11,6 +11,7 @@
#ifndef RTC_BASE_SEQUENCED_TASK_CHECKER_IMPL_H_
#define RTC_BASE_SEQUENCED_TASK_CHECKER_IMPL_H_
+#include "rtc_base/criticalsection.h"
#include "rtc_base/thread_checker.h"
namespace rtc {
diff --git a/rtc_base/socket.cc b/rtc_base/socket.cc
new file mode 100644
index 0000000..a9749a4
--- /dev/null
+++ b/rtc_base/socket.cc
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/socket.h"
+
+namespace rtc {
+
+PacketInfo::PacketInfo() = default;
+PacketInfo::PacketInfo(const PacketInfo& info) = default;
+PacketInfo::~PacketInfo() = default;
+
+SentPacket::SentPacket() = default;
+SentPacket::SentPacket(int64_t packet_id, int64_t send_time_ms)
+ : packet_id(packet_id), send_time_ms(send_time_ms) {}
+SentPacket::SentPacket(int64_t packet_id,
+ int64_t send_time_ms,
+ const rtc::PacketInfo& info)
+ : packet_id(packet_id), send_time_ms(send_time_ms), info(info) {}
+
+} // namespace rtc
diff --git a/rtc_base/socket.h b/rtc_base/socket.h
index ca1a302..ee2c73f 100644
--- a/rtc_base/socket.h
+++ b/rtc_base/socket.h
@@ -25,6 +25,7 @@
#include "rtc_base/win32.h"
#endif
+#include "api/optional.h"
#include "rtc_base/basictypes.h"
#include "rtc_base/constructormagic.h"
#include "rtc_base/socketaddress.h"
@@ -123,13 +124,48 @@
return (e == EWOULDBLOCK) || (e == EAGAIN) || (e == EINPROGRESS);
}
-struct SentPacket {
- SentPacket() : packet_id(-1), send_time_ms(-1) {}
- SentPacket(int packet_id, int64_t send_time_ms)
- : packet_id(packet_id), send_time_ms(send_time_ms) {}
+enum class PacketType {
+ kUnknown,
+ kData,
+ kIceConnectivityCheck,
+ kIceConnectivityCheckResponse,
+ kStunMessage,
+ kTurnMessage,
+};
- int packet_id;
- int64_t send_time_ms;
+enum class PacketInfoProtocolType {
+ kUnknown,
+ kUdp,
+ kTcp,
+ kSsltcp,
+ kTls,
+};
+
+struct PacketInfo {
+ PacketInfo();
+ PacketInfo(const PacketInfo& info);
+ ~PacketInfo();
+
+ PacketType packet_type = PacketType::kUnknown;
+ PacketInfoProtocolType protocol = PacketInfoProtocolType::kUnknown;
+ // A unique id assigned by the network manager, and rtc::nullopt if not set.
+ rtc::Optional<uint16_t> network_id;
+ size_t packet_size_bytes = 0;
+ size_t turn_overhead_bytes = 0;
+ SocketAddress local_socket_address;
+ SocketAddress remote_socket_address;
+};
+
+struct SentPacket {
+ SentPacket();
+ SentPacket(int64_t packet_id, int64_t send_time_ms);
+ SentPacket(int64_t packet_id,
+ int64_t send_time_ms,
+ const rtc::PacketInfo& info);
+
+ int64_t packet_id = -1;
+ int64_t send_time_ms = -1;
+ rtc::PacketInfo info;
};
// General interface for the socket implementations of various networks. The
diff --git a/rtc_base/socket_unittest.cc b/rtc_base/socket_unittest.cc
index a31cc02..be958d3 100644
--- a/rtc_base/socket_unittest.cc
+++ b/rtc_base/socket_unittest.cc
@@ -224,7 +224,7 @@
ss_->CreateAsyncSocket(loopback.family(), SOCK_STREAM));
sink.Monitor(client.get());
EXPECT_EQ(AsyncSocket::CS_CLOSED, client->GetState());
- EXPECT_PRED1(IsUnspecOrEmptyIP, client->GetLocalAddress().ipaddr());
+ EXPECT_TRUE(IsUnspecOrEmptyIP(client->GetLocalAddress().ipaddr()));
// Create server and listen.
std::unique_ptr<AsyncSocket> server(
diff --git a/rtc_base/socketaddress.cc b/rtc_base/socketaddress.cc
index 54a41d4..a58c001 100644
--- a/rtc_base/socketaddress.cc
+++ b/rtc_base/socketaddress.cc
@@ -26,12 +26,11 @@
#include <unistd.h>
#endif
-#include <sstream>
-
#include "rtc_base/byteorder.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/nethelpers.h"
+#include "rtc_base/strings/string_builder.h"
#if defined(WEBRTC_WIN)
#include "rtc_base/win32.h"
@@ -161,21 +160,21 @@
}
std::string SocketAddress::PortAsString() const {
- std::ostringstream ost;
- ost << port_;
- return ost.str();
+ return std::to_string(port_);
}
std::string SocketAddress::ToString() const {
- std::ostringstream ost;
- ost << *this;
- return ost.str();
+ char buf[1024];
+ rtc::SimpleStringBuilder sb(buf);
+ sb << HostAsURIString() << ":" << port();
+ return sb.str();
}
std::string SocketAddress::ToSensitiveString() const {
- std::ostringstream ost;
- ost << HostAsSensitiveURIString() << ":" << port();
- return ost.str();
+ char buf[1024];
+ rtc::SimpleStringBuilder sb(buf);
+ sb << HostAsSensitiveURIString() << ":" << port();
+ return sb.str();
}
bool SocketAddress::FromString(const std::string& str) {
@@ -200,11 +199,6 @@
return true;
}
-std::ostream& operator<<(std::ostream& os, const SocketAddress& addr) {
- os << addr.HostAsURIString() << ":" << addr.port();
- return os;
-}
-
bool SocketAddress::IsAnyIP() const {
return IPIsAny(ip_);
}
diff --git a/rtc_base/socketaddress.h b/rtc_base/socketaddress.h
index d58eed8..90919c2 100644
--- a/rtc_base/socketaddress.h
+++ b/rtc_base/socketaddress.h
@@ -13,6 +13,9 @@
#include <iosfwd>
#include <string>
+#ifdef UNIT_TEST
+#include <ostream> // no-presubmit-check TODO(webrtc:8982)
+#endif // UNIT_TEST
#include <vector>
#include "rtc_base/basictypes.h"
#include "rtc_base/ipaddress.h"
@@ -126,7 +129,12 @@
// Parses hostname:port and [hostname]:port.
bool FromString(const std::string& str);
- friend std::ostream& operator<<(std::ostream& os, const SocketAddress& addr);
+#ifdef UNIT_TEST
+ inline std::ostream& operator<<( // no-presubmit-check TODO(webrtc:8982)
+ std::ostream& os) { // no-presubmit-check TODO(webrtc:8982)
+ return os << HostAsURIString() << ":" << port();
+ }
+#endif // UNIT_TEST
// Determines whether this represents a missing / any IP address.
// That is, 0.0.0.0 or ::.
diff --git a/rtc_base/socketaddress_unittest.cc b/rtc_base/socketaddress_unittest.cc
index 0d168df..fb195b6 100644
--- a/rtc_base/socketaddress_unittest.cc
+++ b/rtc_base/socketaddress_unittest.cc
@@ -255,34 +255,34 @@
TEST(SocketAddressTest, TestEqualityOperators) {
SocketAddress addr1("1.2.3.4", 5678);
SocketAddress addr2("1.2.3.4", 5678);
- EXPECT_PRED2(AreEqual, addr1, addr2);
+ EXPECT_TRUE(AreEqual(addr1, addr2));
addr2 = SocketAddress("0.0.0.1", 5678);
- EXPECT_PRED2(AreUnequal, addr1, addr2);
+ EXPECT_TRUE(AreUnequal(addr1, addr2));
addr2 = SocketAddress("1.2.3.4", 1234);
- EXPECT_PRED2(AreUnequal, addr1, addr2);
+ EXPECT_TRUE(AreUnequal(addr1, addr2));
addr2 = SocketAddress(kTestV6AddrString, 5678);
- EXPECT_PRED2(AreUnequal, addr1, addr2);
+ EXPECT_TRUE(AreUnequal(addr1, addr2));
addr1 = SocketAddress(kTestV6AddrString, 5678);
- EXPECT_PRED2(AreEqual, addr1, addr2);
+ EXPECT_TRUE(AreEqual(addr1, addr2));
addr2 = SocketAddress(kTestV6AddrString, 1234);
- EXPECT_PRED2(AreUnequal, addr1, addr2);
+ EXPECT_TRUE(AreUnequal(addr1, addr2));
addr2 = SocketAddress("fe80::1", 5678);
- EXPECT_PRED2(AreUnequal, addr1, addr2);
+ EXPECT_TRUE(AreUnequal(addr1, addr2));
SocketAddress addr3("a.b.c.d", 1);
SocketAddress addr4("b.b.c.d", 1);
- EXPECT_PRED2(AreUnequal, addr3, addr4);
- EXPECT_PRED2(AreEqual, addr3, addr3);
+ EXPECT_TRUE(AreUnequal(addr3, addr4));
+ EXPECT_TRUE(AreEqual(addr3, addr3));
addr3.SetIP(addr1.ip());
addr4.SetIP(addr1.ip());
- EXPECT_PRED2(AreEqual,addr3, addr4);
+ EXPECT_TRUE(AreEqual(addr3, addr4));
}
bool IsLessThan(const SocketAddress& addr1, const SocketAddress& addr2) {
@@ -299,19 +299,19 @@
EXPECT_FALSE(addr2 < addr1);
addr2 = SocketAddress("1.2.3.4", 5679);
- EXPECT_PRED2(IsLessThan, addr1, addr2);
+ EXPECT_TRUE(IsLessThan(addr1, addr2));
addr2 = SocketAddress("2.2.3.4", 49152);
- EXPECT_PRED2(IsLessThan, addr1, addr2);
+ EXPECT_TRUE(IsLessThan(addr1, addr2));
addr2 = SocketAddress(kTestV6AddrString, 5678);
- EXPECT_PRED2(IsLessThan, addr1, addr2);
+ EXPECT_TRUE(IsLessThan(addr1, addr2));
addr1 = SocketAddress("fe80::1", 5678);
- EXPECT_PRED2(IsLessThan, addr2, addr1);
+ EXPECT_TRUE(IsLessThan(addr2, addr1));
addr2 = SocketAddress("fe80::1", 5679);
- EXPECT_PRED2(IsLessThan, addr1, addr2);
+ EXPECT_TRUE(IsLessThan(addr1, addr2));
addr2 = SocketAddress("fe80::1", 5678);
EXPECT_FALSE(addr1 < addr2);
@@ -319,7 +319,7 @@
SocketAddress addr3("a.b.c.d", 1);
SocketAddress addr4("b.b.c.d", 1);
- EXPECT_PRED2(IsLessThan, addr3, addr4);
+ EXPECT_TRUE(IsLessThan(addr3, addr4));
}
TEST(SocketAddressTest, TestToSensitiveString) {
diff --git a/rtc_base/ssladapter_unittest.cc b/rtc_base/ssladapter_unittest.cc
index 0996b01..c15ecfe 100644
--- a/rtc_base/ssladapter_unittest.cc
+++ b/rtc_base/ssladapter_unittest.cc
@@ -81,7 +81,7 @@
}
int Connect(const std::string& hostname, const rtc::SocketAddress& address) {
- RTC_LOG(LS_INFO) << "Initiating connection with " << address;
+ RTC_LOG(LS_INFO) << "Initiating connection with " << address.ToString();
int rv = ssl_adapter_->Connect(address);
@@ -157,7 +157,7 @@
RTC_LOG(LS_INFO) << ((ssl_mode_ == rtc::SSL_MODE_DTLS) ? "UDP" : "TCP")
<< " server listening on "
- << server_socket_->GetLocalAddress();
+ << server_socket_->GetLocalAddress().ToString();
}
rtc::SocketAddress GetAddress() const {
diff --git a/rtc_base/sslstreamadapter.cc b/rtc_base/sslstreamadapter.cc
index b09c144..d52dc45 100644
--- a/rtc_base/sslstreamadapter.cc
+++ b/rtc_base/sslstreamadapter.cc
@@ -105,7 +105,11 @@
// Note: SRTP_AES128_CM_SHA1_80 is what is required to be supported (by
// draft-ietf-rtcweb-security-arch), but SRTP_AES128_CM_SHA1_32 is allowed as
// well, and saves a few bytes per packet if it ends up selected.
- crypto_suites.push_back(rtc::SRTP_AES128_CM_SHA1_32);
+ // As the cipher suite is potentially insecure, it will only be used if
+ // enabled by both peers.
+ if (crypto_options.enable_aes128_sha1_32_crypto_cipher) {
+ crypto_suites.push_back(rtc::SRTP_AES128_CM_SHA1_32);
+ }
crypto_suites.push_back(rtc::SRTP_AES128_CM_SHA1_80);
return crypto_suites;
}
diff --git a/rtc_base/sslstreamadapter.h b/rtc_base/sslstreamadapter.h
index c04fb34..827dc45 100644
--- a/rtc_base/sslstreamadapter.h
+++ b/rtc_base/sslstreamadapter.h
@@ -80,6 +80,12 @@
// if both sides enable it.
bool enable_gcm_crypto_suites = false;
+ // If set to true, the (potentially insecure) crypto cipher
+ // SRTP_AES128_CM_SHA1_32 will be included in the list of supported ciphers
+ // during negotiation. It will only be used if both peers support it and no
+ // other ciphers get preferred.
+ bool enable_aes128_sha1_32_crypto_cipher = false;
+
// If set to true, encrypted RTP header extensions as defined in RFC 6904
// will be negotiated. They will only be used if both peers support them.
bool enable_encrypted_rtp_header_extensions = false;
diff --git a/rtc_base/string_to_number.cc b/rtc_base/string_to_number.cc
index ad49d64..fe17f34 100644
--- a/rtc_base/string_to_number.cc
+++ b/rtc_base/string_to_number.cc
@@ -8,10 +8,12 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "rtc_base/string_to_number.h"
+
#include <cerrno>
#include <cstdlib>
-#include "rtc_base/string_to_number.h"
+#include "rtc_base/checks.h"
namespace rtc {
namespace string_to_number_internal {
diff --git a/rtc_base/string_to_number_unittest.cc b/rtc_base/string_to_number_unittest.cc
index f5e5b57..538ee10 100644
--- a/rtc_base/string_to_number_unittest.cc
+++ b/rtc_base/string_to_number_unittest.cc
@@ -42,15 +42,16 @@
using T = TypeParam;
constexpr T min_value = std::numeric_limits<T>::lowest();
constexpr T max_value = std::numeric_limits<T>::max();
+ constexpr T zero_value = 0;
const std::string min_string = std::to_string(min_value);
const std::string max_string = std::to_string(max_value);
EXPECT_EQ(min_value, StringToNumber<T>(min_string));
EXPECT_EQ(min_value, StringToNumber<T>(min_string.c_str()));
EXPECT_EQ(max_value, StringToNumber<T>(max_string));
EXPECT_EQ(max_value, StringToNumber<T>(max_string.c_str()));
- EXPECT_EQ(0, StringToNumber<T>("0"));
- EXPECT_EQ(0, StringToNumber<T>("-0"));
- EXPECT_EQ(0, StringToNumber<T>(std::string("-0000000000000")));
+ EXPECT_EQ(zero_value, StringToNumber<T>("0"));
+ EXPECT_EQ(zero_value, StringToNumber<T>("-0"));
+ EXPECT_EQ(zero_value, StringToNumber<T>(std::string("-0000000000000")));
}
TYPED_TEST_P(BasicNumberTest, TestInvalidNumbers) {
diff --git a/rtc_base/stringencode.cc b/rtc_base/stringencode.cc
index 755cb2c..f2a1508 100644
--- a/rtc_base/stringencode.cc
+++ b/rtc_base/stringencode.cc
@@ -139,7 +139,7 @@
size_t hex_encode_with_delimiter(char* buffer, size_t buflen,
const char* csource, size_t srclen,
char delimiter) {
- RTC_DCHECK(buffer); // TODO(grunell): estimate output size
+ RTC_DCHECK(buffer); // TODO(kwiberg): estimate output size
if (buflen == 0)
return 0;
@@ -195,7 +195,7 @@
size_t hex_decode_with_delimiter(char* cbuffer, size_t buflen,
const char* source, size_t srclen,
char delimiter) {
- RTC_DCHECK(cbuffer); // TODO(grunell): estimate output size
+ RTC_DCHECK(cbuffer); // TODO(kwiberg): estimate output size
if (buflen == 0)
return 0;
diff --git a/rtc_base/strings/audio_format_to_string.cc b/rtc_base/strings/audio_format_to_string.cc
new file mode 100644
index 0000000..2d0b53b
--- /dev/null
+++ b/rtc_base/strings/audio_format_to_string.cc
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/strings/audio_format_to_string.h"
+
+#include "rtc_base/strings/string_builder.h"
+
+namespace rtc {
+ std::string ToString(const webrtc::SdpAudioFormat& saf) {
+ char sb_buf[1024];
+ rtc::SimpleStringBuilder sb(sb_buf);
+ sb << "{name: " << saf.name;
+ sb << ", clockrate_hz: " << saf.clockrate_hz;
+ sb << ", num_channels: " << saf.num_channels;
+ sb << ", parameters: {";
+ const char* sep = "";
+ for (const auto& kv : saf.parameters) {
+ sb << sep << kv.first << ": " << kv.second;
+ sep = ", ";
+ }
+ sb << "}}";
+ return sb.str();
+ }
+ std::string ToString(const webrtc::AudioCodecInfo& aci) {
+ char sb_buf[1024];
+ rtc::SimpleStringBuilder sb(sb_buf);
+ sb << "{sample_rate_hz: " << aci.sample_rate_hz;
+ sb << ", num_channels: " << aci.num_channels;
+ sb << ", default_bitrate_bps: " << aci.default_bitrate_bps;
+ sb << ", min_bitrate_bps: " << aci.min_bitrate_bps;
+ sb << ", max_bitrate_bps: " << aci.max_bitrate_bps;
+ sb << ", allow_comfort_noise: " << aci.allow_comfort_noise;
+ sb << ", supports_network_adaption: " << aci.supports_network_adaption;
+ sb << "}";
+ return sb.str();
+ }
+ std::string ToString(const webrtc::AudioCodecSpec& acs) {
+ char sb_buf[1024];
+ rtc::SimpleStringBuilder sb(sb_buf);
+ sb << "{format: " << ToString(acs.format);
+ sb << ", info: " << ToString(acs.info);
+ sb << "}";
+ return sb.str();
+ }
+} // namespace rtc
diff --git a/rtc_base/strings/audio_format_to_string.h b/rtc_base/strings/audio_format_to_string.h
new file mode 100644
index 0000000..de0ce16
--- /dev/null
+++ b/rtc_base/strings/audio_format_to_string.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef RTC_BASE_STRINGS_AUDIO_FORMAT_TO_STRING_H_
+#define RTC_BASE_STRINGS_AUDIO_FORMAT_TO_STRING_H_
+
+#include <string>
+
+#include "api/audio_codecs/audio_format.h"
+
+namespace rtc {
+ std::string ToString(const webrtc::SdpAudioFormat& saf);
+ std::string ToString(const webrtc::AudioCodecInfo& saf);
+ std::string ToString(const webrtc::AudioCodecSpec& acs);
+} // namespace rtc
+
+#endif // RTC_BASE_STRINGS_AUDIO_FORMAT_TO_STRING_H_
diff --git a/rtc_base/strings/string_builder.cc b/rtc_base/strings/string_builder.cc
index 528f099..70a14da 100644
--- a/rtc_base/strings/string_builder.cc
+++ b/rtc_base/strings/string_builder.cc
@@ -67,15 +67,15 @@
}
SimpleStringBuilder& SimpleStringBuilder::operator<<(float f) {
- return AppendFormat("%f", f);
+ return AppendFormat("%g", f);
}
SimpleStringBuilder& SimpleStringBuilder::operator<<(double f) {
- return AppendFormat("%f", f);
+ return AppendFormat("%g", f);
}
SimpleStringBuilder& SimpleStringBuilder::operator<<(long double f) {
- return AppendFormat("%Lf", f);
+ return AppendFormat("%Lg", f);
}
SimpleStringBuilder& SimpleStringBuilder::AppendFormat(const char* fmt, ...) {
diff --git a/rtc_base/strings/string_builder_unittest.cc b/rtc_base/strings/string_builder_unittest.cc
index 8d6312f..aa570eb 100644
--- a/rtc_base/strings/string_builder_unittest.cc
+++ b/rtc_base/strings/string_builder_unittest.cc
@@ -33,7 +33,7 @@
SimpleStringBuilder sb(sb_buf);
sb << 1 << ':' << 2.1 << ":" << 2.2f << ':' << 78187493520ll << ':'
<< 78187493520ul;
- EXPECT_EQ(0, strcmp(sb.str(), "1:2.100000:2.200000:78187493520:78187493520"));
+ EXPECT_EQ(0, strcmp(sb.str(), "1:2.1:2.2:78187493520:78187493520"));
}
TEST(SimpleStringBuilder, Format) {
diff --git a/rtc_base/stringutils.cc b/rtc_base/stringutils.cc
index 8671b52..7664bb8 100644
--- a/rtc_base/stringutils.cc
+++ b/rtc_base/stringutils.cc
@@ -7,6 +7,8 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <algorithm>
+#include <cstdio>
#include "rtc_base/stringutils.h"
#include "rtc_base/checks.h"
@@ -130,4 +132,11 @@
return s.substr(first, last - first + 1);
}
+std::string ToHex(const int i) {
+ char buffer[50];
+ snprintf(buffer, sizeof(buffer), "%x", i);
+
+ return std::string(buffer);
+}
+
} // namespace rtc
diff --git a/rtc_base/stringutils.h b/rtc_base/stringutils.h
index 686402c..b42cfa5 100644
--- a/rtc_base/stringutils.h
+++ b/rtc_base/stringutils.h
@@ -78,7 +78,7 @@
return strncasecmp(s1, s2, n);
}
-#endif // WEBRTC_POSIX
+#endif // WEBRTC_POSIX
///////////////////////////////////////////////////////////////////////////////
// Traits simplifies porting string functions to be CTYPE-agnostic
@@ -91,9 +91,9 @@
template<class CTYPE>
struct Traits {
// STL string type
- //typedef XXX string;
+ // typedef XXX string;
// Null-terminated string
- //inline static const CTYPE* empty_str();
+ // inline static const CTYPE* empty_str();
};
///////////////////////////////////////////////////////////////////////////////
@@ -119,7 +119,7 @@
template<class CTYPE>
const CTYPE* strchrn(const CTYPE* str, size_t slen, CTYPE ch) {
- for (size_t i=0; i<slen && str[i]; ++i) {
+ for (size_t i=0; i < slen && str[i]; ++i) {
if (str[i] == ch) {
return str + i;
}
@@ -312,6 +312,8 @@
// Remove leading and trailing whitespaces.
std::string string_trim(const std::string& s);
+// TODO(jonasolsson): replace with absl::Hex when that becomes available.
+std::string ToHex(const int i);
} // namespace rtc
-#endif // RTC_BASE_STRINGUTILS_H_
+#endif // RTC_BASE_STRINGUTILS_H_
diff --git a/rtc_base/stringutils_unittest.cc b/rtc_base/stringutils_unittest.cc
index 85d0c3c..bb0e7b5 100644
--- a/rtc_base/stringutils_unittest.cc
+++ b/rtc_base/stringutils_unittest.cc
@@ -16,10 +16,10 @@
// Tests for string_match().
TEST(string_matchTest, Matches) {
- EXPECT_TRUE( string_match("A.B.C.D", "a.b.c.d"));
- EXPECT_TRUE( string_match("www.TEST.GOOGLE.COM", "www.*.com"));
- EXPECT_TRUE( string_match("127.0.0.1", "12*.0.*1"));
- EXPECT_TRUE( string_match("127.1.0.21", "12*.0.*1"));
+ EXPECT_TRUE(string_match("A.B.C.D", "a.b.c.d"));
+ EXPECT_TRUE(string_match("www.TEST.GOOGLE.COM", "www.*.com"));
+ EXPECT_TRUE(string_match("127.0.0.1", "12*.0.*1"));
+ EXPECT_TRUE(string_match("127.1.0.21", "12*.0.*1"));
EXPECT_FALSE(string_match("127.0.0.0", "12*.0.*1"));
EXPECT_FALSE(string_match("127.0.0.0", "12*.0.*1"));
EXPECT_FALSE(string_match("127.1.1.21", "12*.0.*1"));
@@ -75,7 +75,7 @@
EXPECT_EQ(1, ascii_string_compare(L"xyz", "xy", 5, identity));
EXPECT_EQ(1, ascii_string_compare(L"abc", "ABB", 5, tolowercase));
}
-#endif // WEBRTC_WIN
+#endif // WEBRTC_WIN
TEST(string_trim_Test, Trimming) {
EXPECT_EQ("temp", string_trim("\n\r\t temp \n\r\t"));
@@ -105,4 +105,10 @@
EXPECT_FALSE(ends_with("", "f"));
}
-} // namespace rtc
+TEST(string_toHexTest, ToHex) {
+ EXPECT_EQ(ToHex(0), "0");
+ EXPECT_EQ(ToHex(0X1243E), "1243e");
+ EXPECT_EQ(ToHex(-20), "ffffffec");
+}
+
+} // namespace rtc
diff --git a/rtc_base/synchronization/BUILD.gn b/rtc_base/synchronization/BUILD.gn
new file mode 100644
index 0000000..9d9ea6f
--- /dev/null
+++ b/rtc_base/synchronization/BUILD.gn
@@ -0,0 +1,38 @@
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../webrtc.gni")
+if (is_android) {
+ import("//build/config/android/config.gni")
+ import("//build/config/android/rules.gni")
+}
+
+rtc_source_set("rw_lock_wrapper") {
+ public = [
+ "rw_lock_wrapper.h",
+ ]
+ sources = [
+ "rw_lock_wrapper.cc",
+ ]
+ deps = [
+ "..:macromagic",
+ "../..:typedefs",
+ ]
+ if (is_win) {
+ sources += [
+ "rw_lock_win.cc",
+ "rw_lock_win.h",
+ ]
+ deps += [ "..:logging" ]
+ } else {
+ sources += [
+ "rw_lock_posix.cc",
+ "rw_lock_posix.h",
+ ]
+ }
+}
diff --git a/system_wrappers/source/rw_lock_posix.cc b/rtc_base/synchronization/rw_lock_posix.cc
similarity index 95%
rename from system_wrappers/source/rw_lock_posix.cc
rename to rtc_base/synchronization/rw_lock_posix.cc
index 412873c..7e37dc9 100644
--- a/system_wrappers/source/rw_lock_posix.cc
+++ b/rtc_base/synchronization/rw_lock_posix.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/source/rw_lock_posix.h"
+#include "rtc_base/synchronization/rw_lock_posix.h"
namespace webrtc {
diff --git a/system_wrappers/source/rw_lock_posix.h b/rtc_base/synchronization/rw_lock_posix.h
similarity index 80%
rename from system_wrappers/source/rw_lock_posix.h
rename to rtc_base/synchronization/rw_lock_posix.h
index d15682b..9a92bcd 100644
--- a/system_wrappers/source/rw_lock_posix.h
+++ b/rtc_base/synchronization/rw_lock_posix.h
@@ -8,14 +8,14 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_SOURCE_RW_LOCK_POSIX_H_
-#define SYSTEM_WRAPPERS_SOURCE_RW_LOCK_POSIX_H_
-
-#include "system_wrappers/include/rw_lock_wrapper.h"
-#include "typedefs.h" // NOLINT(build/include)
+#ifndef RTC_BASE_SYNCHRONIZATION_RW_LOCK_POSIX_H_
+#define RTC_BASE_SYNCHRONIZATION_RW_LOCK_POSIX_H_
#include <pthread.h>
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
+#include "typedefs.h" // NOLINT(build/include)
+
namespace webrtc {
class RWLockPosix : public RWLockWrapper {
@@ -38,4 +38,4 @@
} // namespace webrtc
-#endif // SYSTEM_WRAPPERS_SOURCE_RW_LOCK_POSIX_H_
+#endif // RTC_BASE_SYNCHRONIZATION_RW_LOCK_POSIX_H_
diff --git a/system_wrappers/source/rw_lock_win.cc b/rtc_base/synchronization/rw_lock_win.cc
similarity index 97%
rename from system_wrappers/source/rw_lock_win.cc
rename to rtc_base/synchronization/rw_lock_win.cc
index 23df15a..44cc0a7 100644
--- a/system_wrappers/source/rw_lock_win.cc
+++ b/rtc_base/synchronization/rw_lock_win.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/source/rw_lock_win.h"
+#include "rtc_base/synchronization/rw_lock_win.h"
#include "rtc_base/logging.h"
diff --git a/system_wrappers/source/rw_lock_win.h b/rtc_base/synchronization/rw_lock_win.h
similarity index 63%
rename from system_wrappers/source/rw_lock_win.h
rename to rtc_base/synchronization/rw_lock_win.h
index 41537ba..52ad9bb 100644
--- a/system_wrappers/source/rw_lock_win.h
+++ b/rtc_base/synchronization/rw_lock_win.h
@@ -8,10 +8,10 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_SOURCE_RW_LOCK_WIN_H_
-#define SYSTEM_WRAPPERS_SOURCE_RW_LOCK_WIN_H_
+#ifndef RTC_BASE_SYNCHRONIZATION_RW_LOCK_WIN_H_
+#define RTC_BASE_SYNCHRONIZATION_RW_LOCK_WIN_H_
-#include "system_wrappers/include/rw_lock_wrapper.h"
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
#include <Windows.h>
@@ -20,13 +20,12 @@
class RWLockWin : public RWLockWrapper {
public:
static RWLockWin* Create();
- ~RWLockWin() {}
- virtual void AcquireLockExclusive();
- virtual void ReleaseLockExclusive();
+ void AcquireLockExclusive() override;
+ void ReleaseLockExclusive() override;
- virtual void AcquireLockShared();
- virtual void ReleaseLockShared();
+ void AcquireLockShared() override;
+ void ReleaseLockShared() override;
private:
RWLockWin();
@@ -37,4 +36,4 @@
} // namespace webrtc
-#endif // SYSTEM_WRAPPERS_SOURCE_RW_LOCK_WIN_H_
+#endif // RTC_BASE_SYNCHRONIZATION_RW_LOCK_WIN_H_
diff --git a/system_wrappers/source/rw_lock.cc b/rtc_base/synchronization/rw_lock_wrapper.cc
similarity index 79%
rename from system_wrappers/source/rw_lock.cc
rename to rtc_base/synchronization/rw_lock_wrapper.cc
index c38c44a..c8cd17e 100644
--- a/system_wrappers/source/rw_lock.cc
+++ b/rtc_base/synchronization/rw_lock_wrapper.cc
@@ -8,14 +8,14 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/include/rw_lock_wrapper.h"
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
#include <assert.h>
#if defined(_WIN32)
-#include "system_wrappers/source/rw_lock_win.h"
+#include "rtc_base/synchronization/rw_lock_win.h"
#else
-#include "system_wrappers/source/rw_lock_posix.h"
+#include "rtc_base/synchronization/rw_lock_posix.h"
#endif
namespace webrtc {
diff --git a/system_wrappers/include/rw_lock_wrapper.h b/rtc_base/synchronization/rw_lock_wrapper.h
similarity index 82%
rename from system_wrappers/include/rw_lock_wrapper.h
rename to rtc_base/synchronization/rw_lock_wrapper.h
index a22b6ab..39f52fc 100644
--- a/system_wrappers/include/rw_lock_wrapper.h
+++ b/rtc_base/synchronization/rw_lock_wrapper.h
@@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_INCLUDE_RW_LOCK_WRAPPER_H_
-#define SYSTEM_WRAPPERS_INCLUDE_RW_LOCK_WRAPPER_H_
+#ifndef RTC_BASE_SYNCHRONIZATION_RW_LOCK_WRAPPER_H_
+#define RTC_BASE_SYNCHRONIZATION_RW_LOCK_WRAPPER_H_
#include "rtc_base/thread_annotations.h"
@@ -35,7 +35,8 @@
// provides more compact locking syntax.
class RTC_SCOPED_LOCKABLE ReadLockScoped {
public:
- ReadLockScoped(RWLockWrapper& rw_lock) RTC_SHARED_LOCK_FUNCTION(rw_lock)
+ explicit ReadLockScoped(RWLockWrapper& rw_lock)
+ RTC_SHARED_LOCK_FUNCTION(rw_lock)
: rw_lock_(rw_lock) {
rw_lock_.AcquireLockShared();
}
@@ -48,7 +49,8 @@
class RTC_SCOPED_LOCKABLE WriteLockScoped {
public:
- WriteLockScoped(RWLockWrapper& rw_lock) RTC_EXCLUSIVE_LOCK_FUNCTION(rw_lock)
+ explicit WriteLockScoped(RWLockWrapper& rw_lock)
+ RTC_EXCLUSIVE_LOCK_FUNCTION(rw_lock)
: rw_lock_(rw_lock) {
rw_lock_.AcquireLockExclusive();
}
@@ -61,4 +63,4 @@
} // namespace webrtc
-#endif // SYSTEM_WRAPPERS_INCLUDE_RW_LOCK_WRAPPER_H_
+#endif // RTC_BASE_SYNCHRONIZATION_RW_LOCK_WRAPPER_H_
diff --git a/rtc_base/system/BUILD.gn b/rtc_base/system/BUILD.gn
index 23d802a..a3dc4c1 100644
--- a/rtc_base/system/BUILD.gn
+++ b/rtc_base/system/BUILD.gn
@@ -12,8 +12,27 @@
import("//build/config/android/rules.gni")
}
+rtc_source_set("asm_defines") {
+ sources = [
+ "asm_defines.h",
+ ]
+}
+
rtc_source_set("fallthrough") {
sources = [
"fallthrough.h",
]
}
+
+rtc_source_set("file_wrapper") {
+ sources = [
+ "file_wrapper.cc",
+ "file_wrapper.h",
+ ]
+ deps = [
+ "..:checks",
+ "..:criticalsection",
+ "../..:typedefs",
+ "../..:webrtc_common",
+ ]
+}
diff --git a/system_wrappers/include/asm_defines.h b/rtc_base/system/asm_defines.h
similarity index 90%
rename from system_wrappers/include/asm_defines.h
rename to rtc_base/system/asm_defines.h
index 7f4c80e..e1c7453 100644
--- a/system_wrappers/include/asm_defines.h
+++ b/rtc_base/system/asm_defines.h
@@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_INCLUDE_ASM_DEFINES_H_
-#define SYSTEM_WRAPPERS_INCLUDE_ASM_DEFINES_H_
+#ifndef RTC_BASE_SYSTEM_ASM_DEFINES_H_
+#define RTC_BASE_SYSTEM_ASM_DEFINES_H_
#if defined(__linux__) && defined(__ELF__)
.section .note.GNU-stack,"",%progbits
@@ -63,4 +63,4 @@
.text
-#endif // SYSTEM_WRAPPERS_INCLUDE_ASM_DEFINES_H_
+#endif // RTC_BASE_SYSTEM_ASM_DEFINES_H_
diff --git a/system_wrappers/source/file_impl.cc b/rtc_base/system/file_wrapper.cc
similarity index 97%
rename from system_wrappers/source/file_impl.cc
rename to rtc_base/system/file_wrapper.cc
index 350aaeb..72f5f25 100644
--- a/system_wrappers/source/file_impl.cc
+++ b/rtc_base/system/file_wrapper.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/include/file_wrapper.h"
+#include "rtc_base/system/file_wrapper.h"
#ifdef _WIN32
#include <Windows.h>
@@ -17,6 +17,8 @@
#include <string.h>
#endif
+#include <utility>
+
#include "rtc_base/checks.h"
namespace webrtc {
diff --git a/system_wrappers/include/file_wrapper.h b/rtc_base/system/file_wrapper.h
similarity index 79%
rename from system_wrappers/include/file_wrapper.h
rename to rtc_base/system/file_wrapper.h
index 143da13..4672cc4 100644
--- a/system_wrappers/include/file_wrapper.h
+++ b/rtc_base/system/file_wrapper.h
@@ -8,8 +8,8 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_INCLUDE_FILE_WRAPPER_H_
-#define SYSTEM_WRAPPERS_INCLUDE_FILE_WRAPPER_H_
+#ifndef RTC_BASE_SYSTEM_FILE_WRAPPER_H_
+#define RTC_BASE_SYSTEM_FILE_WRAPPER_H_
#include <stddef.h>
#include <stdio.h>
@@ -18,13 +18,12 @@
#include "rtc_base/criticalsection.h"
#include "typedefs.h" // NOLINT(build/include)
-// Implementation of an InStream and OutStream that can read (exclusive) or
-// write from/to a file.
+// Implementation that can read (exclusive) or write from/to a file.
namespace webrtc {
-// TODO(tommi): Remove the base classes, rename to rtc::File and move to base.
-class FileWrapper : public InStream, public OutStream {
+// TODO(tommi): Rename to rtc::File and move to base.
+class FileWrapper final {
public:
static const size_t kMaxFileNameSize = 1024;
@@ -34,7 +33,7 @@
static FileWrapper Open(const char* file_name_utf8, bool read_only);
FileWrapper(FILE* file, size_t max_size);
- ~FileWrapper() override;
+ ~FileWrapper();
// Support for move semantics.
FileWrapper(FileWrapper&& other);
@@ -61,9 +60,9 @@
int Flush();
// Rewinds the file to the start.
- int Rewind() override;
- int Read(void* buf, size_t length) override;
- bool Write(const void* buf, size_t length) override;
+ int Rewind();
+ int Read(void* buf, size_t length);
+ bool Write(const void* buf, size_t length);
private:
FileWrapper();
@@ -85,4 +84,4 @@
} // namespace webrtc
-#endif // SYSTEM_WRAPPERS_INCLUDE_FILE_WRAPPER_H_
+#endif // RTC_BASE_SYSTEM_FILE_WRAPPER_H_
diff --git a/rtc_base/system/module.mk b/rtc_base/system/module.mk
new file mode 100644
index 0000000..37c23f4
--- /dev/null
+++ b/rtc_base/system/module.mk
@@ -0,0 +1,5 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+include common.mk
diff --git a/rtc_base/thread_checker_impl.cc b/rtc_base/thread_checker_impl.cc
index 6ec5c91..850f2ac 100644
--- a/rtc_base/thread_checker_impl.cc
+++ b/rtc_base/thread_checker_impl.cc
@@ -12,8 +12,6 @@
#include "rtc_base/thread_checker_impl.h"
-#include "rtc_base/platform_thread.h"
-
namespace rtc {
ThreadCheckerImpl::ThreadCheckerImpl() : valid_thread_(CurrentThreadRef()) {
diff --git a/rtc_base/time/BUILD.gn b/rtc_base/time/BUILD.gn
new file mode 100644
index 0000000..f8f6244
--- /dev/null
+++ b/rtc_base/time/BUILD.gn
@@ -0,0 +1,24 @@
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../webrtc.gni")
+if (is_android) {
+ import("//build/config/android/config.gni")
+ import("//build/config/android/rules.gni")
+}
+
+rtc_source_set("timestamp_extrapolator") {
+ sources = [
+ "timestamp_extrapolator.cc",
+ "timestamp_extrapolator.h",
+ ]
+ deps = [
+ "../..:typedefs",
+ "../synchronization:rw_lock_wrapper",
+ ]
+}
diff --git a/system_wrappers/source/timestamp_extrapolator.cc b/rtc_base/time/timestamp_extrapolator.cc
similarity index 92%
rename from system_wrappers/source/timestamp_extrapolator.cc
rename to rtc_base/time/timestamp_extrapolator.cc
index b8c6ba0..bf9f726 100644
--- a/system_wrappers/source/timestamp_extrapolator.cc
+++ b/rtc_base/time/timestamp_extrapolator.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "system_wrappers/include/timestamp_extrapolator.h"
+#include "rtc_base/time/timestamp_extrapolator.h"
#include <algorithm>
@@ -178,12 +178,14 @@
// Forward wrap around
_wrapArounds++;
}
- }
- // This difference will probably be less than -2^31 if we have had a backward
- // wrap around. Since it is casted to a Word32, it should be positive.
- else if (static_cast<int32_t>(_prevWrapTimestamp - ts90khz) > 0) {
- // Backward wrap around
- _wrapArounds--;
+ } else {
+ // This difference will probably be less than -2^31 if we have had a
+ // backward wrap around. Since it is casted to a Word32, it should be
+ // positive.
+ if (static_cast<int32_t>(_prevWrapTimestamp - ts90khz) > 0) {
+ // Backward wrap around
+ _wrapArounds--;
+ }
}
_prevWrapTimestamp = ts90khz;
}
@@ -193,9 +195,9 @@
error = (error > 0) ? std::min(error, _accMaxError)
: std::max(error, -_accMaxError);
_detectorAccumulatorPos =
- std::max(_detectorAccumulatorPos + error - _accDrift, (double)0);
+ std::max(_detectorAccumulatorPos + error - _accDrift, double{0});
_detectorAccumulatorNeg =
- std::min(_detectorAccumulatorNeg + error + _accDrift, (double)0);
+ std::min(_detectorAccumulatorNeg + error + _accDrift, double{0});
if (_detectorAccumulatorPos > _alarmThreshold ||
_detectorAccumulatorNeg < -_alarmThreshold) {
// Alarm
diff --git a/system_wrappers/include/timestamp_extrapolator.h b/rtc_base/time/timestamp_extrapolator.h
similarity index 85%
rename from system_wrappers/include/timestamp_extrapolator.h
rename to rtc_base/time/timestamp_extrapolator.h
index 9418100..6638184 100644
--- a/system_wrappers/include/timestamp_extrapolator.h
+++ b/rtc_base/time/timestamp_extrapolator.h
@@ -8,10 +8,10 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef SYSTEM_WRAPPERS_INCLUDE_TIMESTAMP_EXTRAPOLATOR_H_
-#define SYSTEM_WRAPPERS_INCLUDE_TIMESTAMP_EXTRAPOLATOR_H_
+#ifndef RTC_BASE_TIME_TIMESTAMP_EXTRAPOLATOR_H_
+#define RTC_BASE_TIME_TIMESTAMP_EXTRAPOLATOR_H_
-#include "system_wrappers/include/rw_lock_wrapper.h"
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
@@ -51,4 +51,4 @@
} // namespace webrtc
-#endif // SYSTEM_WRAPPERS_INCLUDE_TIMESTAMP_EXTRAPOLATOR_H_
+#endif // RTC_BASE_TIME_TIMESTAMP_EXTRAPOLATOR_H_
diff --git a/rtc_base/timeutils.h b/rtc_base/timeutils.h
index f602d48..5eb73c7 100644
--- a/rtc_base/timeutils.h
+++ b/rtc_base/timeutils.h
@@ -148,6 +148,8 @@
return min_ == o.min_ && max_ == o.max_;
}
+ bool operator!=(const IntervalRange& o) const { return !operator==(o); }
+
private:
int min_;
int max_;
diff --git a/rtc_base/virtualsocketserver.cc b/rtc_base/virtualsocketserver.cc
index d8771e7..fab5900 100644
--- a/rtc_base/virtualsocketserver.cc
+++ b/rtc_base/virtualsocketserver.cc
@@ -409,7 +409,8 @@
} else if ((SOCK_STREAM == type_) && (CS_CONNECTING == state_)) {
CompleteConnect(data->addr, true);
} else {
- RTC_LOG(LS_VERBOSE) << "Socket at " << local_addr_ << " is not listening";
+ RTC_LOG(LS_VERBOSE) << "Socket at " << local_addr_.ToString()
+ << " is not listening";
server_->Disconnect(server_->LookupBinding(data->addr));
}
delete data;
@@ -805,7 +806,8 @@
VirtualSocket* remote = LookupBinding(remote_addr);
if (!CanInteractWith(socket, remote)) {
RTC_LOG(LS_INFO) << "Address family mismatch between "
- << socket->GetLocalAddress() << " and " << remote_addr;
+ << socket->GetLocalAddress().ToString() << " and "
+ << remote_addr.ToString();
return -1;
}
if (remote != nullptr) {
@@ -813,7 +815,7 @@
msg_queue_->PostDelayed(RTC_FROM_HERE, delay, remote, MSG_ID_CONNECT,
new MessageAddress(addr));
} else {
- RTC_LOG(LS_INFO) << "No one listening at " << remote_addr;
+ RTC_LOG(LS_INFO) << "No one listening at " << remote_addr.ToString();
msg_queue_->PostDelayed(RTC_FROM_HERE, delay, socket, MSG_ID_DISCONNECT);
}
return 0;
@@ -856,17 +858,18 @@
dummy_socket->SetLocalAddress(remote_addr);
if (!CanInteractWith(socket, dummy_socket.get())) {
RTC_LOG(LS_VERBOSE) << "Incompatible address families: "
- << socket->GetLocalAddress() << " and "
- << remote_addr;
+ << socket->GetLocalAddress().ToString() << " and "
+ << remote_addr.ToString();
return -1;
}
- RTC_LOG(LS_VERBOSE) << "No one listening at " << remote_addr;
+ RTC_LOG(LS_VERBOSE) << "No one listening at " << remote_addr.ToString();
return static_cast<int>(data_size);
}
if (!CanInteractWith(socket, recipient)) {
RTC_LOG(LS_VERBOSE) << "Incompatible address families: "
- << socket->GetLocalAddress() << " and " << remote_addr;
+ << socket->GetLocalAddress().ToString() << " and "
+ << remote_addr.ToString();
return -1;
}
diff --git a/rtc_base/win32.h b/rtc_base/win32.h
index 78f66a7..4e91687 100644
--- a/rtc_base/win32.h
+++ b/rtc_base/win32.h
@@ -54,6 +54,7 @@
enum WindowsMajorVersions {
kWindows2000 = 5,
kWindowsVista = 6,
+ kWindows10 = 10,
};
bool GetOsVersion(int* major, int* minor, int* build);
@@ -74,6 +75,11 @@
(major > kWindowsVista || (major == kWindowsVista && minor >= 2)));
}
+inline bool IsWindows10OrLater() {
+ int major;
+ return (GetOsVersion(&major, nullptr, nullptr) && (major >= kWindows10));
+}
+
// Determine the current integrity level of the process.
bool GetCurrentProcessIntegrityLevel(int* level);
diff --git a/system_wrappers/BUILD.gn b/system_wrappers/BUILD.gn
index 93549fd..af778fd 100644
--- a/system_wrappers/BUILD.gn
+++ b/system_wrappers/BUILD.gn
@@ -15,35 +15,20 @@
rtc_static_library("system_wrappers") {
visibility = [ "*" ]
sources = [
- "include/aligned_array.h",
- "include/aligned_malloc.h",
"include/clock.h",
"include/cpu_info.h",
"include/event_wrapper.h",
- "include/file_wrapper.h",
"include/ntp_time.h",
"include/rtp_to_ntp_estimator.h",
- "include/rw_lock_wrapper.h",
"include/sleep.h",
- "include/timestamp_extrapolator.h",
- "source/aligned_malloc.cc",
"source/clock.cc",
"source/cpu_features.cc",
"source/cpu_info.cc",
"source/event.cc",
- "source/event_timer_posix.cc",
- "source/event_timer_posix.h",
"source/event_timer_win.cc",
"source/event_timer_win.h",
- "source/file_impl.cc",
"source/rtp_to_ntp_estimator.cc",
- "source/rw_lock.cc",
- "source/rw_lock_posix.cc",
- "source/rw_lock_posix.h",
- "source/rw_lock_win.cc",
- "source/rw_lock_win.h",
"source/sleep.cc",
- "source/timestamp_extrapolator.cc",
]
defines = []
@@ -58,8 +43,16 @@
"../api:optional",
"../modules:module_api_public",
"../rtc_base:checks",
+ "../rtc_base/synchronization:rw_lock_wrapper",
]
+ if (is_posix || is_fuchsia) {
+ sources += [
+ "source/event_timer_posix.cc",
+ "source/event_timer_posix.h",
+ ]
+ }
+
if (is_android) {
defines += [ "WEBRTC_THREAD_RR" ]
@@ -124,12 +117,6 @@
]
}
-rtc_source_set("asm_defines") {
- sources = [
- "include/asm_defines.h",
- ]
-}
-
rtc_source_set("field_trial_api") {
sources = [
"include/field_trial.h",
@@ -225,10 +212,7 @@
rtc_test("system_wrappers_unittests") {
testonly = true
sources = [
- "source/aligned_array_unittest.cc",
- "source/aligned_malloc_unittest.cc",
"source/clock_unittest.cc",
- "source/event_timer_posix_unittest.cc",
"source/metrics_default_unittest.cc",
"source/metrics_unittest.cc",
"source/ntp_time_unittest.cc",
@@ -236,6 +220,12 @@
]
configs += [ "//build/config/compiler:no_size_t_to_int_warning" ]
+ if (is_posix || is_fuchsia) {
+ sources += [
+ "source/event_timer_posix_unittest.cc",
+ ]
+ }
+
if (!build_with_chromium && is_clang) {
# Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
diff --git a/system_wrappers/OWNERS b/system_wrappers/OWNERS
index 65b8dee..13b08a3 100644
--- a/system_wrappers/OWNERS
+++ b/system_wrappers/OWNERS
@@ -1,9 +1,8 @@
-perkj@webrtc.org
henrika@webrtc.org
-henrikg@webrtc.org
mflodman@webrtc.org
niklas.enbom@webrtc.org
nisse@webrtc.org
+perkj@webrtc.org
# These are for the common case of adding or renaming files. If you're doing
# structural changes, please get a review from a reviewer in this file.
diff --git a/system_wrappers/include/clock.h b/system_wrappers/include/clock.h
index aec5ca5..0164288 100644
--- a/system_wrappers/include/clock.h
+++ b/system_wrappers/include/clock.h
@@ -13,8 +13,8 @@
#include <memory>
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
#include "system_wrappers/include/ntp_time.h"
-#include "system_wrappers/include/rw_lock_wrapper.h"
#include "typedefs.h" // NOLINT(build/include)
namespace webrtc {
diff --git a/system_wrappers/source/clock.cc b/system_wrappers/source/clock.cc
index 631974d..00cdff0 100644
--- a/system_wrappers/source/clock.cc
+++ b/system_wrappers/source/clock.cc
@@ -15,7 +15,7 @@
// Windows needs to be included before mmsystem.h
#include "rtc_base/win32.h"
-#include <MMSystem.h>
+#include <mmsystem.h>
#elif defined(WEBRTC_POSIX)
@@ -25,8 +25,8 @@
#endif // defined(WEBRTC_POSIX)
#include "rtc_base/criticalsection.h"
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
#include "rtc_base/timeutils.h"
-#include "system_wrappers/include/rw_lock_wrapper.h"
namespace webrtc {
diff --git a/system_wrappers/source/cpu_info.cc b/system_wrappers/source/cpu_info.cc
index 0bfe015..c17d59d 100644
--- a/system_wrappers/source/cpu_info.cc
+++ b/system_wrappers/source/cpu_info.cc
@@ -12,10 +12,6 @@
#if defined(WEBRTC_WIN)
#include <windows.h>
-#include <winsock2.h>
-#ifndef EXCLUDE_D3D9
-#include <d3d9.h>
-#endif
#elif defined(WEBRTC_LINUX)
#include <unistd.h>
#endif
diff --git a/system_wrappers/source/event_timer_win.cc b/system_wrappers/source/event_timer_win.cc
index b6a93fe..c2ad9f3 100644
--- a/system_wrappers/source/event_timer_win.cc
+++ b/system_wrappers/source/event_timer_win.cc
@@ -10,7 +10,7 @@
#include "system_wrappers/source/event_timer_win.h"
-#include "Mmsystem.h"
+#include "mmsystem.h"
namespace webrtc {
diff --git a/system_wrappers/source/module.mk b/system_wrappers/source/module.mk
index 92c7dda..3b84a7d 100644
--- a/system_wrappers/source/module.mk
+++ b/system_wrappers/source/module.mk
@@ -5,18 +5,13 @@
include common.mk
system_wrappers_source_CXX_OBJECTS = \
- system_wrappers/source/aligned_malloc.o \
system_wrappers/source/clock.o \
system_wrappers/source/cpu_features.o \
system_wrappers/source/cpu_info.o \
system_wrappers/source/event.o \
system_wrappers/source/event_timer_posix.o \
- system_wrappers/source/file_impl.o \
system_wrappers/source/rtp_to_ntp_estimator.o \
- system_wrappers/source/rw_lock.o \
- system_wrappers/source/rw_lock_posix.o \
- system_wrappers/source/sleep.o \
- system_wrappers/source/timestamp_extrapolator.o
+ system_wrappers/source/sleep.o
CXX_STATIC_LIBRARY(system_wrappers/source/libsystem_wrappers.pic.a): \
$(system_wrappers_source_CXX_OBJECTS) \
