anotherim/anotherim-desktop
2
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Support voice processing on GStreamer 0.14

Browse source
This commit is contained in:
Marvin W 2021-05-02 00:34:17 +02:00
parent 0ad968df36
commit 8044b546d0
No known key found for this signature in database
GPG key ID: 072E9235DB996F2A
3 changed files with 27 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
plugins/rtp/CMakeLists.txt
View file

@ -17,7 +17,7 @@ if(Gst_VERSION VERSION_GREATER "1.16")
endif() endif()
if(WebRTCAudioProcessing_VERSION GREATER "0.4") if(WebRTCAudioProcessing_VERSION GREATER "0.4")
message(WARNING "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far")
unset(WebRTCAudioProcessing_FOUND) unset(WebRTCAudioProcessing_FOUND)
endif() endif()
@ -25,8 +25,9 @@ if(WebRTCAudioProcessing_FOUND)
set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR)
set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala)
set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp)
set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing)
else() else()
message(WARNING "WebRTCAudioProcessing not found, build without voice pre-processing!") message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!")
endif() endif()
vala_precompile(RTP_VALA_C vala_precompile(RTP_VALA_C
@ -53,7 +54,7 @@ DEFINITIONS
add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src)
add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX})
target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 webrtc-audio-processing) target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 ${RTP_VOICE_PROCESSOR_LIB})
set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES PREFIX "")
set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/)

2
plugins/rtp/src/voice_processor.vala
View file

@ -123,7 +123,7 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
} }
analyze_reverse_stream(native, echo_probe.audio_info, buffer); analyze_reverse_stream(native, echo_probe.audio_info, buffer);
if (adjust_delay_timeout_id == 0 && echo_probe != null) { if (adjust_delay_timeout_id == 0 && echo_probe != null) {
adjust_delay_timeout_id = Timeout.add(5000, adjust_delay); adjust_delay_timeout_id = Timeout.add(1000, adjust_delay);
} }
} }

37
plugins/rtp/src/voice_processor_native.cpp
View file

@ -11,6 +11,8 @@
struct _DinoPluginsRtpVoiceProcessorNative { struct _DinoPluginsRtpVoiceProcessorNative {
webrtc::AudioProcessing *apm; webrtc::AudioProcessing *apm;
gint stream_delay; gint stream_delay;
gint last_median;
gint last_poor_delays;
}; };
extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) {
@ -26,6 +28,8 @@ extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay)
config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(true, 85)); config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(true, 85));
native->apm = webrtc::AudioProcessing::Create(config); native->apm = webrtc::AudioProcessing::Create(config);
native->stream_delay = stream_delay; native->stream_delay = stream_delay;
native->last_median = 0;
native->last_poor_delays = 0;
return native; return native;
} }
@ -65,19 +69,19 @@ dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAud
webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
webrtc::AudioProcessing *apm = native->apm; webrtc::AudioProcessing *apm = native->apm;
GstAudioBuffer audio_buffer; GstMapInfo map;
gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READ); gst_buffer_map(buffer, &map, GST_MAP_READ);
webrtc::AudioFrame frame; webrtc::AudioFrame frame;
frame.num_channels_ = info->channels; frame.num_channels_ = info->channels;
frame.sample_rate_hz_ = info->rate; frame.sample_rate_hz_ = info->rate;
frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf;
memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf);
int err = apm->AnalyzeReverseStream(&frame); int err = apm->AnalyzeReverseStream(&frame);
if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err);
gst_audio_buffer_unmap(&audio_buffer); gst_buffer_unmap(buffer, &map);
} }
extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) {
@ -101,14 +105,17 @@ extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *nati
extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) {
_DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr;
webrtc::AudioProcessing *apm = native->apm; webrtc::AudioProcessing *apm = native->apm;
int median, std; int median, std, poor_delays;
float fraction_poor_delays; float fraction_poor_delays;
apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays);
if (fraction_poor_delays < 0) return; poor_delays = (int)(fraction_poor_delays * 100.0);
g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return;
if (fraction_poor_delays > 0.5) { g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays);
native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); native->last_median = median;
g_debug("voice_processor_native.cpp: Adjusted stream delay %i", native->stream_delay); native->last_poor_delays = poor_delays;
if (poor_delays > 90) {
native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384);
g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay);
} }
} }
@ -118,21 +125,21 @@ dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *
webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false);
webrtc::AudioProcessing *apm = native->apm; webrtc::AudioProcessing *apm = native->apm;
GstAudioBuffer audio_buffer; GstMapInfo map;
gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READWRITE); gst_buffer_map(buffer, &map, GST_MAP_READWRITE);
webrtc::AudioFrame frame; webrtc::AudioFrame frame;
frame.num_channels_ = info->channels; frame.num_channels_ = info->channels;
frame.sample_rate_hz_ = info->rate; frame.sample_rate_hz_ = info->rate;
frame.samples_per_channel_ = info->rate / 100; frame.samples_per_channel_ = info->rate / 100;
memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf);
apm->set_stream_delay_ms(native->stream_delay); apm->set_stream_delay_ms(native->stream_delay);
int err = apm->ProcessStream(&frame); int err = apm->ProcessStream(&frame);
if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf);
if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err);
gst_audio_buffer_unmap(&audio_buffer); gst_buffer_unmap(buffer, &map);
} }
extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) {