webrtc: Replace dsp and echoprobe by gst webrtcdsp and webrtcechoprobe

.github/workflows/build.yml

@@ -9,8 +9,8 @@ jobs:
           fetch-depth: 0
       - run: sudo apt-get update
       - run: sudo apt-get remove libunwind-14-dev
-      - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-4-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev libwebrtc-audio-processing-dev libadwaita-1-dev libsignal-protocol-c-dev libcanberra-dev
-      - run: ./configure --release --no-debug --with-tests --enable-plugin=notification-sound --prefix=/usr
+      - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-4-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev libadwaita-1-dev libsignal-protocol-c-dev libcanberra-dev
+      - run: ./configure --release --no-debug --with-tests --enable-plugin=notification-sound --prefix=/usr --without-webrtc
       - run: cmake --build build
       - run: cmake --build build --target=test
       - name: Build DEB-package
@@ -38,8 +38,8 @@ jobs:
           fetch-depth: 0
       - run: sudo apt-get update
       - run: sudo apt-get remove libunwind-14-dev
-      - run: sudo apt-get install -y build-essential gettext libadwaita-1-dev libcanberra-dev libgcrypt20-dev libgee-0.8-dev libgpgme-dev libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libgtk-4-dev libnice-dev libnotify-dev libqrencode-dev libsignal-protocol-c-dev libsoup2.4-dev libsqlite3-dev libsrtp2-dev libwebrtc-audio-processing-dev meson valac
-      - run: meson setup build -Dcrypto-backend=auto -Dplugin-ice=enabled -Duse-soup2=true
+      - run: sudo apt-get install -y build-essential gettext libadwaita-1-dev libcanberra-dev libgcrypt20-dev libgee-0.8-dev libgpgme-dev libgstreamer-plugins-base1.0-dev libgstreamer1.0-dev libgtk-4-dev libnice-dev libnotify-dev libqrencode-dev libsignal-protocol-c-dev libsoup2.4-dev libsqlite3-dev libsrtp2-dev meson valac
+      - run: meson setup build -Duse-soup2=true -Dplugin-rtp-webrtc-audio-processing=disabled
       - run: meson compile -C build
       - run: meson test -C build
   build-flatpak:

CMakeLists.txt

@@ -12,6 +12,14 @@ endif ()
 
 include(CTest)
 
+option(PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING "Use WebRTC audio processing" ON)
+option(WITH_WASAPI "Use wasapi instead of directsound on windows" ON)
+
+# https://gitlab.kitware.com/cmake/cmake/-/issues/19804
+if (WIN32)
+    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .dll)
+endif()
+
 # Prepare Plugins
 set(DEFAULT_PLUGINS omemo;openpgp;http-files;ice;rtp)
 if (WIN32)

README.md

@@ -73,7 +73,7 @@ Make sure to install all [dependencies](https://github.com/dino/dino/wiki/Build#
 
 If you want to use `meson` build system, follow the next instructions:
 
-    meson setup build -Dcrypto-backend=gnutls -Dplugin-ice=enabled
+    meson setup build
     meson configure --prefix $PWD/build/install --libdir lib build
     meson compile -C build
     meson install -C build
@@ -84,6 +84,12 @@ If your `nice` library depends on `libsoup-2.4` (consider `ldd` output for the `
 Skip `meson configure` step, if you want to install the program globally.
 You can specify any convenient directory in the option `--prefix` where the program will be installed.
 
+If there is no `webrtcdsp` plugin in your system (check this by calling `gst-inspect-1.0 webrtcdsp`) you should pass extra argument:
+
+* `--without-webrtcdsp` for `./configure`;
+* `-Dplugin-rtp-webrtc-audio-processing=disabled` for `meson`;
+* `-DPLUGIN_RTP_WEBRTC_AUDIO_PROCESSING=OFF` for `cmake`.
+
 In addition, there is a git version of this package for **Arch Linux** on [AUR](https://aur.archlinux.org/packages/dino-plus-git)
 
 Build on Windows (x86_64)

build-win64.sh

@@ -74,7 +74,6 @@ prepare()
        mingw64/mingw-w64-x86_64-nsis \
        mingw64/mingw-w64-x86_64-libsignal-protocol-c \
        mingw64/mingw-w64-x86_64-icu \
-       mingw64/mingw-w64-x86_64-webrtc-audio-processing \
        mingw64/mingw-w64-x86_64-meson \
        git \
        make \

cmake/FindGstWebrtcDsp.cmake

@@ -0,0 +1,11 @@
+find_library(GstWebrtcDsp_LIBRARY gstwebrtcdsp PATH_SUFFIXES gstreamer-1.0)
+
+if(GstWebrtcDsp_LIBRARY_FOUND)
+    find_package(Gst)
+    set(GstWebrtcDsp_VERSION ${Gst_VERSION})
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GstWebrtcDsp
+    REQUIRED_VARS GstWebrtcDsp_LIBRARY
+    VERSION_VAR GstWebrtcDsp_VERSION)

configure

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 OPTS=`getopt -o "h" --long \
-help,fetch-only,no-debug,disable-fast-vapi,with-tests,release,with-libsoup3,\
+help,fetch-only,no-debug,disable-fast-vapi,with-tests,release,with-libsoup3,without-webrtcdsp,\
 enable-plugin:,disable-plugin:,\
 prefix:,program-prefix:,exec-prefix:,lib-suffix:,\
 bindir:,libdir:,includedir:,datadir:,\
@@ -21,6 +21,7 @@ DISABLE_FAST_VAPI=
 LIB_SUFFIX=
 NO_DEBUG=
 USE_SOUP3=
+PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING=yes
 
 EXEC_PREFIX=
 BINDIR=
@@ -54,6 +55,7 @@ Configuration:
   --no-debug                 Build without debug symbols
   --release                  Configure to build an optimized release version
   --with-libsoup3            Build with libsoup-3.0
+  --without-webrtcdsp        Build without WebRTC audio processing
   --with-tests               Also build tests.
 
 Plugin configuration:
@@ -109,6 +111,7 @@ while true; do
         --valac-flags ) VALACFLAGS="$2"; shift; shift ;;
         --lib-suffix ) LIB_SUFFIX="$2"; shift; shift ;;
         --with-libsoup3 ) USE_SOUP3=yes; shift ;;
+        --without-webrtcdsp ) PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING=no; shift ;;
         --disable-fast-vapi ) DISABLE_FAST_VAPI=yes; shift ;;
         --no-debug ) NO_DEBUG=yes; shift ;;
         --release ) BUILD_TYPE=RelWithDebInfo; shift ;;
@@ -219,6 +222,7 @@ cmake -G "$cmake_type" \
     -DBIN_INSTALL_DIR="$BINDIR" \
     -DINCLUDE_INSTALL_DIR="$INCLUDEDIR" \
     -DLIB_INSTALL_DIR="$LIBDIR" \
+    -DPLUGIN_RTP_WEBRTC_AUDIO_PROCESSING="$PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING" \
     -Wno-dev \
     .. || exit 9
 

meson.build

@@ -58,6 +58,8 @@ else
     libsoup_version = '3.0'
 endif
 
+cc = meson.get_compiler('c')
+
 dep_gdk_pixbuf = dependency('gdk-pixbuf-2.0')
 dep_gee = dependency('gee-0.8')
 dep_gio = dependency('gio-2.0')
@@ -81,9 +83,12 @@ dep_libsrtp2 = dependency('libsrtp2', disabler: true, required: plugin_crypto)
 dep_libsignal_protocol_c = dependency('libsignal-protocol-c', version: ['>=2.3.2', '<2.3.4'], disabler: true, required: get_option('plugin-omemo'))
 dep_libsoup = dependency('libsoup-@0@'.format(libsoup_version), disabler: true, required: get_option('plugin-http-files'))
 dep_nice = dependency('nice', version: '>=0.1.15', disabler: true, required: get_option('plugin-ice'))
-dep_m = meson.get_compiler('c').find_library('m', required: false)
+dep_m = cc.find_library('m', required: false)
 dep_sqlite3 = dependency('sqlite3', version: '>=3.24')
-dep_webrtc_audio_processing = dependency('webrtc-audio-processing', version: ['>=0.2', '<0.4'], required: get_option('plugin-rtp-webrtc-audio-processing'))
+
+dep_gstreamer_bad = dependency('gstreamer-plugins-bad-1.0', disabler: true, required: get_option('plugin-rtp-webrtc-audio-processing'))
+gstpluginsdir = dep_gstreamer_bad.get_variable('pluginsdir')
+dep_webrtcdsp = cc.find_library('gstwebrtcdsp', dirs: gstpluginsdir, disabler: true, required: get_option('plugin-rtp-webrtc-audio-processing'))
 
 prog_git = find_program('git', required: false)
 prog_python = python.find_installation()

meson_options.txt

@@ -16,3 +16,4 @@ option('plugin-rtp-vp9', type: 'feature', value: 'disabled', description: 'VP9 c
 option('plugin-rtp-webrtc-audio-processing', type: 'feature', description: 'Voice preprocessing')
 
 option('use-soup2', type: 'boolean', value: false, description: 'Use libsoup version 2 instead of 3')
+option('with-wasapi', type: 'boolean', value: true, description: 'Use wasapi insted of directsound on windows')

plugins/rtp/CMakeLists.txt

@@ -1,5 +1,13 @@
 find_package(GLib ${GLib_GLOBAL_VERSION} REQUIRED)
-find_package(WebRTCAudioProcessing 0.2)
+
+set(RTP_DEFINITIONS)
+set(RTP_EXTRA_OPTIONS)
+
+if(PLUGIN_RTP_WEBRTC_AUDIO_PROCESSING)
+    set(EXTRA_RTP_PACKAGES GstWebrtcDsp)
+    list(APPEND RTP_DEFINITIONS WITH_VOICE_PROCESSOR)
+endif()
+
 find_packages(RTP_PACKAGES REQUIRED
     Gee
     GLib
@@ -11,11 +19,9 @@ find_packages(RTP_PACKAGES REQUIRED
     GstAudio
     GstRtp
     GstVideo
+    ${EXTRA_RTP_PACKAGES}
 )
 
-set(RTP_DEFINITIONS)
-set(RTP_EXTRA_OPTIONS)
-
 if(GstRtp_VERSION VERSION_GREATER_EQUAL "1.16")
     set(RTP_DEFINITIONS ${RTP_DEFINITIONS} GST_1_16)
 endif()
@@ -52,18 +58,8 @@ if(RTP_ENABLE_MSDK)
     set(RTP_DEFINITIONS ${RTP_DEFINITIONS} ENABLE_MSDK)
 endif()
 
-if(WebRTCAudioProcessing_VERSION GREATER "0.4")
-    message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far")
-    unset(WebRTCAudioProcessing_FOUND)
-endif()
-
-if(WebRTCAudioProcessing_FOUND)
-    set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR)
-    set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala)
-    set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp)
-    set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing)
-else()
-    message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!")
+if(WIN32 AND WITH_WASAPI)
+    list(APPEND RTP_DEFINITIONS WITH_WASAPI)
 endif()
 
 vala_precompile(RTP_VALA_C

plugins/rtp/meson.build

@@ -30,12 +30,8 @@ c_args = [
 vala_args = [
     '--vapidir', meson.current_source_dir() / 'vapi',
 ]
-if dep_webrtc_audio_processing.found()
-    dependencies += [dep_webrtc_audio_processing]
-    sources += files(
-        'src/voice_processor.vala',
-        'src/voice_processor_native.cpp',
-    )
+if dep_webrtcdsp.found()
+    dependencies += [dep_webrtcdsp]
     vala_args += ['-D', 'WITH_VOICE_PROCESSOR']
 endif
 if dep_gstreamer_rtp.version() == 'unknown' or dep_gstreamer_rtp.version().version_compare('>=1.16')
@@ -59,5 +55,8 @@ endif
 if get_option('plugin-rtp-vp9').allowed()
     vala_args += ['-D', 'ENABLE_VP9']
 endif
+if host_machine.system() == 'windows' and get_option('with-wasapi')
+    vala_args += ['-D', 'WITH_WASAPI']
+endif
 lib_rtp = shared_library('rtp', sources, name_prefix: '', c_args: c_args, vala_args: vala_args, include_directories: include_directories('src'), dependencies: dependencies, kwargs: install_options)
 dep_rtp = declare_dependency(link_with: lib_rtp, include_directories: include_directories('.'))

plugins/rtp/src/device.vala

@@ -475,13 +475,12 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
         Gst.Bin bin = new Gst.Bin("voiceprocessorbin");
         Gst.Element converter = Gst.ElementFactory.make("audioconvert", @"dsp_convert_$id");
         Gst.Element resampler = Gst.ElementFactory.make("audioresample", @"dsp_resmaple_$id");
-
-        Gst.Element voiceproc = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume);
-        voiceproc.name = @"dsp_$id";
+        Gst.Element voiceproc = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id");
 
         bin.add_many(converter, resampler, voiceproc);
         converter.link(resampler);
         resampler.link(voiceproc);
+        voiceproc.@set("probe", "webrtcechoprobe0");
 
         Gst.Pad sink_pad = bin.find_unlinked_pad(Gst.PadDirection.SINK);
         Gst.Pad src_pad = bin.find_unlinked_pad(Gst.PadDirection.SRC);

plugins/rtp/src/plugin.vala

@@ -42,30 +42,41 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
         if (pause_count < 0) warning("Pause count below zero!");
     }
 
+    private bool is_skipped(Gst.Device device) {
+        bool is_noprops = device.properties == null;
+#if WITH_WASAPI
+        bool is_disabled_sound = device.properties.get_string("device.api") == "directsound";
+#else
+        bool is_disabled_sound = device.properties.get_string("device.api") == "wasapi";
+#endif
+        bool is_proplist = device.properties.has_name("pipewire-proplist") && device.has_classes("Audio");
+        bool is_monitor = device.properties.get_string("device.class") == "monitor";
+        bool is_in_devices = devices.any_match((it) => it.matches(device));
+        return is_noprops || is_disabled_sound || is_proplist || is_monitor || is_in_devices;
+    }
+
     private void init_device_monitor() {
         if (device_monitor != null) return;
         device_monitor = new Gst.DeviceMonitor();
         device_monitor.show_all = true;
         device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message);
         device_monitor.start();
         foreach (Gst.Device device in device_monitor.get_devices()) {
-            if (device.properties == null) continue;
-            if (device.properties.get_string("device.api") == "wasapi") continue;
-            if (device.properties.has_name("pipewire-proplist") && device.has_classes("Audio")) continue;
-            if (device.properties.get_string("device.class") == "monitor") continue;
-            if (devices.any_match((it) => it.matches(device))) continue;
+            if (is_skipped(device)) continue;
+            debug(@"(Init) Add name=$(device.name) device=$(device.display_name)");
             devices.add(new Device(this, device));
         }
     }
 
 #if WITH_VOICE_PROCESSOR
-    private Gst.Element create_echo_bin(Gst.Element element) {
+    private Gst.Element create_echo_probe() {
         Gst.Bin bin = new Gst.Bin("echoprobebin");
         Gst.Element converter = Gst.ElementFactory.make("audioconvert", "echo_convert_");
         Gst.Element resampler = Gst.ElementFactory.make("audioresample", "echo_resample_");
+        Gst.Element webrtcechoprobe = Gst.ElementFactory.make("webrtcechoprobe", "webrtcechoprobe0");
 
-        bin.add_many(element, converter, resampler);
-        element.link(converter);
+        bin.add_many(webrtcechoprobe, converter, resampler);
+        webrtcechoprobe.link(converter);
         converter.link(resampler);
 
         Gst.Pad sink_pad = bin.find_unlinked_pad(Gst.PadDirection.SINK);
@@ -100,11 +111,8 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
 
 #if WITH_VOICE_PROCESSOR
         // Audio echo probe
-        echoprobe = new EchoProbe();
-        if (echoprobe != null) {
-            echoprobe = create_echo_bin(echoprobe);
-            pipe.add(echoprobe);
-        }
+        echoprobe = create_echo_probe();
+        pipe.add(echoprobe);
 #endif
 
         // Pipeline
@@ -229,20 +237,23 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
         switch (message.type) {
             case Gst.MessageType.DEVICE_ADDED:
                 message.parse_device_added(out gst_device);
-                if (devices.any_match((it) => it.matches(gst_device))) return Source.CONTINUE;
+                if (is_skipped(gst_device)) return Source.CONTINUE;
                 device = new Device(this, gst_device);
+                debug(@"(Notify) Add name=$(gst_device.name) device=$(gst_device.display_name)");
                 devices.add(device);
                 break;
 #if GST_1_16
             case Gst.MessageType.DEVICE_CHANGED:
                 message.parse_device_changed(out gst_device, out old_gst_device);
                 device = devices.first_match((it) => it.matches(old_gst_device));
+                debug(@"(Notify) Change name=$(gst_device.name) device=$(gst_device.display_name)");
                 if (device != null) device.update(gst_device);
                 break;
 #endif
             case Gst.MessageType.DEVICE_REMOVED:
                 message.parse_device_removed(out gst_device);
                 device = devices.first_match((it) => it.matches(gst_device));
+                debug(@"(Notify) Remove name=$(gst_device.name) device=$(gst_device.display_name)");
                 if (device != null) devices.remove(device);
                 break;
             default: