FS-11014 [core] add vad to core

Makefile.am

@@ -179,6 +179,10 @@ if HAVE_GUMBO
 CORE_CFLAGS += -DSWITCH_HAVE_GUMBO $(LIBGUMBO_CFLAGS)
 endif
 
+if HAVE_FVAD
+CORE_CFLAGS += -DSWITCH_HAVE_FVAD $(LIBFVAD_CFLAGS)
+endif
+
 ##
 ## libfreeswitch
 ##
@@ -241,9 +245,9 @@ CORE_LIBS+=libfreeswitch_libyuv.la
 endif
 
 lib_LTLIBRARIES	         = libfreeswitch.la
-libfreeswitch_la_CFLAGS  = $(CORE_CFLAGS) $(SQLITE_CFLAGS) $(GUMBO_CFLAGS) $(FREETYPE_CFLAGS) $(CURL_CFLAGS) $(PCRE_CFLAGS) $(SPEEX_CFLAGS) $(LIBEDIT_CFLAGS) $(openssl_CFLAGS)  $(AM_CFLAGS)
+libfreeswitch_la_CFLAGS  = $(CORE_CFLAGS) $(SQLITE_CFLAGS) $(GUMBO_CFLAGS) $(FVAD_CFLAGS) $(FREETYPE_CFLAGS) $(CURL_CFLAGS) $(PCRE_CFLAGS) $(SPEEX_CFLAGS) $(LIBEDIT_CFLAGS) $(openssl_CFLAGS)  $(AM_CFLAGS)
 libfreeswitch_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) $(PLATFORM_CORE_LDFLAGS) -no-undefined
-libfreeswitch_la_LIBADD  = $(CORE_LIBS) $(APR_LIBS) $(SQLITE_LIBS) $(GUMBO_LIBS) $(FREETYPE_LIBS) $(CURL_LIBS) $(PCRE_LIBS) $(SPEEX_LIBS) $(LIBEDIT_LIBS) $(openssl_LIBS) $(PLATFORM_CORE_LIBS)
+libfreeswitch_la_LIBADD  = $(CORE_LIBS) $(APR_LIBS) $(SQLITE_LIBS) $(GUMBO_LIBS) $(FVAD_LIBS) $(FREETYPE_LIBS) $(CURL_LIBS) $(PCRE_LIBS) $(SPEEX_LIBS) $(LIBEDIT_LIBS) $(openssl_LIBS) $(PLATFORM_CORE_LIBS)
 libfreeswitch_la_DEPENDENCIES = $(BUILT_SOURCES)
 
 if HAVE_PNG
@@ -312,6 +316,7 @@ library_include_HEADERS = \
 	src/include/switch_utf8.h \
 	src/include/switch_msrp.h \
 	src/include/switch_vpx.h \
+	src/include/switch_vad.h \
 	libs/libteletone/src/libteletone_detect.h \
 	libs/libteletone/src/libteletone_generate.h \
 	libs/libteletone/src/libteletone.h \
@@ -395,6 +400,7 @@ libfreeswitch_la_SOURCES = \
 	src/switch_hashtable.c\
 	src/switch_utf8.c \
 	src/switch_msrp.c \
+	src/switch_vad.c \
 	src/switch_vpx.c \
 	libs/libtpl-1.5/src/tpl.c \
 	libs/libteletone/src/libteletone_detect.c \

configure.ac

@@ -1310,6 +1310,10 @@ PKG_CHECK_MODULES([GUMBO], [gumbo >= 0.10.1],[
   AM_CONDITIONAL([HAVE_GUMBO],[true])],[
   AC_MSG_RESULT([no]); AM_CONDITIONAL([HAVE_GUMBO],[false])])
 
+PKG_CHECK_MODULES([FVAD], [libfvad >= 1.0],[
+  AM_CONDITIONAL([HAVE_FVAD],[true])],[
+  AC_MSG_RESULT([no]); AM_CONDITIONAL([HAVE_FVAD],[false])])
+
 PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.6.20])
 PKG_CHECK_MODULES([CURL], [libcurl >= 7.19])
 PKG_CHECK_MODULES([PCRE], [libpcre >= 7.8])

src/include/switch.h

@@ -145,6 +145,7 @@
 #include "switch_core_video.h"
 #include "switch_jitterbuffer.h"
 #include "switch_estimators.h"
+#include "switch_vad.h"
 #include <libteletone.h>
 
 

src/include/switch_types.h

@@ -622,6 +622,14 @@ typedef enum {
 } switch_vad_flag_enum_t;
 typedef uint32_t switch_vad_flag_t;
 
+typedef enum {
+	SWITCH_VAD_STATE_NONE,
+	SWITCH_VAD_STATE_START_TALKING,
+	SWITCH_VAD_STATE_TALKING,
+	SWITCH_VAD_STATE_STOP_TALKING,
+	SWITCH_VAD_STATE_ERROR
+} switch_vad_state_t;
+typedef struct switch_vad_s switch_vad_t;
 
 typedef struct error_period {
 	int64_t start;

src/include/switch_vad.h

@@ -0,0 +1,66 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2018, Anthony Minessale II <anthm@freeswitch.org>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthm@freeswitch.org>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Seven Du <dujinfang@gmail.com>
+ *
+ *
+ * switch_vad.h VAD code with optional libfvad
+ *
+ */
+/*!
+  \defgroup vad1 VAD code with optional libfvad
+  \ingroup core1
+  \{
+*/
+#ifndef FREESWITCH_VAD_H
+#define FREESWITCH_VAD_H
+
+SWITCH_BEGIN_EXTERN_C
+
+SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels);
+
+/*
+ * Valid modes are -1 ("disable fvad, using native"), 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive").
+ * The default mode is -1.
+*/
+
+SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode);
+SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val);
+SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples);
+SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad);
+SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad);
+
+SWITCH_END_EXTERN_C
+#endif
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

src/mod/applications/mod_dptools/mod_dptools.c

@@ -6128,6 +6128,73 @@ SWITCH_STANDARD_APP(deduplicate_dtmf_app_function)
 	}
 }
 
+SWITCH_STANDARD_APP(vad_test_function)
+{
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+	switch_codec_implementation_t imp = { 0 };
+	switch_vad_t *vad;
+	switch_frame_t *frame = { 0 };
+	switch_vad_state_t vad_state;
+	int mode = -1;
+	const char *var = NULL;
+	int tmp;
+
+	if (!zstr(data)) {
+		mode = atoi(data);
+
+		if (mode > 3) mode = 3;
+	}
+
+	switch_core_session_raw_read(session);
+	switch_core_session_get_read_impl(session, &imp);
+
+	vad = switch_vad_init(imp.samples_per_second, imp.number_of_channels);
+	switch_assert(vad);
+	switch_vad_set_mode(vad, mode);
+
+	if ((var = switch_channel_get_variable(channel, "vad_hangover_len"))) {
+		tmp = atoi(var);
+
+		if (tmp > 0) switch_vad_set_param(vad, "hangover_len", tmp);
+	}
+
+	if ((var = switch_channel_get_variable(channel, "vad_thresh"))) {
+		tmp = atoi(var);
+
+		if (tmp > 0) switch_vad_set_param(vad, "thresh", tmp);
+	}
+
+	if ((var = switch_channel_get_variable(channel, "vad_timeout_len"))) {
+		tmp = atoi(var);
+
+		if (tmp > 0) switch_vad_set_param(vad, "timeout_len", tmp);
+	}
+
+	while(switch_channel_ready(channel)) {
+		switch_core_session_read_frame(session, &frame, SWITCH_IO_FLAG_NONE, 0);
+
+		if (switch_test_flag(frame, SFF_CNG)) {
+			continue;
+		}
+
+		vad_state = switch_vad_process(vad, frame->data, frame->datalen / 2);
+
+		if (vad_state == SWITCH_VAD_STATE_START_TALKING) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "START TALKING\n");
+			switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
+		} else if (vad_state == SWITCH_VAD_STATE_STOP_TALKING) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "STOP TALKING\n");
+		} else if (vad_state == SWITCH_VAD_STATE_TALKING) {
+			switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
+		} else {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "vad_state: %d\n", vad_state);
+		}
+	}
+
+	switch_vad_destroy(&vad);
+	switch_core_session_reset(session, SWITCH_TRUE, SWITCH_TRUE);
+}
+
 #define SPEAK_DESC "Speak text to a channel via the tts interface"
 #define DISPLACE_DESC "Displace audio from a file to the channels input"
 #define SESS_REC_DESC "Starts a background recording of the entire session"
@@ -6462,6 +6529,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_dptools_load)
 	SWITCH_ADD_APP(app_interface, "pickup", "Pickup", "Pickup a call", pickup_function, PICKUP_SYNTAX, SAF_SUPPORT_NOMEDIA);
 	SWITCH_ADD_APP(app_interface, "deduplicate_dtmf", "Prevent duplicate inband + 2833 dtmf", "", deduplicate_dtmf_app_function, "[only_rtp]", SAF_SUPPORT_NOMEDIA);
 
+	SWITCH_ADD_APP(app_interface, "vad_test", "VAD test", "VAD test, mode = -1(default), 0, 1, 2, 3", vad_test_function, "[mode]", SAF_NONE);
 
 	SWITCH_ADD_DIALPLAN(dp_interface, "inline", inline_dialplan_hunt);
 

src/switch_vad.c

@@ -0,0 +1,219 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2018, Anthony Minessale II <anthm@freeswitch.org>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthm@freeswitch.org>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Seven Du <dujinfang@gmail.com>
+ *
+ *
+ * switch_vad.c VAD code with optional libfvad
+ *
+ */
+
+#include <switch.h>
+
+#ifdef SWITCH_HAVE_FVAD
+#include <fvad.h>
+#endif
+
+struct switch_vad_s {
+	int talking;
+	int talked;
+	int talk_hits;
+	int hangover;
+	int hangover_len;
+	int divisor;
+	int thresh;
+	int timeout_len;
+	int timeout;
+	int channels;
+	int sample_rate;
+
+	int _hangover_len;
+	int _thresh;
+	int _timeout_len;
+#ifdef SWITCH_HAVE_FVAD
+	Fvad *fvad;
+#endif
+};
+
+SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels)
+{
+	switch_vad_t *vad = malloc(sizeof(switch_vad_t));
+
+	if (!vad) return NULL;
+
+	memset(vad, 0, sizeof(*vad));
+	vad->sample_rate = sample_rate ? sample_rate : 8000;
+	vad->channels = channels;
+	vad->_hangover_len = 25;
+	vad->_thresh = 100;
+	vad->_timeout_len = 25;
+
+	switch_vad_reset(vad);
+
+	return vad;
+}
+
+//Valid modes are 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive"). The default mode is 0.
+SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode)
+{
+#ifdef SWITCH_HAVE_FVAD
+	int ret;
+
+	if (mode < 0) return 0;
+
+	vad->fvad = fvad_new();
+	ret = fvad_set_mode(vad->fvad, mode);
+	fvad_set_sample_rate(vad->fvad, vad->sample_rate);
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "libfvad started, mode = %d\n", mode);
+	return ret;
+#else
+	return 0;
+#endif
+}
+
+SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val)
+{
+	if (!key) return;
+
+	if (!strcmp(key, "hangover_len")) {
+		vad->hangover_len = vad->_hangover_len = val;
+	} else if (!strcmp(key, "thresh")) {
+		vad->thresh = vad->_thresh = val;
+	} else if (!strcmp(key, "timeout_len")) {
+		vad->timeout = vad->timeout_len = vad->_timeout_len = val;
+	}
+}
+
+SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad)
+{
+#ifdef SWITCH_HAVE_FVAD
+	if (vad->fvad) {
+		fvad_reset(vad->fvad);
+		return;
+	}
+#endif
+
+	vad->talking = 0;
+	vad->talked = 0;
+	vad->talk_hits = 0;
+	vad->hangover = 0;
+	vad->hangover_len = vad->_hangover_len;
+	vad->divisor = vad->sample_rate / 8000;
+	vad->thresh = vad->_thresh;
+	vad->timeout_len = vad->_timeout_len;
+	vad->timeout = vad->timeout_len;
+}
+
+SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples)
+{
+	int energy = 0, j = 0, count = 0;
+	int score = 0;
+	switch_vad_state_t vad_state = SWITCH_VAD_STATE_NONE;
+
+#ifdef SWITCH_HAVE_FVAD
+	if (vad->fvad) {
+		int ret = fvad_process(vad->fvad, data, samples);
+
+		// printf("%d ", ret); fflush(stdout);
+
+		score = vad->thresh + ret - 1;
+	} else {
+#endif
+
+	for (energy = 0, j = 0, count = 0; count < samples; count++) {
+		energy += abs(data[j]);
+		j += vad->channels;
+	}
+
+	score = (uint32_t) (energy / (samples / vad->divisor));
+
+#ifdef SWITCH_HAVE_FVAD
+	}
+#endif
+
+	// printf("%d ", score); fflush(stdout);
+	// printf("yay %d %d %d\n", score, vad->hangover, vad->talking);
+
+	if (vad->talking && score < vad->thresh) {
+		if (vad->hangover > 0) {
+			vad->hangover--;
+		} else {// if (hangover <= 0) {
+			vad->talking = 0;
+			vad->talk_hits = 0;
+			vad->hangover = 0;
+		}
+	} else {
+		if (score >= vad->thresh) {
+			vad_state = vad->talking ? SWITCH_VAD_STATE_TALKING : SWITCH_VAD_STATE_START_TALKING;
+			vad->talking = 1;
+			vad->hangover = vad->hangover_len;
+		}
+	}
+
+	// printf("WTF %d %d %d\n", score, vad->talked, vad->talking);
+
+	if (vad->talking) {
+		vad->talk_hits++;
+		// printf("WTF %d %d %d\n", vad->talking, vad->talk_hits, vad->talked);
+		if (vad->talk_hits > 10) {
+			vad->talked = 1;
+			vad_state = SWITCH_VAD_STATE_TALKING;
+		}
+	} else {
+		vad->talk_hits = 0;
+	}
+
+	if (vad->timeout > 0 && !vad->talking) {
+		vad->timeout--;
+	}
+
+	if ((vad->talked && !vad->talking)) {
+		// printf("NOT TALKING ANYMORE\n");
+		vad->talked = 0;
+		vad->timeout = vad->timeout_len;
+		vad_state = SWITCH_VAD_STATE_STOP_TALKING;
+	} else {
+		// if (vad->skip > 0) {
+		// 	vad->skip--;
+		// }
+	}
+
+	if (vad_state) return vad_state;
+
+	return SWITCH_VAD_STATE_NONE;
+}
+
+SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad)
+{
+	if (*vad) {
+
+#ifdef SWITCH_HAVE_FVAD
+		if ((*vad)->fvad) free ((*vad)->fvad);
+#endif
+
+		free(*vad);
+		*vad = NULL;
+	}
+}