From 6eb2276cd642c474d56cd2a2d668c212bc6bbcaa Mon Sep 17 00:00:00 2001 From: Chris Rienzo Date: Thu, 11 Oct 2018 18:45:11 +0000 Subject: [PATCH] FS-11453 [mod_rayo] add support for FS JSON ASR result. --- src/mod/event_handlers/mod_rayo/nlsml.c | 83 +++++++++++-------- src/mod/event_handlers/mod_rayo/nlsml.h | 1 + .../mod_rayo/rayo_input_component.c | 69 ++++++++++++++- .../mod_rayo/test_nlsml/Makefile | 6 +- 4 files changed, 120 insertions(+), 39 deletions(-) diff --git a/src/mod/event_handlers/mod_rayo/nlsml.c b/src/mod/event_handlers/mod_rayo/nlsml.c index 79f8157e9f..752ddc3efa 100644 --- a/src/mod/event_handlers/mod_rayo/nlsml.c +++ b/src/mod/event_handlers/mod_rayo/nlsml.c @@ -1,6 +1,6 @@ /* * mod_rayo for FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application - * Copyright (C) 2013-2014, Grasshopper + * Copyright (C) 2013-2018, Grasshopper * * Version: MPL 1.1 * @@ -30,6 +30,7 @@ #include #include "nlsml.h" +#include "iks_helpers.h" struct nlsml_parser; @@ -417,48 +418,60 @@ static int isdtmf(const char digit) } /** - * Construct an NLSML result for digit match - * @param digits the matching digits + * Construct an NLSML result for match + * @param match the matching digits or text + * @param interpretation the optional digit interpretation + * @param mode dtmf or speech + * @param confidence 0-100 + * @return the NLSML + */ +iks *nlsml_create_match(const char *match, const char *interpretation, const char *mode, int confidence) +{ + iks *result = iks_new("result"); + iks_insert_attrib(result, "xmlns", NLSML_NS); + iks_insert_attrib(result, "xmlns:xf", "http://www.w3.org/2000/xforms"); + if (!zstr(match)) { + iks *interpretation_node = iks_insert(result, "interpretation"); + iks *input_node = iks_insert(interpretation_node, "input"); + iks *instance_node = iks_insert(interpretation_node, "instance"); + iks_insert_attrib(input_node, "mode", mode); + iks_insert_attrib_printf(input_node, "confidence", "%d", confidence); + iks_insert_cdata(input_node, match, strlen(match)); + if (zstr(interpretation)) { + iks_insert_cdata(instance_node, match, strlen(match)); + } else { + iks_insert_cdata(instance_node, interpretation, strlen(interpretation)); + } + } + return result; +} + +/** + * Construct an NLSML result for match + * @param match the matching digits or text * @param interpretation the optional digit interpretation * @return the NLSML */ iks *nlsml_create_dtmf_match(const char *digits, const char *interpretation) { - iks *result = iks_new("result"); - iks_insert_attrib(result, "xmlns", NLSML_NS); - iks_insert_attrib(result, "xmlns:xf", "http://www.w3.org/2000/xforms"); - if (!zstr(digits)) { - int first = 1; - int i; - int num_digits = strlen(digits); - switch_stream_handle_t stream = { 0 }; - - iks *interpretation_node = iks_insert(result, "interpretation"); - iks *input_node = iks_insert(interpretation_node, "input"); - iks *instance_node = iks_insert(interpretation_node, "instance"); - iks_insert_attrib(input_node, "mode", "dtmf"); - iks_insert_attrib(input_node, "confidence", "100"); - - SWITCH_STANDARD_STREAM(stream); - for (i = 0; i < num_digits; i++) { - if (isdtmf(digits[i])) { - if (first) { - stream.write_function(&stream, "%c", digits[i]); - first = 0; - } else { - stream.write_function(&stream, " %c", digits[i]); - } + iks *result = NULL; + int first = 1; + int i; + int num_digits = strlen(digits); + switch_stream_handle_t stream = { 0 }; + SWITCH_STANDARD_STREAM(stream); + for (i = 0; i < num_digits; i++) { + if (isdtmf(digits[i])) { + if (first) { + stream.write_function(&stream, "%c", digits[i]); + first = 0; + } else { + stream.write_function(&stream, " %c", digits[i]); } } - iks_insert_cdata(input_node, stream.data, strlen(stream.data)); - - if (zstr(interpretation)) { - iks_insert_cdata(instance_node, stream.data, strlen(stream.data)); - } else { - iks_insert_cdata(instance_node, interpretation, strlen(interpretation)); - } - switch_safe_free(stream.data); } + result = nlsml_create_match((const char *)stream.data, interpretation, "dtmf", 100); + switch_safe_free(stream.data); return result; } diff --git a/src/mod/event_handlers/mod_rayo/nlsml.h b/src/mod/event_handlers/mod_rayo/nlsml.h index f5be079ba5..475abd091e 100644 --- a/src/mod/event_handlers/mod_rayo/nlsml.h +++ b/src/mod/event_handlers/mod_rayo/nlsml.h @@ -44,6 +44,7 @@ extern void nlsml_destroy(void); enum nlsml_match_type nlsml_parse(const char *result, const char *uuid); iks *nlsml_normalize(const char *result); extern iks *nlsml_create_dtmf_match(const char *digits, const char *interpretation); +extern iks *nlsml_create_match(const char *digits, const char *interpretation, const char *mode, int confidence); #endif diff --git a/src/mod/event_handlers/mod_rayo/rayo_input_component.c b/src/mod/event_handlers/mod_rayo/rayo_input_component.c index dc7a71c9d9..13311d5444 100644 --- a/src/mod/event_handlers/mod_rayo/rayo_input_component.c +++ b/src/mod/event_handlers/mod_rayo/rayo_input_component.c @@ -876,6 +876,30 @@ static iks *start_timers_call_input_component(struct rayo_actor *component, stru return iks_new_iq_result(iq); } +/** + * Get text / error from result + */ +static const char *get_detected_speech_result_text(cJSON *result_json, double *confidence, const char **error_text) +{ + const char *result_text = NULL; + const char *text = cJSON_GetObjectCstr(result_json, "text"); + if (confidence) { + *confidence = 0.0; + } + if (!zstr(text)) { + cJSON *json_confidence = cJSON_GetObjectItem(result_json, "confidence"); + if (json_confidence && json_confidence->valuedouble > 0.0) { + *confidence = json_confidence->valuedouble; + } else { + *confidence = 100.0; + } + result_text = text; + } else if (error_text) { + *error_text = cJSON_GetObjectCstr(result_json, "error"); + } + return result_text; +} + /** * Handle speech detection event */ @@ -905,7 +929,50 @@ static void on_detected_speech_event(switch_event_t *event) if (zstr(result)) { rayo_component_send_complete(component, INPUT_NOMATCH); } else { - if (strchr(result, '<')) { + if (result[0] == '{') { + // internal FS JSON format + cJSON *json_result = cJSON_Parse(result); + if (json_result) { + // examine result to determine what happened + double confidence = 0.0; + const char *error_text = NULL; + const char *result_text = NULL; + result_text = get_detected_speech_result_text(json_result, &confidence, &error_text); + if (!zstr(result_text)) { + // got result... send as NLSML + iks *result = nlsml_create_match(result_text, NULL, "speech", (int)confidence); + /* notify of match */ + switch_log_printf(SWITCH_CHANNEL_UUID_LOG(uuid), SWITCH_LOG_DEBUG, "MATCH = %s\n", result_text); + send_match_event(RAYO_COMPONENT(component), result); + iks_delete(result); + } else if (zstr(error_text)) { + // unknown error + switch_log_printf(SWITCH_CHANNEL_UUID_LOG(uuid), SWITCH_LOG_WARNING, "No matching text nor error in result: %s!\n", result); + rayo_component_send_complete(component, INPUT_NOMATCH); + } else if (!strcmp(error_text, "no_input")) { + // no input error + rayo_component_send_complete(component, INPUT_NOINPUT); + } else if (!strcmp(error_text, "no_match")) { + // no match error + rayo_component_send_complete(component, INPUT_NOMATCH); + } else { + // generic error + iks *response = rayo_component_create_complete_event(component, COMPONENT_COMPLETE_ERROR); + iks *error = NULL; + if ((error = iks_find(response, "complete"))) { + if ((error = iks_find(error, "error"))) { + iks_insert_cdata(error, error_text, strlen(error_text)); + } + } + rayo_component_send_complete_event(component, response); + } + cJSON_Delete(json_result); + } else { + // failed to parse JSON result + switch_log_printf(SWITCH_CHANNEL_UUID_LOG(uuid), SWITCH_LOG_WARNING, "Failed to parse JSON result: %s!\n", result); + rayo_component_send_complete(component, INPUT_NOMATCH); + } + } else if (strchr(result, '<')) { /* got an XML result */ enum nlsml_match_type match_type = nlsml_parse(result, uuid); switch (match_type) { diff --git a/src/mod/event_handlers/mod_rayo/test_nlsml/Makefile b/src/mod/event_handlers/mod_rayo/test_nlsml/Makefile index f363f55590..8104690733 100644 --- a/src/mod/event_handlers/mod_rayo/test_nlsml/Makefile +++ b/src/mod/event_handlers/mod_rayo/test_nlsml/Makefile @@ -3,7 +3,7 @@ BASE=../../../../.. IKS_DIR=$(BASE)/libs/iksemel IKS_LA=$(IKS_DIR)/src/libiksemel.la LOCAL_CFLAGS += -I../ -I$(BASE)/libs/iksemel/include -LOCAL_OBJS= $(PCRE_LA) $(IKS_LA) main.o ../nlsml.o +LOCAL_OBJS= $(PCRE_LA) $(IKS_LA) main.o ../nlsml.o ../iks_helpers.o LOCAL_SOURCES= main.c include $(BASE)/build/modmake.rules @@ -12,7 +12,7 @@ $(IKS_LA): $(IKS_DIR) $(IKS_DIR)/.update @$(TOUCH_TARGET) local_all: - libtool --mode=link gcc main.o ../nlsml.o -o test test_nlsml.la - + libtool --mode=link gcc main.o ../nlsml.o ../iks_helpers.o test_nlsml.la ../../../../../.libs/libfreeswitch.la ../../../../../libs/iksemel/src/.libs/libiksemel.a -lpcre -lssl -lcrypto -g -ggdb -O2 -pthread -o test + local_clean: -rm test