From 6d8d1a09d0a64adcd3c29abeb545fe2fe7034694 Mon Sep 17 00:00:00 2001 From: Brian West Date: Thu, 2 Jul 2009 00:54:48 +0000 Subject: [PATCH] commit e8f60761378fe392d80e89d0e3481316119e9809 Author: achaloyan Date: Tue Jun 30 14:44:09 2009 +0000 Further build and install integration git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1030 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 5b410e0dfc1852ee5c0d56d64b326d5130aed18a Author: achaloyan Date: Tue Jun 30 13:24:27 2009 +0000 Added utility project (preparesphinx) to copy the stuff pocketsphinx requires to run git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1029 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 2457575de160b378affdfa7a37cac1282d0024ca Author: achaloyan Date: Tue Jun 30 13:18:59 2009 +0000 Added a few more comments in config file git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1028 f001bc3a-424a-0410-80a0-a715b8f413a8 commit ffc40b15b409a79bdea286898ad1e8694fc1623c Author: garmt.noname@gmail.com Date: Tue Jun 30 10:38:54 2009 +0000 Added resampling to 8kHz so that it works with freeswitch (specify only L16/96/8000 codec in profile of media_engine/rtpfactory). Changed logging to DEBUG level rather than INFO. Added channel_guard for stop response. git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1027 f001bc3a-424a-0410-80a0-a715b8f413a8 commit d11439611186b46f1bfabc036b7e5d76f33f8b0e Author: achaloyan Date: Mon Jun 29 19:46:54 2009 +0000 Added entries for PocketSphinx (mrcppocketsphinx) and Flite (mrcpflite) plugins into unimrcpserver.xml (disabled by default) git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1026 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 63bc73426ba4efdf648a28cd3c1ff1daaef5bb49 Author: achaloyan Date: Mon Jun 29 15:04:01 2009 +0000 Added enumeration of pocketsphinx models (narrowband, wideband), supported wideband either git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1025 f001bc3a-424a-0410-80a0-a715b8f413a8 commit d11439611186b46f1bfabc036b7e5d76f33f8b0e Author: achaloyan Date: Mon Jun 29 19:46:54 2009 +0000 Added entries for PocketSphinx (mrcppocketsphinx) and Flite (mrcpflite) plugins into unimrcpserver.xml (disabled by default) git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1026 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 63bc73426ba4efdf648a28cd3c1ff1daaef5bb49 Author: achaloyan Date: Mon Jun 29 15:04:01 2009 +0000 Added enumeration of pocketsphinx models (narrowband, wideband), supported wideband either git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1025 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 04970484e4357e2a1c3c4385840640caada33468 Author: achaloyan Date: Mon Jun 29 13:21:35 2009 +0000 Removed engine->guard, as all relevant calls are made within the context of the same thread git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1024 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 9bac2f3abdcfea5397aca4b86e209af090631e7a Author: achaloyan Date: Mon Jun 29 13:15:30 2009 +0000 Initialized 16kHz codec descriptor for flite channel, since available flite voice are in 16kHz. git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1023 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 4e902eb985b433416723f15646d3e99d385d18cb Author: achaloyan Date: Sun Jun 28 20:05:22 2009 +0000 Do not create bridge if resampling is required. Several sampling rates are supported, but there is no resampling yet. git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1022 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 6d35b1246a7061e4c8f3f608bb17e146870d63bd Author: achaloyan Date: Sun Jun 28 18:14:25 2009 +0000 Added makefile target to install pocketsphinx.xml with make install git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1021 f001bc3a-424a-0410-80a0-a715b8f413a8 commit c2b75c89d57c02bd8d4360aebcb7406ecbf90eb0 Author: achaloyan Date: Sun Jun 28 18:10:01 2009 +0000 Set svn props (eol:native) git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1020 f001bc3a-424a-0410-80a0-a715b8f413a8 commit dd91ebea823dd2169e8c30f0cfe87fa199e1a0c2 Author: achaloyan Date: Sun Jun 28 17:46:46 2009 +0000 Loaded pocketsphinx's properties from config file git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1019 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 2ba91890593d7a64136e675bb937efd9a2542cc7 Author: garmt.noname@gmail.com Date: Sun Jun 28 12:29:54 2009 +0000 Removed session tasks, most channel tasks, flite voices are no longer global git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1018 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 0d739127f9267b3ad871d1a53a863802f101a6b5 Author: achaloyan Date: Sat Jun 27 09:15:20 2009 +0000 Implemented save_waveform, utterance will be saved in the filesystem git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1017 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 4ffd282ddf54ad861d73f36567ad201d135feff5 Author: achaloyan Date: Sat Jun 27 08:24:19 2009 +0000 Set 2 digits precision (digits after the decimal point) while generating float type values (Issue-35). git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1016 f001bc3a-424a-0410-80a0-a715b8f413a8 commit 90446f5e6ece40e91fd5b340a45e6773e4e80a0f Author: achaloyan Date: Sat Jun 27 07:42:52 2009 +0000 Set noinut and recognition timeouts if specified in RECOGNIZE request, reset input timer on partial match git-svn-id: https://unimrcp.googlecode.com/svn/trunk@1015 f001bc3a-424a-0410-80a0-a715b8f413a8 git-svn-id: http://svn.freeswitch.org/svn/freeswitch/trunk@14104 d0543943-73ff-0310-b7d9-9358b9ac24b2 --- libs/unimrcp/.update | 2 +- libs/unimrcp/build/acmacros/flite.m4 | 6 +- libs/unimrcp/build/acmacros/pocketsphinx.m4 | 3 + libs/unimrcp/build/tools/preparesphinx.vcproj | 69 ++ libs/unimrcp/conf/unimrcpserver.xml | 2 + .../apr-toolkit/include/apt_text_stream.h | 2 +- libs/unimrcp/libs/mpf/src/mpf_context.c | 6 + .../plugins/mrcp-flite/src/mrcp_flite.c | 652 +++++++----------- .../plugins/mrcp-pocketsphinx/Makefile.am | 11 +- .../mrcp-pocketsphinx/conf/pocketsphinx.xml | 24 + .../include/pocketsphinx_properties.h | 64 ++ .../mrcp-pocketsphinx/mrcppocketsphinx.vcproj | 8 + .../mrcp-pocketsphinx/src/mrcp_pocketsphinx.c | 166 +++-- .../src/pocketsphinx_properties.c | 209 ++++++ libs/unimrcp/unimrcp.sln | 5 + 15 files changed, 776 insertions(+), 453 deletions(-) create mode 100644 libs/unimrcp/build/tools/preparesphinx.vcproj create mode 100644 libs/unimrcp/plugins/mrcp-pocketsphinx/conf/pocketsphinx.xml create mode 100644 libs/unimrcp/plugins/mrcp-pocketsphinx/include/pocketsphinx_properties.h create mode 100644 libs/unimrcp/plugins/mrcp-pocketsphinx/src/pocketsphinx_properties.c diff --git a/libs/unimrcp/.update b/libs/unimrcp/.update index facb9c90bf..61425074b3 100644 --- a/libs/unimrcp/.update +++ b/libs/unimrcp/.update @@ -1 +1 @@ -Fri Jun 26 15:16:40 CDT 2009 +Wed Jul 1 19:53:07 CDT 2009 diff --git a/libs/unimrcp/build/acmacros/flite.m4 b/libs/unimrcp/build/acmacros/flite.m4 index 414098d134..a1991bc16b 100644 --- a/libs/unimrcp/build/acmacros/flite.m4 +++ b/libs/unimrcp/build/acmacros/flite.m4 @@ -6,15 +6,13 @@ AC_DEFUN([UNIMRCP_CHECK_FLITE], AC_MSG_CHECKING([for Flite]) AC_ARG_WITH(flite, - [ --with-flite=PATH prefix for installed Flite or - path to Flite build tree], + [ --with-flite=PATH path to Flite build tree], [flite_path=$withval], - [flite_path="/usr/local"] + [flite_path="/usr/src/flite"] ) found_flite="no" - dnl TO BE DONE flite_libdir="build/libs" for dir in $flite_path ; do cd $dir && flite_dir=`pwd` && cd - > /dev/null diff --git a/libs/unimrcp/build/acmacros/pocketsphinx.m4 b/libs/unimrcp/build/acmacros/pocketsphinx.m4 index 4c7205559b..8f1eadcebe 100644 --- a/libs/unimrcp/build/acmacros/pocketsphinx.m4 +++ b/libs/unimrcp/build/acmacros/pocketsphinx.m4 @@ -21,6 +21,7 @@ AC_DEFUN([UNIMRCP_CHECK_POCKETSPHINX], found_pocketsphinx="yes" UNIMRCP_POCKETSPHINX_INCLUDES="`pkg-config --cflags $dir/$pocketsphinx_config`" UNIMRCP_POCKETSPHINX_LIBS="`pkg-config --libs $dir/$pocketsphinx_config`" + UNIMRCP_POCKETSPHINX_MODELS= pocketsphinx_version="`pkg-config --modversion $dir/$pocketsphinx_config`" break fi @@ -28,6 +29,7 @@ AC_DEFUN([UNIMRCP_CHECK_POCKETSPHINX], found_pocketsphinx="yes" UNIMRCP_POCKETSPHINX_INCLUDES="-I$pocketsphinx_dir/include" UNIMRCP_POCKETSPHINX_LIBS="$pocketsphinx_dir/$pocketsphinx_srcdir/libpocketsphinx/libpocketsphinx.la" + UNIMRCP_POCKETSPHINX_MODELS="$pocketsphinx_dir/model" pocketsphinx_version="`pkg-config --modversion $pocketsphinx_dir/pocketsphinx.pc`" break fi @@ -46,5 +48,6 @@ esac AC_SUBST(UNIMRCP_POCKETSPHINX_INCLUDES) AC_SUBST(UNIMRCP_POCKETSPHINX_LIBS) + AC_SUBST(UNIMRCP_POCKETSPHINX_MODELS) fi ]) diff --git a/libs/unimrcp/build/tools/preparesphinx.vcproj b/libs/unimrcp/build/tools/preparesphinx.vcproj new file mode 100644 index 0000000000..8012033516 --- /dev/null +++ b/libs/unimrcp/build/tools/preparesphinx.vcproj @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/libs/unimrcp/conf/unimrcpserver.xml b/libs/unimrcp/conf/unimrcpserver.xml index 6cf213d53b..eac086bd4f 100644 --- a/libs/unimrcp/conf/unimrcpserver.xml +++ b/libs/unimrcp/conf/unimrcpserver.xml @@ -54,6 +54,8 @@ + + diff --git a/libs/unimrcp/libs/apr-toolkit/include/apt_text_stream.h b/libs/unimrcp/libs/apr-toolkit/include/apt_text_stream.h index f5fe1c30c0..e5aca8ab67 100644 --- a/libs/unimrcp/libs/apr-toolkit/include/apt_text_stream.h +++ b/libs/unimrcp/libs/apr-toolkit/include/apt_text_stream.h @@ -122,7 +122,7 @@ static APR_INLINE float apt_float_value_parse(const apt_str_t *str) /** Generate float value */ static APR_INLINE apt_bool_t apt_float_value_generate(float value, apt_text_stream_t *stream) { - int length = sprintf(stream->pos,"%.1f",value); + int length = sprintf(stream->pos,"%.2f",value); if(length <= 0) { return FALSE; } diff --git a/libs/unimrcp/libs/mpf/src/mpf_context.c b/libs/unimrcp/libs/mpf/src/mpf_context.c index 0b8bc2ed17..b209154753 100644 --- a/libs/unimrcp/libs/mpf/src/mpf_context.c +++ b/libs/unimrcp/libs/mpf/src/mpf_context.c @@ -208,6 +208,12 @@ static mpf_object_t* mpf_context_connection_create(mpf_context_t *context, mpf_t object = mpf_null_bridge_create(source,sink,context->pool); } else { + if(rx_codec->descriptor->sampling_rate != tx_codec->descriptor->sampling_rate) { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING, + "Resampling is not supported now. " + "Try to configure and use the same sampling rate on both ends"); + return NULL; + } if(rx_codec->vtable && rx_codec->vtable->decode) { /* set decoder before bridge */ mpf_audio_stream_t *decoder = mpf_decoder_create(source,context->pool); diff --git a/libs/unimrcp/plugins/mrcp-flite/src/mrcp_flite.c b/libs/unimrcp/plugins/mrcp-flite/src/mrcp_flite.c index a64d91ebbe..1db710ed0c 100644 --- a/libs/unimrcp/plugins/mrcp-flite/src/mrcp_flite.c +++ b/libs/unimrcp/plugins/mrcp-flite/src/mrcp_flite.c @@ -37,7 +37,6 @@ typedef struct flite_synth_engine_t flite_synth_engine_t; typedef struct flite_synth_channel_t flite_synth_channel_t; -typedef struct flite_synth_msg_t flite_synth_msg_t; /** Declaration of synthesizer engine methods */ static apt_bool_t flite_synth_engine_destroy(mrcp_resource_engine_t *engine); @@ -58,6 +57,16 @@ static apt_bool_t flite_synth_channel_open(mrcp_engine_channel_t *channel); static apt_bool_t flite_synth_channel_close(mrcp_engine_channel_t *channel); static apt_bool_t flite_synth_channel_request_process(mrcp_engine_channel_t *channel, mrcp_message_t *request); +static apt_bool_t flite_synth_channel_close_t(mrcp_engine_channel_t *channel); // wait for speak thread + +/** flite channel methods for processing MRCP channel request **/ +static apt_bool_t flite_synth_channel_speak(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response); +static apt_bool_t flite_synth_channel_stop(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response); +static apt_bool_t flite_synth_channel_pause(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response); +static apt_bool_t flite_synth_channel_resume(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response); +// static apt_bool_t flite_synth_channel_setparams(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response); +// static apt_bool_t flite_synth_channel_getparams(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response); + static const struct mrcp_engine_channel_method_vtable_t channel_vtable = { flite_synth_channel_destroy, flite_synth_channel_open, @@ -83,34 +92,25 @@ static const mpf_audio_stream_vtable_t audio_stream_vtable = { /** Declaration of flite synthesizer engine */ struct flite_synth_engine_t { - apt_consumer_task_t *task; int iChannels; - apr_thread_mutex_t *guard; + struct { + cst_voice *awb; + cst_voice *kal; + cst_voice *rms; + cst_voice *slt; + } voices; }; -// flite stuff -APT_BEGIN_EXTERN_C - +/** declarations for flite voices **/ cst_voice *register_cmu_us_awb(void); -void unregister_cmu_us_awb(cst_voice * v); - cst_voice *register_cmu_us_kal(void); -void unregister_cmu_us_kal(cst_voice * v); - cst_voice *register_cmu_us_rms(void); -void unregister_cmu_us_rms(cst_voice * v); - cst_voice *register_cmu_us_slt(void); +void unregister_cmu_us_awb(cst_voice * v); +void unregister_cmu_us_kal(cst_voice * v); +void unregister_cmu_us_rms(cst_voice * v); void unregister_cmu_us_slt(cst_voice * v); -APT_END_EXTERN_C - -static struct { - cst_voice *awb; - cst_voice *kal; - cst_voice *rms; - cst_voice *slt; -} voices; /** Declaration of flite synthesizer channel */ struct flite_synth_channel_t { @@ -122,27 +122,12 @@ struct flite_synth_channel_t { mpf_buffer_t *audio_buffer; // Audio buffer int iId; // Synth channel simultaneous reference count cst_voice *voice; - cst_wave *wave; apr_pool_t *pool; apt_consumer_task_t *task; + apr_thread_mutex_t *channel_guard; }; -typedef enum flite_synth_msg_type_e { - flite_synth_MSG_OPEN_CHANNEL, - flite_synth_MSG_CLOSE_CHANNEL, - flite_synth_MSG_REQUEST_PROCESS -} flite_synth_msg_type_e; - /** Declaration of flite synthesizer task message */ -struct flite_synth_msg_t { - flite_synth_msg_type_e type; - mrcp_engine_channel_t *channel; - mrcp_message_t *request; -}; - -/* mutex: may be flite library is not thread safe*/ -static apr_thread_mutex_t *flite_mutex; - struct flite_speak_msg_t { flite_synth_channel_t *channel; mrcp_message_t *request; @@ -150,16 +135,9 @@ struct flite_speak_msg_t { typedef struct flite_speak_msg_t flite_speak_msg_t; -// all calls to the Flite API functions (after initialization) -// will be carried out using a separate task -static apt_bool_t flite_synth_msg_signal(flite_synth_msg_type_e type, mrcp_engine_channel_t *channel, mrcp_message_t *request); -static apt_bool_t flite_synth_msg_process(apt_task_t *task, apt_task_msg_t *msg); -static apt_bool_t flite_synth_channel_open_t(mrcp_engine_channel_t * channel); -static apt_bool_t flite_synth_channel_close_t(mrcp_engine_channel_t * channel); - -// and we have a special task for the actual synthesis - +// we have a special task for the actual synthesis - // the task is created when a mrcp speak message is received -static apt_bool_t flite_speak_msg_process(apt_task_t *task, apt_task_msg_t *msg); +static apt_bool_t flite_speak(apt_task_t *task, apt_task_msg_t *msg); /** Declare this macro to use log routine of the server where the plugin is loaded from */ MRCP_PLUGIN_LOGGER_IMPLEMENT @@ -169,44 +147,9 @@ MRCP_PLUGIN_DECLARE(mrcp_resource_engine_t*) mrcp_plugin_create(apr_pool_t *pool { /* create flite engine */ flite_synth_engine_t *flite_engine = (flite_synth_engine_t *) apr_palloc(pool,sizeof(flite_synth_engine_t)); - apt_task_msg_pool_t *msg_pool; - apt_task_vtable_t *task_vtable = 0; - + flite_engine->iChannels = 0; - /* create task/thread to run flite engine in the context of this task */ - msg_pool = apt_task_msg_pool_create_dynamic(sizeof(flite_synth_msg_t),pool); - flite_engine->task = apt_consumer_task_create(flite_engine,msg_pool,pool); - if (!flite_engine->task) - { - apt_log(APT_LOG_MARK, APT_PRIO_ERROR, "MRCP_PLUGIN_DECLARE cannot create task"); - return NULL; - } - task_vtable = apt_consumer_task_vtable_get(flite_engine->task); - if (task_vtable) - { - task_vtable->process_msg = flite_synth_msg_process; - } - else - { - apt_log(APT_LOG_MARK, APT_PRIO_ERROR, "MRCP_PLUGIN_DECLARE cannot use task vtable"); - return NULL; - } - - /* create flite mutex */ - if (apr_thread_mutex_create(&flite_mutex,APR_THREAD_MUTEX_DEFAULT,pool) != APR_SUCCESS) - { - apt_log(APT_LOG_MARK, APT_PRIO_ERROR,"Failed to create flite mutex"); - return NULL; - } - - /* create channel mutex */ - if (apr_thread_mutex_create(&flite_engine->guard,APR_THREAD_MUTEX_DEFAULT,pool) != APR_SUCCESS) - { - apt_log(APT_LOG_MARK, APT_PRIO_ERROR, "Failed to create channel guard"); - return NULL; - } - /* create resource engine base */ return mrcp_resource_engine_create( MRCP_SYNTHESIZER_RESOURCE, /* MRCP resource identifier */ @@ -218,17 +161,7 @@ MRCP_PLUGIN_DECLARE(mrcp_resource_engine_t*) mrcp_plugin_create(apr_pool_t *pool /** Destroy synthesizer engine */ static apt_bool_t flite_synth_engine_destroy(mrcp_resource_engine_t *engine) { - flite_synth_engine_t *flite_engine = (flite_synth_engine_t *) engine->obj; - - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_engine_destroy"); - - if(flite_engine->task) { - apt_task_t *task = apt_consumer_task_base_get(flite_engine->task); - apt_task_destroy(task); - flite_engine->task = NULL; - } - - apr_thread_mutex_destroy(flite_engine->guard); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_engine_destroy"); return TRUE; } @@ -236,19 +169,16 @@ static apt_bool_t flite_synth_engine_destroy(mrcp_resource_engine_t *engine) static apt_bool_t flite_synth_engine_open(mrcp_resource_engine_t *engine) { flite_synth_engine_t *flite_engine = (flite_synth_engine_t *) engine->obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_engine_open"); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_engine_open"); flite_init(); - voices.awb = register_cmu_us_awb(); - voices.kal = register_cmu_us_kal(); - voices.rms = register_cmu_us_rms(); - voices.slt = register_cmu_us_slt(); - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "fliteInitialize success"); + flite_engine->voices.awb = register_cmu_us_awb(); + flite_engine->voices.kal = register_cmu_us_kal(); + flite_engine->voices.rms = register_cmu_us_rms(); + flite_engine->voices.slt = register_cmu_us_slt(); + + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite init success"); - if (flite_engine->task) { - apt_task_t *task = apt_consumer_task_base_get(flite_engine->task); - apt_task_start(task); - } return TRUE; } @@ -256,18 +186,12 @@ static apt_bool_t flite_synth_engine_open(mrcp_resource_engine_t *engine) static apt_bool_t flite_synth_engine_close(mrcp_resource_engine_t *engine) { flite_synth_engine_t *flite_engine = (flite_synth_engine_t *) engine->obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_engine_close"); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_engine_close"); - if (flite_engine->task) - { - apt_task_t *task = apt_consumer_task_base_get(flite_engine->task); - apt_task_terminate(task,TRUE); - } - - unregister_cmu_us_awb(voices.awb); - unregister_cmu_us_kal(voices.kal); - unregister_cmu_us_rms(voices.rms); - unregister_cmu_us_slt(voices.slt); + unregister_cmu_us_awb(flite_engine->voices.awb); + unregister_cmu_us_kal(flite_engine->voices.kal); + unregister_cmu_us_rms(flite_engine->voices.rms); + unregister_cmu_us_slt(flite_engine->voices.slt); return TRUE; } @@ -279,24 +203,31 @@ static mrcp_engine_channel_t* flite_synth_engine_channel_create(mrcp_resource_en flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) apr_palloc(pool,sizeof(flite_synth_channel_t)); mpf_codec_descriptor_t *codec_descriptor = NULL; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_engine_channel_create"); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_engine_channel_create"); -// codec_descriptor = (mpf_codec_descriptor_t *) apr_palloc(pool,sizeof(mpf_codec_descriptor_t)); +// codec_descriptor = (mpf_codec_descriptor_t *) apr_palloc(pool,sizeof(mpf_codec_descriptor_t)); // mpf_codec_descriptor_init(codec_descriptor); // codec_descriptor->channel_count = 1; // codec_descriptor->payload_type = 96; -// apt_string_set(&codec_descriptor->name,"L16"); +// apt_string_set(&codec_descriptor->name,"LPCM"); // codec_descriptor->sampling_rate = 16000; -// + synth_channel->flite_engine = (flite_synth_engine_t *) engine->obj; synth_channel->speak_request = NULL; // no active speak request in progress synth_channel->stop_response = NULL; synth_channel->paused = FALSE; synth_channel->pool = pool; - synth_channel->wave = NULL; + synth_channel->audio_buffer = NULL; synth_channel->voice = NULL; synth_channel->iId = 0; + if (apr_thread_mutex_create(&synth_channel->channel_guard,APR_THREAD_MUTEX_DEFAULT,pool) != APR_SUCCESS) + { + apt_log(APT_LOG_MARK, APT_PRIO_ERROR, "Failed to create channel guard"); + return NULL; + } + + /* create engine channel base */ synth_channel->channel = mrcp_engine_source_channel_create( engine, /* resource engine */ @@ -313,9 +244,7 @@ static mrcp_engine_channel_t* flite_synth_engine_channel_create(mrcp_resource_en } synth_channel->audio_buffer = mpf_buffer_create(pool); - apr_thread_mutex_lock(synth_channel->flite_engine->guard); synth_channel->iId = ++synth_channel->flite_engine->iChannels; - apr_thread_mutex_unlock(synth_channel->flite_engine->guard); apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_engine_channel_create created channel %d", synth_channel->iId); @@ -326,24 +255,24 @@ static mrcp_engine_channel_t* flite_synth_engine_channel_create(mrcp_resource_en static apt_bool_t flite_synth_channel_destroy(mrcp_engine_channel_t *channel) { flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_destroy - channel %d", synth_channel->iId); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_destroy - channel %d", synth_channel->iId); if(synth_channel->task) { apt_task_t *task = apt_consumer_task_base_get(synth_channel->task); - if (!apt_task_destroy(task)) + if (!task || !apt_task_destroy(task)) { apt_log(APT_LOG_MARK, APT_PRIO_WARNING, "Speak task destroy failed - channel %d", synth_channel->iId); } else { - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "Speak task destroyed - channel %d", synth_channel->iId); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "Speak task destroyed - channel %d", synth_channel->iId); } } + synth_channel->task = NULL; - - apr_thread_mutex_lock(synth_channel->flite_engine->guard); synth_channel->flite_engine->iChannels--; - apr_thread_mutex_unlock(synth_channel->flite_engine->guard); + + apr_thread_mutex_destroy(synth_channel->channel_guard); return TRUE; } @@ -352,210 +281,30 @@ static apt_bool_t flite_synth_channel_destroy(mrcp_engine_channel_t *channel) static apt_bool_t flite_synth_channel_open(mrcp_engine_channel_t *channel) { flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_open - channel %d", synth_channel->iId); - return flite_synth_msg_signal(flite_synth_MSG_OPEN_CHANNEL,channel,NULL); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_open - channel %d", synth_channel->iId); + + synth_channel->voice = synth_channel->flite_engine->voices.awb; + return mrcp_engine_channel_open_respond(channel,TRUE); } /** Close engine channel (asynchronous response MUST be sent)*/ static apt_bool_t flite_synth_channel_close(mrcp_engine_channel_t *channel) { flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_close - channel %d", synth_channel->iId); - return flite_synth_msg_signal(flite_synth_MSG_CLOSE_CHANNEL,channel,NULL); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_close - channel %d", synth_channel->iId); + // create thread to wait for speak thread to terminate + flite_synth_channel_close_t(channel); + return TRUE; } /** Process MRCP channel request (asynchronous response MUST be sent)*/ static apt_bool_t flite_synth_channel_request_process(mrcp_engine_channel_t *channel, mrcp_message_t *request) { flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_request_process - channel %d", synth_channel->iId); - return flite_synth_msg_signal(flite_synth_MSG_REQUEST_PROCESS,channel,request); -} - -/** Open engine channel */ -static apt_bool_t flite_synth_channel_open_t(mrcp_engine_channel_t *channel) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_open_t - channel %d", synth_channel->iId); - - synth_channel->voice = voices.awb; - return TRUE; -} - -static apt_bool_t flite_synth_channel_close_t(mrcp_engine_channel_t *channel) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_close_t - channel %d", synth_channel->iId); - - if (synth_channel->task) - { - apt_task_t *task = apt_consumer_task_base_get(synth_channel->task); - if (!apt_task_terminate(task,TRUE)) - { - apt_log(APT_LOG_MARK, APT_PRIO_WARNING, "Speak task terminate failed - channel %d", synth_channel->iId); - } - else - { - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "Speak task terminated - channel %d", synth_channel->iId); - apt_task_destroy(task); - synth_channel->task = 0; - } - } - if (synth_channel->wave) - { - delete_wave(synth_channel->wave); - synth_channel->wave = 0; - } - else - { - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "No wave buffer to delete - channel %d", synth_channel->iId); - } - return TRUE; -} - - -/** Process SPEAK request */ -static apt_bool_t flite_synth_channel_speak(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_speak - channel %d", synth_channel->iId); - - if (!synth_channel->speak_request) - { - apt_task_msg_pool_t *msg_pool = apt_task_msg_pool_create_dynamic( sizeof(flite_speak_msg_t),synth_channel->pool); - apt_task_vtable_t *task_vtable = 0; - apt_task_t * task = 0; - apt_task_msg_t *msg = 0; - flite_speak_msg_t *flite_msg = 0; - - /* create task/thread to run flite so this function is not blocking */ - apt_log(APT_LOG_MARK,APT_PRIO_NOTICE,"Create flite speak task - channel: %d", synth_channel->iId); - synth_channel->task = apt_consumer_task_create(synth_channel, msg_pool, synth_channel->pool); - if (!synth_channel->task) - { - apt_log(APT_LOG_MARK,APT_PRIO_ERROR, "flite_synth_channel_speak failed to create flite speak task - channel:%d", synth_channel->iId); - return FALSE; - } - - task_vtable = apt_consumer_task_vtable_get(synth_channel->task); - if (!task_vtable) - { - apt_log(APT_LOG_MARK,APT_PRIO_ERROR, "flite_synth_channel_speak cannot use flite speak task vtable - channel:%d", synth_channel->iId); - return FALSE; - } - - task_vtable->process_msg = flite_speak_msg_process; - synth_channel->speak_request = request; - - apt_log(APT_LOG_MARK,APT_PRIO_INFO, "Start task - channel %d", synth_channel->iId); - task = apt_consumer_task_base_get(synth_channel->task); - if (apt_task_start(task) == FALSE) - { - apt_log(APT_LOG_MARK,APT_PRIO_ERROR, "flite_synth_channel_speak failed to start task - channel: %d", synth_channel->iId); - apt_task_destroy(task); - return FALSE; - } - - msg = apt_task_msg_acquire(msg_pool); - msg->type = TASK_MSG_USER; - flite_msg = (flite_speak_msg_t*) msg->data; - flite_msg->channel = synth_channel; - flite_msg->request = request; - apt_log(APT_LOG_MARK,APT_PRIO_INFO, "Send signal to start speech synthesis - channel:%d", synth_channel->iId); - if (apt_task_msg_signal(task,msg)) - { - response->start_line.request_state = MRCP_REQUEST_STATE_INPROGRESS; - mrcp_engine_channel_message_send(channel,response); - } - else - { - apt_log(APT_LOG_MARK,APT_PRIO_INFO, "Failed to send signal to start speech synthesis - channel:%d", synth_channel->iId); - apt_task_destroy(task); - return FALSE; - } - } - return TRUE; -} - -static apt_bool_t flite_speak_msg_process(apt_task_t *task, apt_task_msg_t *msg) -{ - flite_speak_msg_t *flite_msg = (flite_speak_msg_t*)msg->data; - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) flite_msg->channel; - apr_time_t start = 0; - apr_time_t elapsed = 0; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "< flite_speak_msg_process speak - channel %d", synth_channel->iId); - - // probably not necessary !? - mpf_buffer_restart(synth_channel->audio_buffer); - - // just sequential stuff - start = apr_time_now(); // in microsec - if (synth_channel->speak_request->body.length) - { - cst_wave * wave = synth_channel->wave; - wave = flite_text_to_wave(synth_channel->speak_request->body.buf, synth_channel->voice); - } - - elapsed = (apr_time_now() - start)/1000; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "TTS (chan %d) took %ld millisec", synth_channel->iId, elapsed); - - if (!synth_channel->stop_response) - { - // this will notify the callback that feeds the client that synthesis is complete - mpf_buffer_event_write(synth_channel->audio_buffer, MEDIA_FRAME_TYPE_EVENT); - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "> flite_speak_msg_process speak - end of TTS - %d", synth_channel->iId); - } - else - { - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "> flite_speak_msg_process speak - channel %d", synth_channel->iId); - } - return TRUE; -} - -/** Process STOP request */ -static apt_bool_t flite_synth_channel_stop(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_stop - channel %d", synth_channel->iId); - /* store the request, make sure there is no more activity and only then send the response */ - - /* TODO this should probably be mutexed */ - synth_channel->stop_response = response; - return TRUE; -} - -/** Process PAUSE request */ -static apt_bool_t flite_synth_channel_pause(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_pause - channel %d", synth_channel->iId); - - synth_channel->paused = TRUE; - /* send asynchronous response */ - mrcp_engine_channel_message_send(channel,response); - return TRUE; -} - -/** Process RESUME request */ -static apt_bool_t flite_synth_channel_resume(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_resume - channel %d", synth_channel->iId); - - synth_channel->paused = FALSE; - /* send asynchronous response */ - mrcp_engine_channel_message_send(channel,response); - return TRUE; -} - -/** Dispatch MRCP request */ -static apt_bool_t flite_synth_channel_request_dispatch(mrcp_engine_channel_t *channel, mrcp_message_t *request) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_bool_t processed = FALSE; mrcp_message_t *response = mrcp_response_create(request,request->pool); + apt_bool_t processed = FALSE; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_request_dispatch - channel %d", synth_channel->iId); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_request_process - channel %d", synth_channel->iId); switch(request->start_line.method_id) { case SYNTHESIZER_SET_PARAMS: @@ -563,13 +312,13 @@ static apt_bool_t flite_synth_channel_request_dispatch(mrcp_engine_channel_t *ch // if (!strcasecmp(voice_name, "awb")) { // synth_channel->voice = voices.awb; // } else if (!strcasecmp(voice_name, "kal")) { - // synth_channel->voice = voices.awb; + // synth_channel->voice = voices.kal; // } else if (!strcasecmp(voice_name, "rms")) { - // synth_channel->voice = voices.awb; + // synth_channel->voice = voices.rms; // } else if (!strcasecmp(voice_name, "slt")) { - // synth_channel->voice = voices.awb; + // synth_channel->voice = voices.slt; // } else { - // apt_log(APT_LOG_MARK, APT_PRIO_INFO, "Valid voice names are awb, kal, rms or slt"); + // apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "Valid voice names are awb, kal, rms or slt"); // } break; case SYNTHESIZER_GET_PARAMS: @@ -601,6 +350,195 @@ static apt_bool_t flite_synth_channel_request_dispatch(mrcp_engine_channel_t *ch mrcp_engine_channel_message_send(channel,response); } return TRUE; + +} + +static apt_bool_t flite_synth_channel_close_t(mrcp_engine_channel_t *channel) +{ + flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_close_t - channel %d", synth_channel->iId); + + if (synth_channel->task) + { + apt_task_t *task = apt_consumer_task_base_get(synth_channel->task); + if (!apt_task_terminate(task,TRUE)) + { + apt_log(APT_LOG_MARK, APT_PRIO_WARNING, "Speak task terminate failed - channel %d", synth_channel->iId); + } + else + { + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "Speak task terminated - channel %d", synth_channel->iId); + apt_task_destroy(task); + synth_channel->task = 0; + } + } + mrcp_engine_channel_close_respond(channel); + return TRUE; +} + + +/** Process SPEAK request */ +static apt_bool_t flite_synth_channel_speak(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) +{ + flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; + apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_channel_speak - channel %d", synth_channel->iId); + + if (!synth_channel->speak_request) + { + apt_task_msg_pool_t *msg_pool = apt_task_msg_pool_create_dynamic( sizeof(flite_speak_msg_t),synth_channel->pool); + apt_task_vtable_t *task_vtable = 0; + apt_task_t * task = 0; + apt_task_msg_t *msg = 0; + flite_speak_msg_t *flite_msg = 0; + + /* create task/thread to run flite so this function is not blocking */ + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "Create flite speak task - channel: %d", synth_channel->iId); + synth_channel->task = apt_consumer_task_create(synth_channel, msg_pool, synth_channel->pool); + if (!synth_channel->task) + { + apt_log(APT_LOG_MARK,APT_PRIO_ERROR, "flite_synth_channel_speak failed to create flite speak task - channel:%d", synth_channel->iId); + return FALSE; + } + + task_vtable = apt_consumer_task_vtable_get(synth_channel->task); + if (!task_vtable) + { + apt_log(APT_LOG_MARK,APT_PRIO_ERROR, "flite_synth_channel_speak cannot use flite speak task vtable - channel:%d", synth_channel->iId); + return FALSE; + } + + task_vtable->process_msg = flite_speak; + synth_channel->speak_request = request; + + apt_log(APT_LOG_MARK,APT_PRIO_DEBUG, "Start task - channel %d", synth_channel->iId); + task = apt_consumer_task_base_get(synth_channel->task); + if (apt_task_start(task) == FALSE) + { + apt_log(APT_LOG_MARK,APT_PRIO_ERROR, "flite_synth_channel_speak failed to start task - channel: %d", synth_channel->iId); + apt_task_destroy(task); + return FALSE; + } + + msg = apt_task_msg_acquire(msg_pool); + msg->type = TASK_MSG_USER; + flite_msg = (flite_speak_msg_t*) msg->data; + flite_msg->channel = synth_channel; + flite_msg->request = request; + apt_log(APT_LOG_MARK,APT_PRIO_DEBUG, "Send signal to start speech synthesis - channel:%d", synth_channel->iId); + if (apt_task_msg_signal(task,msg)) + { + response->start_line.request_state = MRCP_REQUEST_STATE_INPROGRESS; + mrcp_engine_channel_message_send(channel,response); + } + else + { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING, "Failed to send signal to start speech synthesis - channel:%d", synth_channel->iId); + apt_task_destroy(task); + return FALSE; + } + } + return TRUE; +} + +static apt_bool_t flite_speak(apt_task_t *task, apt_task_msg_t *msg) +{ + flite_speak_msg_t *flite_msg = (flite_speak_msg_t*)msg->data; + flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) flite_msg->channel; + apr_time_t start = 0; + apr_time_t elapsed = 0; + apr_time_t stamp = 0; + + mrcp_message_t *stop_response = 0; + mpf_codec_t * codec = mrcp_engine_source_stream_codec_get(synth_channel->channel); + apr_uint16_t rate = codec->descriptor->sampling_rate; + + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "< flite_speak_msg_process speak - channel %d", synth_channel->iId); + + // just sequential stuff + start = apr_time_now(); // in microsec + if (synth_channel->speak_request->body.length) + { + // TODO + // create small units of text from synth_channel->speak_request->body.buf ( , . ? ! but ... + // synthesize small unit and store in audio_buffer + // check for stop + // pause resume state could improve performance + // you can "pause" generating new speech from a unit of text + // by checking the (decreasing) size of the audio_buffer + // no need to generate more speech samples than can be listened to... + cst_wave *wave = 0; + wave = flite_text_to_wave(synth_channel->speak_request->body.buf, synth_channel->voice); + if (wave && cst_wave_num_samples(wave)) + { + int generated = (cst_wave_num_samples(wave)/cst_wave_sample_rate(wave)*1000); + stamp = apr_time_now(); + elapsed = (stamp - start)/1000; + apt_log(APT_LOG_MARK, APT_PRIO_INFO, "TTS (chan %d) took %"APR_TIME_T_FMT" to generate %d of speech (in millisec)", synth_channel->iId, elapsed, generated); + + if (rate != 16000) + { + cst_wave_resample(wave, rate); + elapsed = (apr_time_now() - stamp)/1000; + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "TTS resampling to %d on (chan %d) took %"APR_TIME_T_FMT" millisec", rate, synth_channel->iId, elapsed); + } + mpf_buffer_audio_write(synth_channel->audio_buffer, cst_wave_samples(wave), cst_wave_num_samples(wave) * 2); + delete_wave(wave); + } + } + + apr_thread_mutex_lock(synth_channel->channel_guard); + stop_response = synth_channel->stop_response; + apr_thread_mutex_unlock(synth_channel->channel_guard); + + if (!stop_response) + { + // this will notify the callback that feeds the client that synthesis is complete + mpf_buffer_event_write(synth_channel->audio_buffer, MEDIA_FRAME_TYPE_EVENT); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "> flite_speak_msg_process speak - end of TTS - %d", synth_channel->iId); + } + else + { + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "> flite_speak_msg_process speak - channel %d", synth_channel->iId); + } + return TRUE; +} + +/** Process STOP request */ +static apt_bool_t flite_synth_channel_stop(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) +{ + flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_stop - channel %d", synth_channel->iId); + /* store the request, make sure there is no more activity and only then send the response */ + + apr_thread_mutex_lock(synth_channel->channel_guard); + synth_channel->stop_response = response; + apr_thread_mutex_unlock(synth_channel->channel_guard); + + return TRUE; +} + +/** Process PAUSE request */ +static apt_bool_t flite_synth_channel_pause(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) +{ + flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_pause - channel %d", synth_channel->iId); + + synth_channel->paused = TRUE; + /* send asynchronous response */ + mrcp_engine_channel_message_send(channel,response); + return TRUE; +} + +/** Process RESUME request */ +static apt_bool_t flite_synth_channel_resume(mrcp_engine_channel_t *channel, mrcp_message_t *request, mrcp_message_t *response) +{ + flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_channel_resume - channel %d", synth_channel->iId); + + synth_channel->paused = FALSE; + /* send asynchronous response */ + mrcp_engine_channel_message_send(channel,response); + return TRUE; } /** Callback is called from MPF engine context to destroy any additional data associated with audio stream */ @@ -613,16 +551,16 @@ static apt_bool_t flite_synth_stream_destroy(mpf_audio_stream_t *stream) /** Callback is called from MPF engine context to perform any action before open */ static apt_bool_t flite_synth_stream_open(mpf_audio_stream_t *stream) { - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) stream->obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_stream_open - channel %d", synth_channel->iId); +// flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) stream->obj; +// apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_stream_open - channel %d", synth_channel->iId); return TRUE; } /** Callback is called from MPF engine context to perform any action after close */ static apt_bool_t flite_synth_stream_close(mpf_audio_stream_t *stream) { - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) stream->obj; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_stream_close - channel %d", synth_channel->iId); +// flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) stream->obj; +// apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_stream_close - channel %d", synth_channel->iId); return TRUE; } @@ -631,7 +569,7 @@ static apt_bool_t flite_synth_speak_complete_raise(flite_synth_channel_t *synth_ { mrcp_message_t *message = 0; mrcp_synth_header_t * synth_header = 0; - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_speak_complete_raise - channel %d", synth_channel->iId); + apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_speak_complete_raise - channel %d", synth_channel->iId); if (!synth_channel->speak_request) { return FALSE; @@ -664,7 +602,6 @@ static apt_bool_t flite_synth_speak_complete_raise(flite_synth_channel_t *synth_ static apt_bool_t flite_synth_stream_read(mpf_audio_stream_t *stream, mpf_frame_t *frame) { flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) stream->obj; - cst_wave * wave = synth_channel->wave; if (synth_channel->stop_response && synth_channel->speak_request) { /* send asynchronous response to STOP request */ @@ -674,17 +611,12 @@ static apt_bool_t flite_synth_stream_read(mpf_audio_stream_t *stream, mpf_frame_ return TRUE; } - if (wave && cst_wave_num_samples(wave)) - { - mpf_buffer_audio_write(synth_channel->audio_buffer, cst_wave_samples(wave), cst_wave_num_samples(wave) * 2); - } - /* check if there is active SPEAK request and it isn't in paused state */ if (synth_channel->speak_request && synth_channel->paused == FALSE) { /* normal processing */ mpf_buffer_frame_read(synth_channel->audio_buffer,frame); -// apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_stream_read - channel %d - size %d", synth_channel->iId, mpf_buffer_get_size(synth_channel->audio_buffer)); +// apt_log(APT_LOG_MARK, APT_PRIO_DEBUG, "flite_synth_stream_read - channel %d - size %d", synth_channel->iId, mpf_buffer_get_size(synth_channel->audio_buffer)); if((frame->type & MEDIA_FRAME_TYPE_EVENT) == MEDIA_FRAME_TYPE_EVENT) { @@ -692,58 +624,4 @@ static apt_bool_t flite_synth_stream_read(mpf_audio_stream_t *stream, mpf_frame_ } } return TRUE; -} - -static apt_bool_t flite_synth_msg_signal(flite_synth_msg_type_e type, mrcp_engine_channel_t *channel, mrcp_message_t *request) -{ - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) channel->method_obj; - apt_bool_t status = FALSE; - flite_synth_engine_t *flite_engine = synth_channel->flite_engine; - apt_task_t *task = apt_consumer_task_base_get(flite_engine->task); - apt_task_msg_t *msg = apt_task_msg_get(task); - - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_msg_signal - channel %d", synth_channel->iId); - - if (msg) { - flite_synth_msg_t *flite_msg; - msg->type = TASK_MSG_USER; - flite_msg = (flite_synth_msg_t*) msg->data; - - flite_msg->type = type; - flite_msg->channel = channel; - flite_msg->request = request; - status = apt_task_msg_signal(task,msg); - } - else - { - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_msg_signal - task msg not found %d", synth_channel->iId); - } - return status; -} - -static apt_bool_t flite_synth_msg_process(apt_task_t *task, apt_task_msg_t *msg) -{ - flite_synth_msg_t *flite_msg = (flite_synth_msg_t*)msg->data; - flite_synth_channel_t *synth_channel = (flite_synth_channel_t *) flite_msg->channel->method_obj; - - apt_log(APT_LOG_MARK, APT_PRIO_INFO, "flite_synth_msg_process - channel %d", synth_channel->iId); - - switch(flite_msg->type) { - case flite_synth_MSG_OPEN_CHANNEL: - /* open channel and send asynch response */ - flite_synth_channel_open_t(flite_msg->channel); - mrcp_engine_channel_open_respond(flite_msg->channel,TRUE); - break; - case flite_synth_MSG_CLOSE_CHANNEL: - /* close channel, make sure there is no activity and send asynch response */ - flite_synth_channel_close_t(flite_msg->channel); - mrcp_engine_channel_close_respond(flite_msg->channel); - break; - case flite_synth_MSG_REQUEST_PROCESS: - flite_synth_channel_request_dispatch(flite_msg->channel,flite_msg->request); - break; - default: - break; - } - return TRUE; -} +} \ No newline at end of file diff --git a/libs/unimrcp/plugins/mrcp-pocketsphinx/Makefile.am b/libs/unimrcp/plugins/mrcp-pocketsphinx/Makefile.am index 5b0c27b3d9..a38f3a56b7 100644 --- a/libs/unimrcp/plugins/mrcp-pocketsphinx/Makefile.am +++ b/libs/unimrcp/plugins/mrcp-pocketsphinx/Makefile.am @@ -15,6 +15,15 @@ INCLUDES = -Iinclude \ plugin_LTLIBRARIES = mrcppocketsphinx.la -mrcppocketsphinx_la_SOURCES = src/mrcp_pocketsphinx.c +mrcppocketsphinx_la_SOURCES = src/mrcp_pocketsphinx.c \ + src/pocketsphinx_properties.c mrcppocketsphinx_la_LDFLAGS = -module $(PLUGIN_LT_VERSION) mrcppocketsphinx_la_LIBADD = $(UNIMRCP_POCKETSPHINX_LIBS) $(UNIMRCP_SPHINXBASE_LIBS) -lm + + +install-data-local: + test -d $(confdir) || $(mkinstalldirs) $(confdir) + test -f $(confdir)/pocketsphinx.xml || $(INSTALL) -m 644 conf/pocketsphinx.xml $(confdir) + test -d $(datadir)/wsj1 || $(mkinstalldirs) $(datadir)/wsj1; \ + $(INSTALL) -m 644 $(UNIMRCP_POCKETSPHINX_MODELS)/hmm/wsj1/* $(datadir)/wsj1; \ + $(INSTALL) -m 644 $(UNIMRCP_POCKETSPHINX_MODELS)/lm/cmudict.0.6d $(datadir)/default.dic diff --git a/libs/unimrcp/plugins/mrcp-pocketsphinx/conf/pocketsphinx.xml b/libs/unimrcp/plugins/mrcp-pocketsphinx/conf/pocketsphinx.xml new file mode 100644 index 0000000000..f9bbb6a6b3 --- /dev/null +++ b/libs/unimrcp/plugins/mrcp-pocketsphinx/conf/pocketsphinx.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + diff --git a/libs/unimrcp/plugins/mrcp-pocketsphinx/include/pocketsphinx_properties.h b/libs/unimrcp/plugins/mrcp-pocketsphinx/include/pocketsphinx_properties.h new file mode 100644 index 0000000000..e37e55b940 --- /dev/null +++ b/libs/unimrcp/plugins/mrcp-pocketsphinx/include/pocketsphinx_properties.h @@ -0,0 +1,64 @@ +/* + * Copyright 2008 Arsen Chaloyan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __POCKETSPHINX_PROPERTIES_H__ +#define __POCKETSPHINX_PROPERTIES_H__ + +/** + * @file pocketsphinx_properties.h + * @brief PocketSphinx Properties + */ + +#include "apt_dir_layout.h" + +APT_BEGIN_EXTERN_C + +typedef enum { + POCKETSPHINX_MODEL_NARROWBAND, + POCKETSPHINX_MODEL_WIDEBAND + +} pocketsphinx_model_e; + +/** Declaration of pocketsphinx properties */ +typedef struct pocketsphinx_properties_t pocketsphinx_properties_t; + +/** Pocketsphinx properties */ +struct pocketsphinx_properties_t { + const char *data_dir; + const char *dictionary; + const char *model_8k; + const char *model_16k; + pocketsphinx_model_e preferred_model; + + apr_size_t sensitivity_level; + apr_size_t sensitivity_timeout; + + apr_size_t no_input_timeout; + apr_size_t recognition_timeout; + apr_size_t partial_result_timeout; + + apt_bool_t save_waveform; + const char *save_waveform_dir; +}; + +apt_bool_t pocketsphinx_properties_load(pocketsphinx_properties_t *properties, + const char *file_path, + const apt_dir_layout_t *dir_layout, + apr_pool_t *pool); + +APT_END_EXTERN_C + +#endif /*__POCKETSPHINX_PROPERTIES_H__*/ diff --git a/libs/unimrcp/plugins/mrcp-pocketsphinx/mrcppocketsphinx.vcproj b/libs/unimrcp/plugins/mrcp-pocketsphinx/mrcppocketsphinx.vcproj index c04f99707f..73714e9a00 100644 --- a/libs/unimrcp/plugins/mrcp-pocketsphinx/mrcppocketsphinx.vcproj +++ b/libs/unimrcp/plugins/mrcp-pocketsphinx/mrcppocketsphinx.vcproj @@ -149,6 +149,10 @@ Filter="h;hpp;hxx;hm;inl;inc;xsd" UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}" > + + + + diff --git a/libs/unimrcp/plugins/mrcp-pocketsphinx/src/mrcp_pocketsphinx.c b/libs/unimrcp/plugins/mrcp-pocketsphinx/src/mrcp_pocketsphinx.c index e4403a37a6..3a16fb7579 100644 --- a/libs/unimrcp/plugins/mrcp-pocketsphinx/src/mrcp_pocketsphinx.c +++ b/libs/unimrcp/plugins/mrcp-pocketsphinx/src/mrcp_pocketsphinx.c @@ -35,14 +35,15 @@ #include "mrcp_generic_header.h" #include "mrcp_message.h" #include "mpf_activity_detector.h" +#include "pocketsphinx_properties.h" #include "apt_log.h" +#define POCKETSPHINX_CONFFILE_NAME "pocketsphinx.xml" #define RECOGNIZER_SIDRES(recognizer) (recognizer)->channel->id.buf, "pocketsphinx" typedef struct pocketsphinx_engine_t pocketsphinx_engine_t; typedef struct pocketsphinx_recognizer_t pocketsphinx_recognizer_t; -typedef struct pocketsphinx_properties_t pocketsphinx_properties_t; /** Methods of recognition engine */ static apt_bool_t pocketsphinx_engine_destroy(mrcp_resource_engine_t *engine); @@ -86,23 +87,14 @@ static const mpf_audio_stream_vtable_t audio_stream_vtable = { /** Pocketsphinx engine (engine is an aggregation of recognizers) */ struct pocketsphinx_engine_t { - mrcp_resource_engine_t *base; -}; - -/** Pocketsphinx properties */ -struct pocketsphinx_properties_t { - const char *dictionary; - const char *model_8k; - const char *model_16k; - apr_size_t noinput_timeout; - apr_size_t recognition_timeout; - apr_size_t partial_result_timeout; + /* Resource engine base */ + mrcp_resource_engine_t *base; + /** Properties loaded from config file */ + pocketsphinx_properties_t properties; }; /** Pocketsphinx channel (recognizer) */ struct pocketsphinx_recognizer_t { - /** Back pointer to engine */ - pocketsphinx_engine_t *engine; /** Engine channel base */ mrcp_engine_channel_t *channel; @@ -110,12 +102,12 @@ struct pocketsphinx_recognizer_t { ps_decoder_t *decoder; /** Configuration */ cmd_ln_t *config; - /** Properties (to be loaded from config file) */ + /** Recognizer properties coppied from defualt engine properties */ pocketsphinx_properties_t properties; /** Is input timer started */ apt_bool_t is_input_timer_on; /** Noinput timeout */ - apr_size_t noinput_timeout; + apr_size_t no_input_timeout; /** Recognition timeout */ apr_size_t recognition_timeout; /** Timeout elapsed since the last partial result checking */ @@ -126,6 +118,8 @@ struct pocketsphinx_recognizer_t { const char *grammar_id; /** Table of defined grammars (key=content-id, value=grammar-file-path) */ apr_table_t *grammar_table; + /** File to write waveform to if save_waveform is on */ + apr_file_t *waveform; /** Voice activity detector */ mpf_activity_detector_t *detector; @@ -170,33 +164,42 @@ MRCP_PLUGIN_DECLARE(mrcp_resource_engine_t*) mrcp_plugin_create(apr_pool_t *pool } /** Destroy pocketsphinx engine */ -static apt_bool_t pocketsphinx_engine_destroy(mrcp_resource_engine_t *engine) +static apt_bool_t pocketsphinx_engine_destroy(mrcp_resource_engine_t *resource_engine) { return TRUE; } /** Open pocketsphinx engine */ -static apt_bool_t pocketsphinx_engine_open(mrcp_resource_engine_t *engine) +static apt_bool_t pocketsphinx_engine_open(mrcp_resource_engine_t *resource_engine) { + pocketsphinx_engine_t *engine = resource_engine->obj; + const apt_dir_layout_t *dir_layout = resource_engine->dir_layout; + + char *file_path = NULL; + apr_filepath_merge(&file_path,dir_layout->conf_dir_path,POCKETSPHINX_CONFFILE_NAME,0,resource_engine->pool); + + /* load properties */ + pocketsphinx_properties_load(&engine->properties,file_path,dir_layout,resource_engine->pool); return TRUE; } /** Close pocketsphinx engine */ -static apt_bool_t pocketsphinx_engine_close(mrcp_resource_engine_t *engine) +static apt_bool_t pocketsphinx_engine_close(mrcp_resource_engine_t *resource_engine) { return TRUE; } /** Create pocketsphinx recognizer */ -static mrcp_engine_channel_t* pocketsphinx_engine_recognizer_create(mrcp_resource_engine_t *engine, apr_pool_t *pool) +static mrcp_engine_channel_t* pocketsphinx_engine_recognizer_create(mrcp_resource_engine_t *resource_engine, apr_pool_t *pool) { mrcp_engine_channel_t *channel; + mpf_codec_descriptor_t *codec_descriptor; + pocketsphinx_engine_t *engine = resource_engine->obj; pocketsphinx_recognizer_t *recognizer = apr_palloc(pool,sizeof(pocketsphinx_recognizer_t)); -// recognizer->engine = engine; recognizer->decoder = NULL; recognizer->config = NULL; recognizer->is_input_timer_on = FALSE; - recognizer->noinput_timeout = 0; + recognizer->no_input_timeout = 0; recognizer->recognition_timeout = 0; recognizer->partial_result_timeout = 0; recognizer->last_result = NULL; @@ -211,10 +214,24 @@ static mrcp_engine_channel_t* pocketsphinx_engine_recognizer_create(mrcp_resourc recognizer->close_requested = FALSE; recognizer->grammar_id = NULL; recognizer->grammar_table = apr_table_make(pool,1); - + recognizer->waveform = NULL; + + /* copy default properties loaded from config */ + recognizer->properties = engine->properties; + + codec_descriptor = (mpf_codec_descriptor_t *) apr_palloc(pool,sizeof(mpf_codec_descriptor_t)); + mpf_codec_descriptor_init(codec_descriptor); + codec_descriptor->channel_count = 1; + codec_descriptor->payload_type = 96; + apt_string_set(&codec_descriptor->name,"LPCM"); + codec_descriptor->sampling_rate = 8000; + if(recognizer->properties.preferred_model == POCKETSPHINX_MODEL_WIDEBAND) { + codec_descriptor->sampling_rate = 16000; + } + /* create engine channel base */ channel = mrcp_engine_sink_channel_create( - engine, /* resource engine */ + resource_engine, /* resource engine */ &channel_vtable, /* virtual methods table of engine channel */ &audio_stream_vtable, /* virtual methods table of audio stream */ recognizer, /* object to associate */ @@ -295,33 +312,23 @@ static apt_bool_t pocketsphinx_recognizer_request_process(mrcp_engine_channel_t return TRUE; } - - -/** Load pocketsphinx properties [RECOG] */ -static apt_bool_t pocketsphinx_properties_load(pocketsphinx_recognizer_t *recognizer) -{ - mrcp_engine_channel_t *channel = recognizer->channel; - const apt_dir_layout_t *dir_layout = channel->engine->dir_layout; - pocketsphinx_properties_t *properties = &recognizer->properties; - - properties->dictionary = apt_datadir_filepath_get(dir_layout,"pocketsphinx/default.dic",channel->pool); - properties->model_8k = apt_datadir_filepath_get(dir_layout,"pocketsphinx/communicator",channel->pool); - properties->model_16k = apt_datadir_filepath_get(dir_layout,"pocketsphinx/wsj1",channel->pool); - - properties->noinput_timeout = 5000; - properties->recognition_timeout = 15000; - properties->partial_result_timeout = 100; - - return TRUE; -} - /** Initialize pocketsphinx decoder [RECOG] */ static apt_bool_t pocketsphinx_decoder_init(pocketsphinx_recognizer_t *recognizer, const char *grammar) { - apt_log(APT_LOG_MARK,APT_PRIO_DEBUG,"Init Config "APT_SIDRES_FMT,RECOGNIZER_SIDRES(recognizer)); + const char *model = recognizer->properties.model_8k; + const char *rate = "8000"; + if(recognizer->properties.preferred_model == POCKETSPHINX_MODEL_WIDEBAND) { + model = recognizer->properties.model_16k; + rate = "16000"; + } + + apt_log(APT_LOG_MARK,APT_PRIO_DEBUG,"Init Config rate [%s] dictionary [%s] "APT_SIDRES_FMT, + rate, + recognizer->properties.dictionary, + RECOGNIZER_SIDRES(recognizer)); recognizer->config = cmd_ln_init(recognizer->config, ps_args(), FALSE, - "-samprate", "8000", - "-hmm", recognizer->properties.model_8k, + "-samprate", rate, + "-hmm", model, "-jsgf", grammar, "-dict", recognizer->properties.dictionary, "-frate", "50", @@ -469,7 +476,7 @@ static apt_bool_t pocketsphinx_define_grammar(pocketsphinx_recognizer_t *recogni return FALSE; } - grammar_file_name = apr_psprintf(channel->pool,"pocketsphinx/%s-%s.gram",channel->id.buf,content_id); + grammar_file_name = apr_psprintf(channel->pool,"%s-%s.gram",channel->id.buf,content_id); grammar_file_path = apt_datadir_filepath_get(dir_layout,grammar_file_name,channel->pool); apt_log(APT_LOG_MARK,APT_PRIO_INFO,"Create Grammar File [%s] "APT_SIDRES_FMT, @@ -519,6 +526,7 @@ static apt_bool_t pocketsphinx_define_grammar(pocketsphinx_recognizer_t *recogni /** Process RECOGNIZE request [RECOG] */ static apt_bool_t pocketsphinx_recognize(pocketsphinx_recognizer_t *recognizer, mrcp_message_t *request, mrcp_message_t *response) { + mrcp_engine_channel_t *channel = recognizer->channel; mrcp_recog_header_t *request_recog_header; mrcp_recog_header_t *response_recog_header = mrcp_resource_header_prepare(response); if(!response_recog_header) { @@ -541,16 +549,43 @@ static apt_bool_t pocketsphinx_recognize(pocketsphinx_recognizer_t *recognizer, if(mrcp_resource_header_property_check(request,RECOGNIZER_HEADER_START_INPUT_TIMERS) == TRUE) { recognizer->is_input_timer_on = request_recog_header->start_input_timers; } + if(mrcp_resource_header_property_check(request,RECOGNIZER_HEADER_NO_INPUT_TIMEOUT) == TRUE) { + recognizer->properties.no_input_timeout = request_recog_header->no_input_timeout; + } + if(mrcp_resource_header_property_check(request,RECOGNIZER_HEADER_RECOGNITION_TIMEOUT) == TRUE) { + recognizer->properties.recognition_timeout = request_recog_header->recognition_timeout; + } + if(mrcp_resource_header_property_check(request,RECOGNIZER_HEADER_SAVE_WAVEFORM) == TRUE) { + recognizer->properties.save_waveform = request_recog_header->save_waveform; + } + } + + /* check if waveform (utterance) should be saved */ + if(recognizer->properties.save_waveform == TRUE) { + apr_status_t rv; + const char *waveform_file_name = apr_psprintf(channel->pool,"utter-%s-%d.pcm", + channel->id.buf,request->start_line.request_id); + char *waveform_file_path = NULL; + apr_filepath_merge(&waveform_file_path,recognizer->properties.save_waveform_dir, + waveform_file_name,0,channel->pool); + + apt_log(APT_LOG_MARK,APT_PRIO_INFO,"Open Waveform File [%s] "APT_SIDRES_FMT, + waveform_file_path,RECOGNIZER_SIDRES(recognizer)); + rv = apr_file_open(&recognizer->waveform,waveform_file_path,APR_CREATE|APR_TRUNCATE|APR_WRITE|APR_BINARY, + APR_OS_DEFAULT,channel->pool); + if(rv != APR_SUCCESS) { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Cannot Open Waveform File to Write [%s] "APT_SIDRES_FMT, + waveform_file_path,RECOGNIZER_SIDRES(recognizer)); + } } response->start_line.request_state = MRCP_REQUEST_STATE_INPROGRESS; /* send asynchronous response */ - mrcp_engine_channel_message_send(recognizer->channel,response); - + mrcp_engine_channel_message_send(channel,response); /* reset */ mpf_activity_detector_reset(recognizer->detector); - recognizer->noinput_timeout = 0; + recognizer->no_input_timeout = 0; recognizer->recognition_timeout = 0; recognizer->partial_result_timeout = 0; recognizer->last_result = NULL; @@ -560,7 +595,7 @@ static apt_bool_t pocketsphinx_recognize(pocketsphinx_recognizer_t *recognizer, return TRUE; } -/** Process GET-RESULTS request [RECOG] */ +/** Process GET-RESULT request [RECOG] */ static apt_bool_t pocketsphinx_get_result(pocketsphinx_recognizer_t *recognizer, mrcp_message_t *request, mrcp_message_t *response) { if(pocketsphinx_result_build(recognizer,response) != TRUE) { @@ -609,6 +644,11 @@ static apt_bool_t pocketsphinx_recognition_complete(pocketsphinx_recognizer_t *r recognizer->inprogress_recog = NULL; ps_end_utt(recognizer->decoder); + if(recognizer->waveform) { + apr_file_close(recognizer->waveform); + recognizer->waveform = NULL; + } + if(recognizer->stop_response) { /* recognition has been stopped, send STOP response instead */ mrcp_message_t *response = recognizer->stop_response; @@ -690,13 +730,10 @@ static apt_bool_t pocketsphinx_request_dispatch(pocketsphinx_recognizer_t *recog static void* APR_THREAD_FUNC pocketsphinx_recognizer_run(apr_thread_t *thread, void *data) { pocketsphinx_recognizer_t *recognizer = data; - apt_bool_t status; apt_log(APT_LOG_MARK,APT_PRIO_INFO,"Run Recognition Thread "APT_SIDRES_FMT, RECOGNIZER_SIDRES(recognizer)); - status = pocketsphinx_properties_load(recognizer); - /** Send response to channel_open request */ - mrcp_engine_channel_open_respond(recognizer->channel,status); + mrcp_engine_channel_open_respond(recognizer->channel,TRUE); do { /** Wait for MRCP requests */ @@ -760,7 +797,7 @@ static apt_bool_t pocketsphinx_start_of_input(pocketsphinx_recognizer_t *recogni /* set request state */ message->start_line.request_state = MRCP_REQUEST_STATE_INPROGRESS; - /* send asynch event */ + /* send asynchronous event */ return mrcp_engine_channel_message_send(recognizer->channel,message); } @@ -811,6 +848,12 @@ static apt_bool_t pocketsphinx_stream_write(mpf_audio_stream_t *stream, const mp return TRUE; } + if(recognizer->waveform) { + /* write utterance to file */ + apr_size_t size = frame->codec_frame.size; + apr_file_write(recognizer->waveform,frame->codec_frame.buffer,&size); + } + if(ps_process_raw( recognizer->decoder, (const int16 *)frame->codec_frame.buffer, @@ -835,13 +878,18 @@ static apt_bool_t pocketsphinx_stream_write(mpf_audio_stream_t *stream, const mp recognizer->last_result = apr_pstrdup(recognizer->channel->pool,hyp); apt_log(APT_LOG_MARK,APT_PRIO_INFO,"Get Recognition Partial Result [%s] Score [%d] "APT_SIDRES_FMT, hyp,score,RECOGNIZER_SIDRES(recognizer)); + + /* reset input timer as we have partial match now */ + if(score != 0 && recognizer->is_input_timer_on) { + recognizer->is_input_timer_on = FALSE; + } } } } if(recognizer->is_input_timer_on) { - recognizer->noinput_timeout += CODEC_FRAME_TIME_BASE; - if(recognizer->noinput_timeout == recognizer->properties.noinput_timeout) { + recognizer->no_input_timeout += CODEC_FRAME_TIME_BASE; + if(recognizer->no_input_timeout == recognizer->properties.no_input_timeout) { apt_log(APT_LOG_MARK,APT_PRIO_INFO,"Noinput Timeout Elapsed "APT_SIDRES_FMT, RECOGNIZER_SIDRES(recognizer)); pocketsphinx_end_of_input(recognizer,RECOGNIZER_COMPLETION_CAUSE_NO_INPUT_TIMEOUT); diff --git a/libs/unimrcp/plugins/mrcp-pocketsphinx/src/pocketsphinx_properties.c b/libs/unimrcp/plugins/mrcp-pocketsphinx/src/pocketsphinx_properties.c new file mode 100644 index 0000000000..2b8fe94e1c --- /dev/null +++ b/libs/unimrcp/plugins/mrcp-pocketsphinx/src/pocketsphinx_properties.c @@ -0,0 +1,209 @@ +/* + * Copyright 2008 Arsen Chaloyan + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "pocketsphinx_properties.h" +#include "apt_log.h" + +static const apr_xml_elem* pocketsphinx_document_load(const char *file_path, apr_pool_t *pool) +{ + apr_xml_parser *parser = NULL; + apr_xml_doc *doc = NULL; + const apr_xml_elem *root; + apr_file_t *fd = NULL; + apr_status_t rv; + + apt_log(APT_LOG_MARK,APT_PRIO_DEBUG,"Open PocketSphinx Config File [%s]",file_path); + rv = apr_file_open(&fd,file_path,APR_READ|APR_BINARY,0,pool); + if(rv != APR_SUCCESS) { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Failed to Open PocketSphinx Config File [%s]",file_path); + return FALSE; + } + + rv = apr_xml_parse_file(pool,&parser,&doc,fd,2000); + if(rv != APR_SUCCESS) { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Failed to Parse PocketSphinx Config File [%s]",file_path); + apr_file_close(fd); + return FALSE; + } + + root = doc->root; + if(!root || strcasecmp(root->name,"pocketsphinx") != 0) { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Unknown Document <%s>",root->name); + apr_file_close(fd); + return FALSE; + } + + apr_file_close(fd); + return root; +} + +static apt_bool_t sensitivity_properties_load(pocketsphinx_properties_t *properties, const apr_xml_elem *elem, apr_pool_t *pool) +{ + const apr_xml_attr *attr; + for(attr = elem->attr; attr; attr = attr->next) { + if(strcasecmp(attr->name,"level") == 0) { + properties->sensitivity_level = atol(attr->value); + } + else if(strcasecmp(attr->name,"timeout") == 0) { + properties->sensitivity_timeout = atol(attr->value); + } + else { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Unknown Attribute <%s>",attr->name); + } + } + return TRUE; +} + +static apt_bool_t timer_properties_load(pocketsphinx_properties_t *properties, const apr_xml_elem *elem, apr_pool_t *pool) +{ + const apr_xml_attr *attr; + for(attr = elem->attr; attr; attr = attr->next) { + if(strcasecmp(attr->name,"noinput-timeout") == 0) { + properties->no_input_timeout = atol(attr->value); + } + else if(strcasecmp(attr->name,"recognition-timeout") == 0) { + properties->recognition_timeout = atol(attr->value); + } + else { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Unknown Attribute <%s>",attr->name); + } + } + return TRUE; +} + +static apt_bool_t model_properties_load(pocketsphinx_properties_t *properties, const apr_xml_elem *elem, apr_pool_t *pool) +{ + const apr_xml_attr *attr; + for(attr = elem->attr; attr; attr = attr->next) { + if(strcasecmp(attr->name,"dir") == 0) { + properties->data_dir = apr_pstrdup(pool,attr->value); + } + else if(strcasecmp(attr->name,"narrowband") == 0) { + properties->model_8k = apr_pstrdup(pool,attr->value); + } + else if(strcasecmp(attr->name,"wideband") == 0) { + properties->model_16k = apr_pstrdup(pool,attr->value); + } + else if(strcasecmp(attr->name,"dictionary") == 0) { + properties->dictionary = apr_pstrdup(pool,attr->value); + } + else if(strcasecmp(attr->name,"preferred") == 0) { + if(strcasecmp(attr->value,"narrowband") == 0) { + properties->preferred_model = POCKETSPHINX_MODEL_NARROWBAND; + } + else if(strcasecmp(attr->value,"wideband") == 0) { + properties->preferred_model = POCKETSPHINX_MODEL_WIDEBAND; + } + } + else { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Unknown Attribute <%s>",attr->name); + } + } + return TRUE; +} + +static apt_bool_t save_waveform_properties_load(pocketsphinx_properties_t *properties, const apr_xml_elem *elem, apr_pool_t *pool) +{ + const apr_xml_attr *attr; + for(attr = elem->attr; attr; attr = attr->next) { + if(strcasecmp(attr->name,"dir") == 0) { + properties->save_waveform_dir = apr_pstrdup(pool,attr->value); + } + else if(strcasecmp(attr->name,"enable") == 0) { + properties->save_waveform = atoi(attr->value); + } + else { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Unknown Attribute <%s>",attr->name); + } + } + return TRUE; +} + +apt_bool_t pocketsphinx_properties_load(pocketsphinx_properties_t *properties, + const char *file_path, + const apt_dir_layout_t *dir_layout, + apr_pool_t *pool) +{ + const apr_xml_elem *elem; + const apr_xml_elem *root; + char *path = NULL; + + /* reset or set default properties */ + properties->data_dir = NULL; + properties->dictionary = NULL; + properties->model_8k = NULL; + properties->model_16k = NULL; + properties->preferred_model = POCKETSPHINX_MODEL_NARROWBAND; + + properties->no_input_timeout = 10000; + properties->recognition_timeout = 15000; + properties->partial_result_timeout = 100; + + properties->save_waveform = TRUE; + properties->save_waveform_dir = NULL; + + root = pocketsphinx_document_load(file_path,pool); + if(root) { + for(elem = root->first_child; elem; elem = elem->next) { + if(strcasecmp(elem->name,"sensitivity") == 0) { + sensitivity_properties_load(properties,elem,pool); + } + else if(strcasecmp(elem->name,"timers") == 0) { + timer_properties_load(properties,elem,pool); + } + else if(strcasecmp(elem->name,"model") == 0) { + model_properties_load(properties,elem,pool); + } + else if(strcasecmp(elem->name,"save-waveform") == 0) { + save_waveform_properties_load(properties,elem,pool); + } + else { + apt_log(APT_LOG_MARK,APT_PRIO_WARNING,"Unknown Element <%s>",elem->name); + } + } + } + + /* verify loaded properties */ + if(!properties->data_dir || *properties->data_dir == '\0') { + properties->data_dir = dir_layout->data_dir_path; + } + if(!properties->save_waveform_dir || *properties->save_waveform_dir == '\0') { + properties->save_waveform_dir = dir_layout->data_dir_path; + } + + if(!properties->dictionary) { + properties->dictionary = "default.dic"; + } + if(!properties->model_8k) { + properties->model_8k = "communicator"; + } + if(!properties->model_16k) { + properties->model_16k = "wsj1"; + } + + if(apr_filepath_merge(&path,properties->data_dir,properties->dictionary,0,pool) == APR_SUCCESS) { + properties->dictionary = path; + } + if(apr_filepath_merge(&path,properties->data_dir,properties->model_8k,0,pool) == APR_SUCCESS) { + properties->model_8k = path; + } + if(apr_filepath_merge(&path,properties->data_dir,properties->model_16k,0,pool) == APR_SUCCESS) { + properties->model_16k = path; + } + + return TRUE; +} diff --git a/libs/unimrcp/unimrcp.sln b/libs/unimrcp/unimrcp.sln index b566bef852..421ed16929 100644 --- a/libs/unimrcp/unimrcp.sln +++ b/libs/unimrcp/unimrcp.sln @@ -148,6 +148,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mrcpflite", "plugins\mrcp-f {843425BE-9A9A-44F4-A4E3-4B57D6ABD53C} = {843425BE-9A9A-44F4-A4E3-4B57D6ABD53C} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "preparesphinx", "build\tools\preparesphinx.vcproj", "{71D62A04-8EF6-4C6B-AC12-0C15A875E53A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -252,6 +254,8 @@ Global {3C614AE8-B611-4D43-A9AF-1CAA440A9F69}.Release|Win32.ActiveCfg = Release|Win32 {56F6FB96-2BC7-4CAE-A8BF-6A0FAEC90556}.Debug|Win32.ActiveCfg = Debug|Win32 {56F6FB96-2BC7-4CAE-A8BF-6A0FAEC90556}.Release|Win32.ActiveCfg = Release|Win32 + {71D62A04-8EF6-4C6B-AC12-0C15A875E53A}.Debug|Win32.ActiveCfg = Debug|Win32 + {71D62A04-8EF6-4C6B-AC12-0C15A875E53A}.Release|Win32.ActiveCfg = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -284,5 +288,6 @@ Global {17A33F3F-BAF5-403F-8EF4-FECDA7D9A335} = {AC4356E8-48A1-4D2D-AFB1-11CF30B974CD} {01D63BF5-7798-4746-852A-4B45229BB735} = {62083CC3-13BF-49EA-BFE8-4C9337C0D82C} {4714EF49-BFD5-4B22-95F7-95A07F1EAC25} = {62083CC3-13BF-49EA-BFE8-4C9337C0D82C} + {71D62A04-8EF6-4C6B-AC12-0C15A875E53A} = {62083CC3-13BF-49EA-BFE8-4C9337C0D82C} EndGlobalSection EndGlobal