diff --git a/Makefile.am b/Makefile.am index ef5747e124..7a6f3b7bd6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -286,6 +286,7 @@ library_include_HEADERS = \ src/include/switch_utils.h \ src/include/switch_rtp.h \ src/include/switch_jitterbuffer.h \ + src/include/switch_estimators.h \ src/include/switch_rtcp_frame.h \ src/include/switch_stun.h \ src/include/switch_nat.h \ @@ -351,6 +352,7 @@ libfreeswitch_la_SOURCES = \ src/switch_regex.c \ src/switch_rtp.c \ src/switch_jitterbuffer.c \ + src/switch_estimators.c \ src/switch_ivr_bridge.c \ src/switch_ivr_originate.c \ src/switch_ivr_async.c \ diff --git a/src/include/switch.h b/src/include/switch.h index d612532612..e38846d6eb 100644 --- a/src/include/switch.h +++ b/src/include/switch.h @@ -144,6 +144,7 @@ #include "switch_core_media.h" #include "switch_core_video.h" #include "switch_jitterbuffer.h" +#include "switch_estimators.h" #include diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index 096bc815fc..6e4c44af2f 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -322,6 +322,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess switch_codec_control_type_t *rtype, void **ret_data); +SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_session_t *session, + switch_media_type_t mtype, + switch_codec_flag_t flag); + #define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \ SCC_VIDEO_GEN_KEYFRAME, SCCT_NONE, NULL, SCCT_NONE, NULL, NULL, NULL) \ diff --git a/src/include/switch_estimators.h b/src/include/switch_estimators.h new file mode 100644 index 0000000000..a8c92c3f02 --- /dev/null +++ b/src/include/switch_estimators.h @@ -0,0 +1,100 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2015, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Dragos Oancea + * + * switch_estimators.h -- Estimators for Packet Loss, Jitter, RTT , etc + * + */ + + +#ifndef SWITCH_ESTIMATORS_H +#define SWITCH_ESTIMATORS_H + + +#include + + +SWITCH_BEGIN_EXTERN_C + +struct kalman_estimator_s { + /* initial values for the Kalman filter */ + float val_estimate_last ; + float P_last ; + /* the noise in the system: + The amount of noise in your measurements and the state-transitions + (e.g. the standard deviation of the signal noise, and how 'wrong' your simplified model + of the state-transitions are) => These are Q and R matrices */ + float Q ; /* the process noise covariance matrix */ + float R ; /* the measurement noise covariance matrix */ + float K; /* P_temp * H^T * (H* P_temp * H^T + R)^-1 */ + float P; /* the Kalman gain (calculated) */ + float val_estimate; /* x_temp_est + K * (z_measured - H * x_temp_est) */ + float val_measured; /* the 'noisy' value we measured */ +}; + +struct cusum_kalman_detector_s { + /* initial values for the CUSUM Kalman filter */ + float val_estimate_last; + float val_desired_last; + float P_last; + float K_last; + float delta; + float measurement_noise_e; + float variance_Re; + float measurement_noise_v; + float variance_Rv; + float g_last; + /*constants per model*/ + float epsilon; + float h; + /* for calculating variance */ + float last_average; + float last_q; + float N; /*how many samples we have so far (eg: how many RTCP we received, granted that we can calculate RTT for each one of them)*/ +}; + +typedef struct kalman_estimator_s kalman_estimator_t; +typedef struct cusum_kalman_detector_s cusum_kalman_detector_t; + +SWITCH_DECLARE(void) switch_kalman_init(kalman_estimator_t *est, float Q, float R); +SWITCH_DECLARE(switch_bool_t) switch_kalman_cusum_init(cusum_kalman_detector_t *detect_change, float epsilon,float h); +SWITCH_DECLARE(switch_bool_t) switch_kalman_estimate(kalman_estimator_t * est, float measurement, int system_model); +SWITCH_DECLARE (switch_bool_t) switch_kalman_cusum_detect_change(cusum_kalman_detector_t * detector, float measurement, float rtt_avg); +SWITCH_DECLARE(switch_bool_t) switch_kalman_is_slow_link(kalman_estimator_t * est_loss, kalman_estimator_t * est_rtt); + +SWITCH_END_EXTERN_C +#endif + +/* For Emacs: + * Local Variables: + * mode:c + * indent-tabs-mode:t + * tab-width:4 + * c-basic-offset:4 + * End: + * For VIM: + * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet: + */ diff --git a/src/include/switch_types.h b/src/include/switch_types.h index cc8539085e..b31366f9b5 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -755,6 +755,8 @@ typedef enum { SWITCH_ZRTP_FLAG_SECURE_MITM_RECV, SWITCH_RTP_FLAG_DEBUG_RTP_READ, SWITCH_RTP_FLAG_DEBUG_RTP_WRITE, + SWITCH_RTP_FLAG_ESTIMATORS, + SWITCH_RTP_FLAG_ADJ_BITRATE_CAP, SWITCH_RTP_FLAG_VIDEO, SWITCH_RTP_FLAG_ENABLE_RTCP, SWITCH_RTP_FLAG_RTCP_MUX, @@ -1631,6 +1633,7 @@ typedef enum { SWITCH_CODEC_FLAG_AAL2 = (1 << 6), SWITCH_CODEC_FLAG_PASSTHROUGH = (1 << 7), SWITCH_CODEC_FLAG_READY = (1 << 8), + SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE = (1 << 14), SWITCH_CODEC_FLAG_HAS_PLC = (1 << 15), SWITCH_CODEC_FLAG_VIDEO_PATCHING = (1 << 16) } switch_codec_flag_enum_t; @@ -2268,6 +2271,7 @@ typedef enum { SCC_VIDEO_BANDWIDTH, SCC_VIDEO_RESET, SCC_AUDIO_PACKET_LOSS, + SCC_AUDIO_ADJUST_BITRATE, SCC_DEBUG, SCC_CODEC_SPECIFIC } switch_codec_control_command_t; @@ -2586,6 +2590,12 @@ typedef struct switch_waitlist_s { struct switch_jb_s; typedef struct switch_jb_s switch_jb_t; +//struct kalman_estimator_s; +//typedef struct kalman_estimator_s kalman_estimator_t; + +//struct cusum_kalman_detector_s; +//typedef struct cusum_kalman_detector_s cusum_kalman_detector_t; + struct switch_img_txt_handle_s; typedef struct switch_img_txt_handle_s switch_img_txt_handle_t; diff --git a/src/mod/codecs/mod_opus/mod_opus.c b/src/mod/codecs/mod_opus/mod_opus.c index 959311b58b..4a46c3f1e3 100644 --- a/src/mod/codecs/mod_opus/mod_opus.c +++ b/src/mod/codecs/mod_opus/mod_opus.c @@ -34,6 +34,11 @@ #include "switch.h" #include "opus.h" +#define SWITCH_OPUS_MIN_BITRATE 6000 +#define SWITCH_OPUS_MAX_BITRATE 510000 + +#define SWITCH_OPUS_MIN_FEC_BITRATE 12400 + SWITCH_MODULE_LOAD_FUNCTION(mod_opus_load); SWITCH_MODULE_DEFINITION(mod_opus, mod_opus_load, NULL, NULL); @@ -91,6 +96,15 @@ struct dec_stats { }; typedef struct dec_stats dec_stats_t; +struct codec_control_state { + int keep_fec; + opus_int32 current_bitrate; + opus_int32 wanted_bitrate; + uint32_t increase_step; + uint32_t decrease_step; +}; +typedef struct codec_control_state codec_control_state_t; + struct opus_context { OpusEncoder *encoder_object; OpusDecoder *decoder_object; @@ -103,6 +117,7 @@ struct opus_context { int look_check; int look_ts; dec_stats_t decoder_stats; + codec_control_state_t control_state; }; struct { @@ -116,6 +131,7 @@ struct { int asymmetric_samplerates; int keep_fec; int fec_decode; + int adjust_bitrate; int debuginfo; uint32_t use_jb_lookahead; switch_mutex_t *mutex; @@ -253,7 +269,7 @@ static switch_status_t switch_opus_fmtp_parse(const char *fmtp, switch_codec_fmt if (!strcasecmp(data, "maxaveragebitrate")) { codec_settings->maxaveragebitrate = atoi(arg); - if (codec_settings->maxaveragebitrate < 6000 || codec_settings->maxaveragebitrate > 510000) { + if (codec_settings->maxaveragebitrate < SWITCH_OPUS_MIN_BITRATE || codec_settings->maxaveragebitrate > SWITCH_OPUS_MAX_BITRATE) { codec_settings->maxaveragebitrate = 0; /* values outside the range between 6000 and 510000 SHOULD be ignored */ } } @@ -600,6 +616,7 @@ static switch_status_t switch_opus_init(switch_codec_t *codec, switch_codec_flag opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(fec_bitrate)); /* will override the maxaveragebitrate set in opus.conf.xml */ opus_codec_settings.maxaveragebitrate = fec_bitrate; + context->control_state.keep_fec = opus_prefs.keep_fec ; } } } @@ -607,6 +624,10 @@ static switch_status_t switch_opus_init(switch_codec_t *codec, switch_codec_flag if (opus_codec_settings.usedtx) { opus_encoder_ctl(context->encoder_object, OPUS_SET_DTX(opus_codec_settings.usedtx)); } + + if (opus_prefs.adjust_bitrate) { + switch_set_flag(codec, SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE); + } } if (decoding) { @@ -972,9 +993,11 @@ static switch_status_t opus_load_config(switch_bool_t reload) opus_prefs.keep_fec = atoi(val); } else if (!strcasecmp(key, "advertise-useinbandfec")) { /*decoder, has meaning only for FMTP: useinbandfec=1 by default */ opus_prefs.fec_decode = atoi(val); + } else if (!strcasecmp(key, "adjust-bitrate")) { /* encoder, this setting will make the encoder adjust its bitrate based on a feedback loop (RTCP). This is not "VBR".*/ + opus_prefs.adjust_bitrate = atoi(val); } else if (!strcasecmp(key, "maxaveragebitrate")) { opus_prefs.maxaveragebitrate = atoi(val); - if (opus_prefs.maxaveragebitrate < 6000 || opus_prefs.maxaveragebitrate > 510000) { + if (opus_prefs.maxaveragebitrate < SWITCH_OPUS_MIN_BITRATE || opus_prefs.maxaveragebitrate > SWITCH_OPUS_MAX_BITRATE) { opus_prefs.maxaveragebitrate = 0; /* values outside the range between 6000 and 510000 SHOULD be ignored */ } } else if (!strcasecmp(key, "maxplaybackrate")) { @@ -1132,13 +1155,68 @@ static switch_status_t switch_opus_control(switch_codec_t *codec, } if (globals.debug || context->debug) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus Adjusting packet loss percent from %d%% to %d%%!\n", + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting packet loss percent from %d%% to %d%%!\n", context->old_plpct, plpct); } } context->old_plpct = plpct; } break; + case SCC_AUDIO_ADJUST_BITRATE: + { + const char *cmd = (const char *)cmd_data; + + if (!zstr(cmd)) { + opus_int32 current_bitrate=context->control_state.current_bitrate; + if (!strcasecmp(cmd, "increase")) { + /* https://wiki.xiph.org/OpusFAQ + "[...]Opus scales from about 6 to 512 kb/s, in increments of 0.4 kb/s (one byte with 20 ms frames). + Opus can have more than 1200 possible bitrates[...]" */ + int br_step = context->control_state.increase_step?context->control_state.increase_step:400; + opus_encoder_ctl(context->encoder_object, OPUS_GET_BITRATE(¤t_bitrate)); + if (opus_prefs.maxaveragebitrate > current_bitrate) { + opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(current_bitrate+br_step)); + if ((context->control_state.keep_fec) && (current_bitrate > SWITCH_OPUS_MIN_FEC_BITRATE)) { + opus_prefs.keep_fec = 1; /* enable back FEC if it was disabled by SCC_AUDIO_ADJUST_BITRATE, we have enough network bandwidth now */ + } + if (globals.debug || context->debug) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (increase)\n", current_bitrate+br_step); + } + } + } else if (!strcasecmp(cmd, "decrease")) { + int br_step = context->control_state.decrease_step?context->control_state.decrease_step:400; + opus_encoder_ctl(context->encoder_object, OPUS_GET_BITRATE(¤t_bitrate)); + if (current_bitrate > SWITCH_OPUS_MIN_BITRATE) { + if ((context->control_state.keep_fec) && (current_bitrate < SWITCH_OPUS_MIN_FEC_BITRATE)) { + opus_prefs.keep_fec = 0; /* no point to try to keep FEC enabled anymore, we're low on network bandwidth (that's why we ended up here) */ + } + opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(current_bitrate-br_step)); + if (globals.debug || context->debug) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (decrease)\n", current_bitrate-br_step); + } + } + } else if (!strcasecmp(cmd, "default")) { + /*restore default bitrate */ + opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(opus_prefs.maxaveragebitrate)); + if (context->control_state.keep_fec) { + opus_prefs.keep_fec = 1; /* enable back FEC, we have enough network bandwidth now */ + } + if (globals.debug || context->debug) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (configured maxaveragebitrate)\n", opus_prefs.maxaveragebitrate); + } + } else { + /* set Opus minimum bitrate */ + opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(SWITCH_OPUS_MIN_BITRATE)); + if (context->control_state.keep_fec) { + opus_prefs.keep_fec = 0; /* do not enforce FEC anymore, we're low on network bandwidth */ + } + if (globals.debug || context->debug) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (minimum)\n", SWITCH_OPUS_MIN_BITRATE); + } + } + } + } + break; default: break; } diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 82be0945bd..5fb25f5772 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -11322,6 +11322,35 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess return SWITCH_STATUS_FALSE; } +SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_session_t *session, + switch_media_type_t mtype, + switch_codec_flag_t flag) { + switch_rtp_engine_t *engine = NULL; + switch_media_handle_t *smh = NULL; + switch_codec_t *codec = NULL; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return SWITCH_FALSE; + } + + if (!(engine = &smh->engines[mtype])) { + return SWITCH_FALSE; + } + + codec = &engine->write_codec; + + if (!switch_core_codec_ready(codec)) { + return SWITCH_FALSE; + } + + if (switch_test_flag(codec, flag)){ + return SWITCH_TRUE; + } + + return SWITCH_FALSE; +} SWITCH_DECLARE(switch_status_t) switch_core_session_write_encoded_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, int stream_id) diff --git a/src/switch_estimators.c b/src/switch_estimators.c new file mode 100644 index 0000000000..3effb6cfa7 --- /dev/null +++ b/src/switch_estimators.c @@ -0,0 +1,263 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2015, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Dragos Oancea + * + * switch_estimators.c -- Estimators and Detectors (try to read into the future: packet loss, jitter, RTT, etc) + * + */ + +#include + +#include +#ifndef _MSC_VER +#include +#endif +#undef PACKAGE_NAME +#undef PACKAGE_STRING +#undef PACKAGE_TARNAME +#undef PACKAGE_VERSION +#undef PACKAGE_BUGREPORT +#undef VERSION +#undef PACKAGE +#undef inline +#include +#include + +#define KALMAN_SYSTEM_MODELS 3 /*loss, jitter, rtt*/ +#define EST_LOSS 0 +#define EST_JITTER 1 +#define EST_RTT 2 + +/* This function initializes the Kalman System Model + * + * xk+1 = A*xk + wk + * zk = H*xk + vk + * xk = state variable (must exist in physical world - measurable ) + * zk = measurment + * wk,vk - white noise + * A = state trasition matrix , (n x n ) matrix + * H = state-to-measurment matrix , ( n x n ) matrix + * Noise covariance: + * Q: Covariance matrix of wk, ( n x n ) diagonal matrix + * R: Covariance matrix of vk , ( m x m ) diagonal matrix + * R: if you want to be affected less by the measurement and get the estimate with less variation, increase R + * Q: if you want to be affected more by the measurement and get the estimate with more variation, decrease Q + * + * (Phil Kim book) + * + */ +void switch_kalman_init(kalman_estimator_t *est, float Q, float R) +{ + est -> val_estimate_last = 0 ; + est -> P_last = 0; + est -> Q = Q; /*accuracy of system model */ /* SYSTEM MODEL: TO BE DEDUCTED */ + est -> R = R; /*accuracy of measurement*/ /* SYSTEM MODEL: TO BE DEDUCTED */ + est -> K = 0; + est -> val_estimate = 0 ; + est -> val_measured = 0 ; // [0-100 %] or [0-5000] or [0-2sec] +} + +/* +CUSUM Kalman functions to detect sudden change over a predefined thereshold. + +y(t) = sampled RTT +x(t)= desired RTT + +Model: +x(t+1) = x(t) + delta(t)*v(t) +y(t) = x(t) + e(t) + +Noisy characteristic of RTT captured by measurment noise e(t) with variance Re. +The step changes in the desired RTT x(t) is modeled as the process noise v(t) +with variance Rv and the discrete variable delta(t) . +If a change occurs at time t, then delta(t) = 1 otherwise delta(t) = 0. + +avg(x(t)) = avg(x(t-1)) + K(t)(y(t) - avg(x(t-1))) +K(t) = P(t-1)/(P(t-1) + Re)) +P(t) = (1-K(t))P(t-1) + delta(t-1)* Rv +e(t) = y(t) - avg(x(t)) +g(t) = max(g(t-1) + e(t) - epsilon,0) +if g(t) > 0 then + delta(t) = 1 // alarm + g(t) = 0 +else + delta(t) = 0 +endif + +constants: + +epsilon = 0.005 +h = 0.05 +*/ +switch_bool_t switch_kalman_cusum_init (cusum_kalman_detector_t *detect_change, float epsilon, float h) +{ + cusum_kalman_detector_t *detector_change = detect_change; + + + if (epsilon < 0 || h < 0) { + return FALSE; + } + + detector_change -> val_estimate_last = 0; + detector_change -> val_desired_last = 0; + detector_change -> P_last = 0; + detector_change -> K_last = 0; + detector_change -> delta = 0; + detector_change -> measurement_noise_e = 0; + detector_change -> variance_Re = 0; + detector_change -> measurement_noise_v = 0; + detector_change -> variance_Rv = 0; + detector_change -> g_last = 0; + /*per system model*/ + detector_change -> epsilon = epsilon; + detector_change -> h = h; + /*variance*/ + detector_change -> last_average = 0; + detector_change -> last_q = 0; + detector_change -> N = 0; + return TRUE; +} + +switch_bool_t switch_kalman_cusum_detect_change(cusum_kalman_detector_t * detector, float measurement, float avg) +{ + float K=0; + float P=0; + float g=0; + float desired_val; + float current_average; + float current_q; + float sample_variance_Re = 0; + + /*variance*/ + + detector->N++; + current_average = detector->last_average + (measurement - detector->last_average)/detector->N ; + if (avg > current_average) { + current_average = avg; + } + current_q = detector-> last_q + (measurement - detector->last_average) * (measurement - current_average); + if (detector->N != 0) + sample_variance_Re = sqrt(current_q/detector->N); + + detector->variance_Re = sample_variance_Re; + detector->variance_Rv = sample_variance_Re; + + if (sample_variance_Re != 0) { + K = detector->P_last / (detector->P_last + detector->variance_Re); + desired_val = detector->val_desired_last + K * (measurement - detector->variance_Re); + P = (1 - K) * detector->P_last + detector->delta * detector->variance_Rv; + detector->measurement_noise_e = measurement - desired_val; + g = detector->g_last + detector->measurement_noise_e - detector->epsilon; + if (g > detector->h) { + detector->delta = 1; + g = 0; + } else { + detector->delta = 0; + } + + /* update last vals for calculating variance */ + detector->last_average = current_average; + /* update lasts (cusum)*/ + detector -> g_last = g; + detector -> P_last = P; + detector -> val_desired_last = desired_val; + } + if (detector->delta == 1) { + return TRUE; + } + return FALSE; +} + +/* Kalman filter abstract ( measure and estimate 1 single value per system model ) + * Given the measurment and the system model together with the current state , + * the function puts an estimate in the estimator struct */ +switch_bool_t switch_kalman_estimate (kalman_estimator_t * est, float measurement, int system_model) +{ + /*system model can be about: loss, jitter, rtt*/ + float val_estimate; + float val_temp_est = est->val_estimate_last; + float P_temp = est->P_last + est->Q; + + if (system_model >= KALMAN_SYSTEM_MODELS) { + return SWITCH_FALSE ; + } + + /*sanitize input a little bit, just in case */ + if (system_model == EST_LOSS ) { + if ((measurement > 100) && (measurement < 0)) { + return SWITCH_FALSE ; + } + } + + if (system_model == EST_JITTER) { + if ((measurement > 10000) && (measurement < 0)) { + return SWITCH_FALSE; + } + } + + if (system_model == EST_RTT) { + if ((measurement > 2 ) && (measurement < 0)) { + return SWITCH_FALSE; + } + } + + /* calculate the Kalman gain */ + est->K = P_temp * (1.0/(P_temp + est->R)); + /* real life measurement */ + est->val_measured = measurement ; + val_estimate = val_temp_est + est->K * (est->val_measured - val_temp_est); + est->P = (1 - est->K) * P_temp; + /*update lasts*/ + est->P_last = est->P; + /* save the estimated value (future) */ + est->val_estimate_last = val_estimate; + return SWITCH_TRUE; +} + +switch_bool_t switch_kalman_is_slow_link(kalman_estimator_t * est_loss, kalman_estimator_t * est_rtt) +{ + float thresh_packet_loss = 5; /* % */ + float thresh_rtt = 0.8 ; /*seconds*/ + + if ((est_loss->val_estimate_last > thresh_packet_loss) && + (est_rtt->val_estimate_last > thresh_rtt )) { + return SWITCH_TRUE; + } + + return SWITCH_FALSE; +} + +/* For Emacs: + * Local Variables: + * mode:c + * indent-tabs-mode:t + * tab-width:4 + * c-basic-offset:4 + * End: + * For VIM: + * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet: + */ + diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 398d3233c9..697c302d4a 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -35,6 +35,7 @@ //#define DEBUG_MISSED_SEQ //#define DEBUG_EXTRA //#define DEBUG_RTCP +#define DEBUG_ESTIMATORS #include #ifndef _MSC_VER @@ -55,6 +56,8 @@ #include #include #include +#include + #define JITTER_LEAD_FRAMES 10 #define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++ @@ -170,6 +173,10 @@ typedef struct { #pragma pack(pop, r1) #endif +#define KALMAN_SYSTEM_MODELS 3 /*loss, jitter, rtt*/ +#define EST_LOSS 0 +#define EST_JITTER 1 +#define EST_RTT 2 typedef struct { switch_rtcp_ext_hdr_t header; @@ -446,6 +453,8 @@ struct switch_rtp { switch_core_session_t *session; payload_map_t **pmaps; payload_map_t *pmap_tail; + kalman_estimator_t *estimators[KALMAN_SYSTEM_MODELS]; + cusum_kalman_detector_t *detectors[KALMAN_SYSTEM_MODELS]; int ice_adj; uint8_t has_rtp; uint8_t has_rtcp; @@ -1860,6 +1869,24 @@ static void rtcp_stats_init(switch_rtp_t *rtp_session) } else { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "rtcp_stats_init: %s ssrc[%u] base_seq[%u]\n", rtp_type(rtp_session), stats->ssrc, stats->base_seq); } + + if (rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] && (switch_core_media_codec_get_cap(rtp_session->session, + SWITCH_MEDIA_TYPE_AUDIO, SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE))) { + kalman_estimator_t *estimators[KALMAN_SYSTEM_MODELS]; + cusum_kalman_detector_t *detectors[KALMAN_SYSTEM_MODELS]; + + rtp_session->flags[SWITCH_RTP_FLAG_ADJ_BITRATE_CAP] = 1; + rtp_session->flags[SWITCH_RTP_FLAG_ESTIMATORS] = 1; + + rtp_session->estimators[EST_LOSS] = switch_core_alloc(rtp_session->pool, sizeof(*estimators[0])); + switch_kalman_init(rtp_session->estimators[EST_LOSS],0.1,0.1); + rtp_session->estimators[EST_RTT] = switch_core_alloc(rtp_session->pool, sizeof(*estimators[0])); + switch_kalman_init(rtp_session->estimators[EST_RTT],0.03,1); + rtp_session->detectors[EST_RTT] = switch_core_alloc(rtp_session->pool, sizeof(*detectors[0])); + switch_kalman_cusum_init(rtp_session->detectors[EST_RTT],0.005,0.5); + rtp_session->detectors[EST_LOSS] = switch_core_alloc(rtp_session->pool, sizeof(*detectors[0])); + switch_kalman_cusum_init(rtp_session->detectors[EST_LOSS],0.005,0.5); + } } static int rtcp_stats(switch_rtp_t *rtp_session) @@ -5908,6 +5935,8 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t uint32_t sec, ntp_sec, ntp_usec, lsr_now; uint32_t lsr; uint32_t packet_ssrc; + double rtt_now = 0; + int rtt_increase = 0, packet_loss_increase=0; now = switch_time_now(); /* number of microseconds since 00:00:00 january 1, 1970 UTC */ sec = (uint32_t)(now/1000000); /* converted to second (NTP most significant bits) */ @@ -5962,10 +5991,6 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t ((float)(uint8_t)percent_fraction * .3)); } - if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) { - switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, SCCT_NONE, NULL, NULL, NULL); - } - rtp_session->rtcp_frame.reports[i].ssrc = ntohl(report->ssrc); rtp_session->rtcp_frame.reports[i].fraction = (uint8_t)report->fraction; rtp_session->rtcp_frame.reports[i].lost = ntohl(report->lost); @@ -5974,7 +5999,6 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t rtp_session->rtcp_frame.reports[i].lsr = ntohl(report->lsr); rtp_session->rtcp_frame.reports[i].dlsr = ntohl(report->dlsr); if (rtp_session->rtcp_frame.reports[i].lsr && !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) { - double rtt_now; switch_time_exp_gmt(&now_hr,now); /* Calculating RTT = A - DLSR - LSR */ rtt_now = (double)(lsr_now - rtp_session->rtcp_frame.reports[i].dlsr - rtp_session->rtcp_frame.reports[i].lsr)/65536; @@ -5992,6 +6016,123 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "RTT average %f\n", rtp_session->rtcp_frame.reports[i].rtt_avg); } + + if (rtp_session->flags[SWITCH_RTP_FLAG_ADJ_BITRATE_CAP] && rtp_session->flags[SWITCH_RTP_FLAG_ESTIMATORS] && !rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + + /* SWITCH_RTP_FLAG_ADJ_BITRATE_CAP : Can the codec change its bitrate on the fly per API command ? */ +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "Current packet loss: [%d %%] Current RTT: [%f ms]\n", percent_fraction, rtt_now); +#endif + + switch_kalman_estimate(rtp_session->estimators[EST_RTT], rtt_now, EST_RTT); + + if (switch_kalman_cusum_detect_change(rtp_session->detectors[EST_RTT], rtt_now, rtp_session->estimators[EST_RTT]->val_estimate_last)) { + /* sudden change in the mean value of RTT */ +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of RTT !\n"); +#endif + rtt_increase = 1; + } + + switch_kalman_estimate(rtp_session->estimators[EST_LOSS], percent_fraction, EST_LOSS); + + if (switch_kalman_cusum_detect_change(rtp_session->detectors[EST_LOSS], percent_fraction, rtp_session->estimators[EST_LOSS]->val_estimate_last)){ + /* sudden change in the mean value of packet loss */ +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of packet loss!\n"); +#endif + packet_loss_increase = 1; + } +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "ESTIMATORS: Packet loss will be: [%f] RTT will be: [%f ms]\n", + rtp_session->estimators[EST_LOSS]->val_estimate_last, rtp_session->estimators[EST_RTT]->val_estimate_last); +#endif + + if (rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) { + /*getting bad*/ + if (switch_kalman_is_slow_link(rtp_session->estimators[EST_LOSS], + rtp_session->estimators[EST_RTT])) { + /* going to minimum bitrate */ +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "Slow link conditions: Loss average: [%d %%], Previous loss: [%d %%]. \ + Going to minimum bitrate!",rtp_session->rtcp_frame.reports[i].loss_avg, old_avg); +#endif + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE, SCCT_STRING, "minimum", SCCT_NONE, NULL, NULL, NULL); + /* if after going to minimum bitrate we still have packet loss then we increase ptime. TODO */ + + } else if (packet_loss_increase && (rtp_session->estimators[EST_LOSS]->val_estimate_last >= 5)) { + /* sudden change in the mean value of packet loss percentage */ + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE, + SCCT_STRING, "decrease", + SCCT_NONE, NULL, NULL, NULL); +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of packet loss percentage !\n"); +#endif + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, + (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, + SCCT_NONE, NULL, NULL, NULL); + + } else if (!rtt_increase && rtp_session->estimators[EST_LOSS]->val_estimate_last >= rtp_session->rtcp_frame.reports[i].loss_avg ) { + /* lossy because of congestion (queues full somewhere -> some packets are dropped , but RTT is good ), packet loss with many small gaps */ +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "packet loss, but RTT is not bad\n"); +#endif + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, + (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, + SCCT_NONE, NULL, NULL, NULL); + + } else if ((rtp_session->estimators[EST_LOSS]->val_estimate_last < 1) && packet_loss_increase) { +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "small packet loss average\n"); +#endif + /*small loss_avg*/ + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE, + SCCT_STRING, "default", + SCCT_NONE, NULL, NULL, NULL); + + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, + (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, + SCCT_NONE, NULL, NULL, NULL); + + } else if ((rtp_session->estimators[EST_LOSS]->val_estimate_last < 5) && + (rtp_session->rtcp_frame.reports[i].rtt_avg < rtp_session->estimators[EST_RTT]->val_estimate_last)) { + + /* estimate that packet loss will decrease, we can increase the bitrate */ + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE, + SCCT_STRING, "increase", + SCCT_NONE, NULL, NULL, NULL); + + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, + (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, + SCCT_NONE, NULL, NULL, NULL); + + } else { + /* *do nothing about bitrate, just pass the packet loss to the codec */ +#ifdef DEBUG_ESTIMATORS + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"do nothing about bitrate, just pass the packet loss to the codec\n"); +#endif + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, + (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, + SCCT_NONE, NULL, NULL, NULL); + } + } + } else { + if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) { + switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, + SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, + (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, + SCCT_NONE, NULL, NULL, NULL); + } + } } rtp_session->rtcp_frame.report_count = (uint16_t)i;