From 7c19615890342918978364da4a6c3731866d13cf Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Wed, 21 Jun 2017 19:26:14 -0500 Subject: [PATCH] FS-10417: [freeswitch-core] Reduce flicker in screen sharing #resolve --- src/include/switch_core.h | 3 +- src/include/switch_core_media.h | 6 ++-- .../mod_conference/conference_video.c | 4 +-- src/switch_core_media.c | 14 +++++--- src/switch_core_video.c | 4 +-- src/switch_jitterbuffer.c | 27 +++++++------- src/switch_rtp.c | 35 ++++++++++++------- 7 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/include/switch_core.h b/src/include/switch_core.h index eddf0f5654..60249f88c2 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -2758,7 +2758,8 @@ SWITCH_DECLARE(int) switch_core_gen_certs(const char *prefix); SWITCH_DECLARE(int) switch_core_cert_gen_fingerprint(const char *prefix, dtls_fingerprint_t *fp); SWITCH_DECLARE(int) switch_core_cert_expand_fingerprint(dtls_fingerprint_t *fp, const char *str); SWITCH_DECLARE(int) switch_core_cert_verify(dtls_fingerprint_t *fp); -SWITCH_DECLARE(switch_status_t) switch_core_session_request_video_refresh(switch_core_session_t *session); +SWITCH_DECLARE(switch_status_t) _switch_core_session_request_video_refresh(switch_core_session_t *session, const char *file, const char *func, int line); +#define switch_core_session_request_video_refresh(_s) _switch_core_session_request_video_refresh(_s, __FILE__, __SWITCH_FUNC__, __LINE__) SWITCH_DECLARE(switch_status_t) switch_core_session_send_and_request_video_refresh(switch_core_session_t *session); SWITCH_DECLARE(int) switch_system(const char *cmd, switch_bool_t wait); SWITCH_DECLARE(int) switch_stream_system_fork(const char *cmd, switch_stream_handle_t *stream); diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index d4690339f2..65dab34232 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -330,11 +330,11 @@ SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_sessio switch_codec_flag_t flag); -#define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \ - SCC_VIDEO_GEN_KEYFRAME, SCCT_NONE, NULL, SCCT_NONE, NULL, NULL, NULL) \ +#define switch_core_media_gen_key_frame(_session) switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(_session), SWITCH_LOG_DEBUG1, "%s Send KeyFrame\n", switch_core_session_get_name(_session)); \ + switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, SCC_VIDEO_GEN_KEYFRAME, SCCT_NONE, NULL, SCCT_NONE, NULL, NULL, NULL) #define switch_core_media_write_bandwidth(_session, _val) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \ - SCC_VIDEO_BANDWIDTH, SCCT_STRING, _val, SCCT_NONE, NULL, NULL, NULL) \ + SCC_VIDEO_BANDWIDTH, SCCT_STRING, _val, SCCT_NONE, NULL, NULL, NULL) SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype); diff --git a/src/mod/applications/mod_conference/conference_video.c b/src/mod/applications/mod_conference/conference_video.c index 76a9eb2448..fb8555d3bb 100644 --- a/src/mod/applications/mod_conference/conference_video.c +++ b/src/mod/applications/mod_conference/conference_video.c @@ -2901,7 +2901,7 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr switch_codec_t *check_codec = NULL; int buflen = SWITCH_RTP_MAX_BUF_LEN; int i = 0; - uint32_t video_key_freq = 10000000; + uint32_t video_key_freq = 0; switch_time_t last_key_time = 0; mcu_layer_t *layer = NULL; switch_frame_t write_frame = { 0 }; @@ -4022,7 +4022,7 @@ void *SWITCH_THREAD_FUNC conference_video_super_muxing_thread_run(switch_thread_ int buflen = SWITCH_RTP_MAX_BUF_LEN; int i = 0; switch_time_t last_key_time = 0; - uint32_t video_key_freq = 10000000; + uint32_t video_key_freq = 0; mcu_layer_t *layer = NULL; switch_frame_t write_frame = { 0 }; uint8_t *packet = NULL; diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 35c0c99716..429d4693d6 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -1806,9 +1806,9 @@ SWITCH_DECLARE(switch_status_t) switch_media_handle_create(switch_media_handle_t session->media_handle->mparams = params; - if (!session->media_handle->mparams->video_key_freq) { - session->media_handle->mparams->video_key_freq = 10000000; - } + //if (!session->media_handle->mparams->video_key_freq) { + // session->media_handle->mparams->video_key_freq = 10000000; + //} if (!session->media_handle->mparams->video_key_first) { session->media_handle->mparams->video_key_first = 1000000; @@ -13720,7 +13720,7 @@ SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session } -SWITCH_DECLARE(switch_status_t) switch_core_session_request_video_refresh(switch_core_session_t *session) +SWITCH_DECLARE(switch_status_t) _switch_core_session_request_video_refresh(switch_core_session_t *session, const char *file, const char *func, int line) { switch_channel_t *channel = switch_core_session_get_channel(session); switch_media_handle_t *smh = NULL; @@ -13740,7 +13740,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_request_video_refresh(switch } smh->last_video_refresh_req = now; - + msg._file = file; + msg._func = func; + msg._line = line; + switch_log_printf(SWITCH_CHANNEL_ID_LOG, file, func, line, switch_core_session_get_uuid(session), + SWITCH_LOG_DEBUG1, "%s Video refresh requested.\n", switch_channel_get_name(session->channel)); msg.from = __FILE__; msg.message_id = SWITCH_MESSAGE_INDICATE_VIDEO_REFRESH_REQ; switch_core_session_receive_message(session, &msg); diff --git a/src/switch_core_video.c b/src/switch_core_video.c index f5a6695811..0a5dcffc6f 100644 --- a/src/switch_core_video.c +++ b/src/switch_core_video.c @@ -2255,8 +2255,8 @@ SWITCH_DECLARE(void) switch_img_patch_hole(switch_image_t *IMG, switch_image_t * } } -#define SWITCH_IMG_MAX_WIDTH 1920 * 2 -#define SWITCH_IMG_MAX_HEIGHT 1080 * 2 +#define SWITCH_IMG_MAX_WIDTH 1920 * 4 +#define SWITCH_IMG_MAX_HEIGHT 1080 * 4 #if !defined(SWITCH_HAVE_YUV) #undef SWITCH_HAVE_PNG diff --git a/src/switch_jitterbuffer.c b/src/switch_jitterbuffer.c index bc927fd916..6a8dba108c 100644 --- a/src/switch_jitterbuffer.c +++ b/src/switch_jitterbuffer.c @@ -505,9 +505,9 @@ static void jb_frame_inc_line(switch_jb_t *jb, int i, int line) if (old_frame_len != jb->frame_len) { jb_debug(jb, 2, "%d Change framelen from %u to %u\n", line, old_frame_len, jb->frame_len); - if (jb->session) { - switch_core_session_request_video_refresh(jb->session); - } + //if (jb->session) { + // switch_core_session_request_video_refresh(jb->session); + //} } } @@ -742,9 +742,9 @@ static inline switch_status_t jb_next_packet_by_seq(switch_jb_t *jb, switch_jb_n jb_debug(jb, 2, "%s", "DROPPED FRAME DETECTED RESYNCING\n"); jb->target_seq = 0; - if (jb->session) { - switch_core_session_request_video_refresh(jb->session); - } + //if (jb->session) { + // switch_core_session_request_video_refresh(jb->session); + //} } } @@ -772,9 +772,9 @@ static inline switch_status_t jb_next_packet_by_seq(switch_jb_t *jb, switch_jb_n jb_frame_inc(jb, 1); } - if (jb->session) { - switch_core_session_request_video_refresh(jb->session); - } + //if (jb->session) { + // switch_core_session_request_video_refresh(jb->session); + //} for (x = 0; x < 10; x++) { increment_seq(jb); @@ -930,9 +930,9 @@ SWITCH_DECLARE(void) switch_jb_reset(switch_jb_t *jb) switch_core_inthash_init(&jb->missing_seq_hash); switch_mutex_unlock(jb->mutex); - if (jb->session) { - switch_core_session_request_video_refresh(jb->session); - } + //if (jb->session) { + // switch_core_session_request_video_refresh(jb->session); + //} } jb_debug(jb, 2, "%s", "RESET BUFFER\n"); @@ -1221,9 +1221,6 @@ SWITCH_DECLARE(switch_status_t) switch_jb_put_packet(switch_jb_t *jb, switch_rtp if (got - want > jb->max_frame_len && got - want > 17) { jb_debug(jb, 2, "Missing %u frames, Resetting\n", got - want); switch_jb_reset(jb); - if (jb->session) { - switch_core_session_request_video_refresh(jb->session); - } } else { if (jb->frame_len < got - want) { diff --git a/src/switch_rtp.c b/src/switch_rtp.c index a21bab2c5b..6ce0467fe3 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -1664,9 +1664,9 @@ static void check_jitter(switch_rtp_t *rtp_session) (rtp_session->stats.inbound.last_processed_seq + 1), lost); rtp_session->stats.inbound.last_loss++; - if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { - switch_core_session_request_video_refresh(rtp_session->session); - } + //if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + //switch_core_session_request_video_refresh(rtp_session->session); + //} if (rtp_session->stats.inbound.last_loss > 0 && rtp_session->stats.inbound.last_loss < LOST_BURST_CAPTURE) { rtp_session->stats.inbound.loss[rtp_session->stats.inbound.last_loss] += lost; @@ -4670,6 +4670,7 @@ SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session) SWITCH_DECLARE(void) switch_rtp_video_loss(switch_rtp_t *rtp_session) { + if (!rtp_write_ready(rtp_session, 0, __LINE__)) { return; } @@ -6212,7 +6213,7 @@ static void handle_nack(switch_rtp_t *rtp_session, uint32_t nack) if (blp & (1 << i)) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "Also Got NACK for seq %u\n", ntohs(seq) + i + 1); /* If they are missing more than one, may as well gen a key frame for good measure */ - switch_core_media_gen_key_frame(rtp_session->session); + //switch_core_media_gen_key_frame(rtp_session->session); if (switch_jb_get_packet_by_seq(rtp_session->vbw, htons(ntohs(seq) + i + 1), (switch_rtp_packet_t *) &send_msg, &bytes) == SWITCH_STATUS_SUCCESS) { if (rtp_session->flags[SWITCH_RTP_FLAG_DEBUG_RTP_WRITE]) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG_CLEAN(rtp_session->session), SWITCH_LOG_CONSOLE, @@ -6248,8 +6249,8 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) msg; if (extp->header.fmt != 15) { // <---- REMOVE WHEN BRIA STOPS SENDING UNSOLICITED REMB - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "PICKED UP %s XRTCP type: %d fmt: %d\n", - rtp_type(rtp_session), msg->header.type, extp->header.fmt); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "%s PICKED UP %s XRTCP type: %d fmt: %d\n", + switch_core_session_get_name(rtp_session->session), rtp_type(rtp_session), msg->header.type, extp->header.fmt); } if (msg->header.type == _RTCP_PT_FIR || @@ -6264,19 +6265,27 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t } } #endif - switch_core_media_gen_key_frame(rtp_session->session); - if (rtp_session->vbw) { - switch_jb_reset(rtp_session->vbw); + + if (switch_core_session_media_flow(rtp_session->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "%s Ignoring FIR/PLI from a sendonly stream.\n", + switch_core_session_get_name(rtp_session->session)); + } else { + switch_core_media_gen_key_frame(rtp_session->session); + if (rtp_session->vbw) { + switch_jb_reset(rtp_session->vbw); + } + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "%s Got FIR/PLI\n", + switch_core_session_get_name(rtp_session->session)); + switch_channel_set_flag(switch_core_session_get_channel(rtp_session->session), CF_VIDEO_REFRESH_REQ); } - - switch_channel_set_flag(switch_core_session_get_channel(rtp_session->session), CF_VIDEO_REFRESH_REQ); } if (msg->header.type == _RTCP_PT_RTPFB && extp->header.fmt == _RTCP_RTPFB_NACK) { uint32_t *nack = (uint32_t *) extp->body; int i; - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "Got NACK count %d\n", ntohs(extp->header.length) - 2); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, "%s Got NACK count %d\n", + switch_core_session_get_name(rtp_session->session), ntohs(extp->header.length) - 2); for (i = 0; i < ntohs(extp->header.length) - 2; i++) { @@ -6284,7 +6293,7 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t nack++; } - switch_core_media_gen_key_frame(rtp_session->session); + //switch_core_media_gen_key_frame(rtp_session->session); } } else {