webrtc video tweaks

This commit is contained in:
Anthony Minessale 2013-07-16 19:15:56 -05:00
parent 02ddafda3f
commit eebac2ea95
4 changed files with 126 additions and 65 deletions

View File

@ -202,7 +202,8 @@ typedef enum {
CFLAG_AUDIO_ALWAYS = (1 << 15), CFLAG_AUDIO_ALWAYS = (1 << 15),
CFLAG_ENDCONF_FORCED = (1 << 16), CFLAG_ENDCONF_FORCED = (1 << 16),
CFLAG_RFC4579 = (1 << 17), CFLAG_RFC4579 = (1 << 17),
CFLAG_FLOOR_CHANGE = (1 << 18) CFLAG_FLOOR_CHANGE = (1 << 18),
CFLAG_VID_FLOOR_LOCK = (1 << 19)
} conf_flag_t; } conf_flag_t;
typedef enum { typedef enum {
@ -1454,12 +1455,11 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe
switch_channel_set_variable(channel, "conference_recording", conference->record_filename); switch_channel_set_variable(channel, "conference_recording", conference->record_filename);
switch_channel_set_variable(channel, CONFERENCE_UUID_VARIABLE, conference->uuid_str); switch_channel_set_variable(channel, CONFERENCE_UUID_VARIABLE, conference->uuid_str);
if (switch_channel_test_flag(channel, CF_VIDEO)) { if (switch_channel_test_flag(channel, CF_VIDEO)) {
if (!switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) { if (switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) {
switch_channel_set_flag(channel, CF_VIDEO_ECHO);
} else {
switch_channel_clear_flag(channel, CF_VIDEO_ECHO); switch_channel_clear_flag(channel, CF_VIDEO_ECHO);
switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
} }
/* Tell the channel to request a fresh vid frame */ /* Tell the channel to request a fresh vid frame */
switch_core_session_refresh_video(member->session); switch_core_session_refresh_video(member->session);
@ -1577,36 +1577,47 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe
return status; return status;
} }
static void conference_set_video_floor_holder(conference_obj_t *conference, conference_member_t *member) static void conference_set_video_floor_holder(conference_obj_t *conference, conference_member_t *member, switch_bool_t force)
{ {
switch_event_t *event; switch_event_t *event;
conference_member_t *old_member = NULL;
int old_id = 0; int old_id = 0;
if (switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) { if (switch_test_flag(conference, CFLAG_VIDEO_BRIDGE) || (!force && switch_test_flag(conference, CFLAG_VID_FLOOR_LOCK))) {
return; return;
} }
if (conference->video_floor_holder) { if (conference->video_floor_holder) {
if (conference->video_floor_holder == member) { if (conference->video_floor_holder == member) {
return; return;
} else { } else {
old_member = conference->video_floor_holder;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping video floor %s\n", switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping video floor %s\n",
switch_channel_get_name(conference->video_floor_holder->channel)); switch_channel_get_name(old_member->channel));
old_id = conference->video_floor_holder->id;
switch_channel_clear_flag(conference->video_floor_holder->channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(conference->video_floor_holder->session);
} }
} }
if ((conference->video_floor_holder = member)) {
switch_mutex_lock(conference->mutex);
if (member) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding video floor %s\n", switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding video floor %s\n",
switch_channel_get_name(conference->video_floor_holder->channel)); switch_channel_get_name(member->channel));
switch_channel_set_flag(member->channel, CF_VIDEO_PASSIVE); switch_channel_set_flag(member->channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(conference->video_floor_holder->session); switch_core_session_refresh_video(member->session);
switch_set_flag(conference, CFLAG_FLOOR_CHANGE); conference->video_floor_holder = member;
} else {
conference->video_floor_holder = NULL;
} }
if (old_id > -1 && test_eflag(conference, EFLAG_FLOOR_CHANGE)) { if (old_member) {
old_id = old_member->id;
switch_channel_clear_flag(old_member->channel, CF_VIDEO_PASSIVE);
}
switch_set_flag(conference, CFLAG_FLOOR_CHANGE);
switch_mutex_unlock(conference->mutex);
if (test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT); switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT);
conference_add_event_data(conference, event); conference_add_event_data(conference, event);
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Action", "video-floor-change"); switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Action", "video-floor-change");
@ -1628,33 +1639,44 @@ static void conference_set_video_floor_holder(conference_obj_t *conference, conf
static void conference_set_floor_holder(conference_obj_t *conference, conference_member_t *member) static void conference_set_floor_holder(conference_obj_t *conference, conference_member_t *member)
{ {
switch_event_t *event; switch_event_t *event;
conference_member_t *old_member = NULL;
int old_id = 0; int old_id = 0;
if (!switch_test_flag(conference, CFLAG_VIDEO_BRIDGE) &&
((conference->video_floor_holder && !member) || (member && switch_channel_test_flag(member->channel, CF_VIDEO)))) {
conference_set_video_floor_holder(conference, member, SWITCH_FALSE);
}
if (conference->floor_holder) { if (conference->floor_holder) {
if (conference->floor_holder == member) { if (conference->floor_holder == member) {
return; return;
} else { } else {
old_id = conference->floor_holder->id; old_member = conference->floor_holder;
if (!conference->video_floor_holder && !switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping floor %s\n",
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Dropping floor %s\n", switch_channel_get_name(old_member->channel));
switch_channel_get_name(conference->floor_holder->channel));
switch_channel_clear_flag(conference->floor_holder->channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(conference->floor_holder->session);
}
} }
} }
if ((conference->floor_holder = member)) { switch_mutex_lock(conference->mutex);
if (!conference->video_floor_holder && !switch_test_flag(conference, CFLAG_VIDEO_BRIDGE)) { if (member) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding floor %s\n", switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Adding floor %s\n",
switch_channel_get_name(conference->floor_holder->channel)); switch_channel_get_name(member->channel));
switch_channel_set_flag(member->channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(conference->floor_holder->session); conference->floor_holder = member;
} } else {
switch_set_flag(conference, CFLAG_FLOOR_CHANGE); conference->floor_holder = NULL;
} }
if (old_id > -1 && test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
if (old_member) {
old_id = old_member->id;
}
switch_set_flag(conference, CFLAG_FLOOR_CHANGE);
switch_mutex_unlock(conference->mutex);
if (test_eflag(conference, EFLAG_FLOOR_CHANGE)) {
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT); switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT);
conference_add_event_data(conference, event); conference_add_event_data(conference, event);
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Action", "floor-change"); switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Action", "floor-change");
@ -1763,7 +1785,7 @@ static switch_status_t conference_del_member(conference_obj_t *conference, confe
if (member == member->conference->video_floor_holder) { if (member == member->conference->video_floor_holder) {
conference_set_video_floor_holder(member->conference, NULL); conference_set_video_floor_holder(member->conference, NULL, SWITCH_TRUE);
} }
member->conference = NULL; member->conference = NULL;
@ -1779,11 +1801,6 @@ static switch_status_t conference_del_member(conference_obj_t *conference, confe
} }
} }
if (switch_channel_test_flag(channel, CF_VIDEO)) {
switch_channel_set_flag(channel, CF_VIDEO_ECHO);
switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
}
conference_send_presence(conference); conference_send_presence(conference);
switch_channel_set_variable(channel, "conference_call_key", NULL); switch_channel_set_variable(channel, "conference_call_key", NULL);
@ -1914,8 +1931,6 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr
if (conference->video_floor_holder) { if (conference->video_floor_holder) {
floor_holder = conference->video_floor_holder; floor_holder = conference->video_floor_holder;
} else {
floor_holder = conference->floor_holder;
} }
@ -1924,7 +1939,7 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr
goto do_continue; goto do_continue;
} }
if (!switch_channel_test_flag(switch_core_session_get_channel(floor_holder->session), CF_VIDEO)) { if (!floor_holder->session || !floor_holder->channel || !switch_channel_test_flag(floor_holder->channel, CF_VIDEO)) {
yield = 100000; yield = 100000;
goto do_continue; goto do_continue;
} }
@ -2074,8 +2089,8 @@ static void *SWITCH_THREAD_FUNC conference_thread_run(switch_thread_t *thread, v
if (switch_test_flag(imember, MFLAG_RUNNING) && imember->session) { if (switch_test_flag(imember, MFLAG_RUNNING) && imember->session) {
switch_channel_t *channel = switch_core_session_get_channel(imember->session); switch_channel_t *channel = switch_core_session_get_channel(imember->session);
if ((!floor_holder || (imember->score_iir > SCORE_IIR_SPEAKING_MAX && (floor_holder->score_iir < SCORE_IIR_SPEAKING_MIN))) && if ((!floor_holder || (imember->score_iir > SCORE_IIR_SPEAKING_MAX && (floor_holder->score_iir < SCORE_IIR_SPEAKING_MIN)))) {// &&
(!switch_test_flag(conference, CFLAG_VID_FLOOR) || switch_channel_test_flag(channel, CF_VIDEO))) { //(!switch_test_flag(conference, CFLAG_VID_FLOOR) || switch_channel_test_flag(channel, CF_VIDEO))) {
floor_holder = imember; floor_holder = imember;
} }
@ -5282,7 +5297,8 @@ static switch_status_t conf_api_sub_clear_vid_floor(conference_obj_t *conference
} }
switch_mutex_lock(conference->mutex); switch_mutex_lock(conference->mutex);
conference_set_video_floor_holder(conference, NULL); switch_clear_flag(conference, CFLAG_VID_FLOOR_LOCK);
//conference_set_video_floor_holder(conference, NULL);
switch_mutex_unlock(conference->mutex); switch_mutex_unlock(conference->mutex);
return SWITCH_STATUS_SUCCESS; return SWITCH_STATUS_SUCCESS;
@ -5307,19 +5323,22 @@ static switch_status_t conf_api_sub_vid_floor(conference_member_t *member, switc
force = 1; force = 1;
} }
if (!force && member->conference->video_floor_holder == member) { if (member->conference->video_floor_holder == member && switch_test_flag(member->conference, CFLAG_VID_FLOOR_LOCK)) {
conference_set_video_floor_holder(member->conference, NULL); switch_clear_flag(member->conference, CFLAG_VID_FLOOR_LOCK);
conference_set_floor_holder(member->conference, member);
if (stream == NULL) { if (stream == NULL) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK floor none\n", member->conference->name); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK video floor auto\n", member->conference->name);
} else { } else {
stream->write_function(stream, "OK floor none\n"); stream->write_function(stream, "OK floor none\n");
} }
} else if (force || member->conference->video_floor_holder == NULL) { } else if (force || member->conference->video_floor_holder == NULL) {
conference_set_video_floor_holder(member->conference, member); switch_set_flag(member->conference, CFLAG_VID_FLOOR_LOCK);
conference_set_video_floor_holder(member->conference, member, SWITCH_TRUE);
if (test_eflag(member->conference, EFLAG_FLOOR_CHANGE)) { if (test_eflag(member->conference, EFLAG_FLOOR_CHANGE)) {
if (stream == NULL) { if (stream == NULL) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK floor %d %s\n", switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "conference %s OK video floor %d %s\n",
member->conference->name, member->id, switch_channel_get_name(member->channel)); member->conference->name, member->id, switch_channel_get_name(member->channel));
} else { } else {
stream->write_function(stream, "OK floor %u\n", member->id); stream->write_function(stream, "OK floor %u\n", member->id);

View File

@ -3792,7 +3792,6 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s Video thread started\n", switch_channel_get_name(session->channel)); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s Video thread started\n", switch_channel_get_name(session->channel));
switch_core_session_refresh_video(session); switch_core_session_refresh_video(session);
switch_channel_set_flag(channel, CF_VIDEO_ECHO);
while (switch_channel_up_nosig(channel)) { while (switch_channel_up_nosig(channel)) {

View File

@ -2625,7 +2625,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_exec(switch_core_session_t *
int scope = 0; int scope = 0;
char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1]; char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
char *app_uuid = uuid_str; char *app_uuid = uuid_str;
if ((app_uuid_var = switch_channel_get_variable(channel, "app_uuid"))) { if ((app_uuid_var = switch_channel_get_variable(channel, "app_uuid"))) {
app_uuid = (char *)app_uuid_var; app_uuid = (char *)app_uuid_var;
switch_channel_set_variable(channel, "app_uuid", NULL); switch_channel_set_variable(channel, "app_uuid", NULL);
@ -2727,8 +2727,19 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_exec(switch_core_session_t *
msg.string_array_arg[1] = expanded; msg.string_array_arg[1] = expanded;
switch_core_session_receive_message(session, &msg); switch_core_session_receive_message(session, &msg);
if (switch_channel_test_flag(channel, CF_VIDEO)) {
switch_channel_set_flag(channel, CF_VIDEO_ECHO);
switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(session);
}
application_interface->application_function(session, expanded); application_interface->application_function(session, expanded);
if (switch_channel_test_flag(channel, CF_VIDEO)) {
switch_channel_set_flag(channel, CF_VIDEO_ECHO);
switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(session);
}
if (switch_event_create(&event, SWITCH_EVENT_CHANNEL_EXECUTE_COMPLETE) == SWITCH_STATUS_SUCCESS) { if (switch_event_create(&event, SWITCH_EVENT_CHANNEL_EXECUTE_COMPLETE) == SWITCH_STATUS_SUCCESS) {
const char *resp = switch_channel_get_variable(session->channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE); const char *resp = switch_channel_get_variable(session->channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE);
switch_channel_event_set_data(session->channel, event); switch_channel_event_set_data(session->channel, event);

View File

@ -54,7 +54,7 @@
#include <switch_version.h> #include <switch_version.h>
#include <switch_ssl.h> #include <switch_ssl.h>
#define FIR_COUNTDOWN 100 #define FIR_COUNTDOWN 50
#define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++ #define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++
#define READ_DEC(rtp_session) switch_mutex_unlock(rtp_session->read_mutex); rtp_session->reading-- #define READ_DEC(rtp_session) switch_mutex_unlock(rtp_session->read_mutex); rtp_session->reading--
@ -270,6 +270,12 @@ typedef struct ts_normalize_s {
uint32_t last_frame; uint32_t last_frame;
uint32_t ts; uint32_t ts;
uint32_t delta; uint32_t delta;
uint32_t delta_ct;
uint32_t delta_ttl;
uint32_t delta_avg;
uint32_t delta_delta;
double delta_percent;
uint8_t m;
} ts_normalize_t; } ts_normalize_t;
struct switch_rtp { struct switch_rtp {
@ -3349,6 +3355,10 @@ SWITCH_DECLARE(void) switch_rtp_break(switch_rtp_t *rtp_session)
return; return;
} }
if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
return;
}
switch_mutex_lock(rtp_session->flag_mutex); switch_mutex_lock(rtp_session->flag_mutex);
rtp_session->flags[SWITCH_RTP_FLAG_BREAK] = 1; rtp_session->flags[SWITCH_RTP_FLAG_BREAK] = 1;
@ -4247,15 +4257,6 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t
return SWITCH_STATUS_SUCCESS; return SWITCH_STATUS_SUCCESS;
} }
if (*bytes && rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
unsigned int diff = ts - rtp_session->last_read_ts;
if (abs(diff) > 10000) {
switch_rtp_video_refresh(rtp_session);
}
}
if (ts) { if (ts) {
rtp_session->last_read_ts = ts; rtp_session->last_read_ts = ts;
} }
@ -4645,7 +4646,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_
goto end; goto end;
} }
// This is dumb // This is dumb
switch_rtp_video_refresh(rtp_session); //switch_rtp_video_refresh(rtp_session);
goto rtcp; goto rtcp;
} }
} }
@ -4712,7 +4713,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_
} }
has_rtcp = 0; has_rtcp = 0;
} else if (rtp_session->rtcp_read_pollfd) { } else if (rtp_session->rtcp_read_pollfd) {
rtcp_poll_status = switch_poll(rtp_session->rtcp_read_pollfd, 1, &rtcp_fdr, 0); rtcp_poll_status = switch_poll(rtp_session->rtcp_read_pollfd, 1, &rtcp_fdr, 0);
} }
@ -5272,7 +5273,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_zerocopy_read_frame(switch_rtp_t *rtp
} }
if (rtp_session->fir_countdown) { if (rtp_session->fir_countdown) {
if (--rtp_session->fir_countdown == 0) { if (--rtp_session->fir_countdown == FIR_COUNTDOWN / 2) {
send_fir(rtp_session); send_fir(rtp_session);
//send_pli(rtp_session); //send_pli(rtp_session);
} }
@ -5483,6 +5484,9 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc) { if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc) {
if (rtp_session->ts_norm.last_ssrc) { if (rtp_session->ts_norm.last_ssrc) {
rtp_session->ts_norm.m = 1;
rtp_session->ts_norm.delta_ct = 1;
rtp_session->ts_norm.delta_ttl = 0;
if (rtp_session->ts_norm.delta) { if (rtp_session->ts_norm.delta) {
rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; rtp_session->ts_norm.ts += rtp_session->ts_norm.delta;
} }
@ -5494,12 +5498,40 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
if (ntohl(send_msg->header.ts) != rtp_session->ts_norm.last_frame) { if (ntohl(send_msg->header.ts) != rtp_session->ts_norm.last_frame) {
rtp_session->ts_norm.delta = ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame; rtp_session->ts_norm.delta = ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame;
if (rtp_session->ts_norm.delta > 0) {
rtp_session->ts_norm.delta_ct++;
if (rtp_session->ts_norm.delta_ct == 1000) {
rtp_session->ts_norm.delta_ct = 1;
rtp_session->ts_norm.delta_ttl = 0;
}
rtp_session->ts_norm.delta_ttl += rtp_session->ts_norm.delta;
rtp_session->ts_norm.delta_avg = rtp_session->ts_norm.delta_ttl / rtp_session->ts_norm.delta_ct;
rtp_session->ts_norm.delta_delta = abs(rtp_session->ts_norm.delta_avg - rtp_session->ts_norm.delta);
rtp_session->ts_norm.delta_percent = (double)((double)rtp_session->ts_norm.delta / (double)rtp_session->ts_norm.delta_avg) * 100.0f;
if (rtp_session->ts_norm.delta_ct > 50 && rtp_session->ts_norm.delta_percent > 125.0) {
//printf("%s diff %d %d (%.2f)\n", switch_core_session_get_name(rtp_session->session),
//rtp_session->ts_norm.delta, rtp_session->ts_norm.delta_avg, rtp_session->ts_norm.delta_percent);
switch_rtp_video_refresh(rtp_session);
}
}
rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; rtp_session->ts_norm.ts += rtp_session->ts_norm.delta;
} }
rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts); rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts);
send_msg->header.ts = htonl(rtp_session->ts_norm.ts); send_msg->header.ts = htonl(rtp_session->ts_norm.ts);
/* wait for a marked frame since we just switched streams */
if (rtp_session->ts_norm.m) {
if (send_msg->header.m) {
rtp_session->ts_norm.m = 0;
} else {
send = 0;
}
}
} }
send_msg->header.ssrc = htonl(rtp_session->ssrc); send_msg->header.ssrc = htonl(rtp_session->ssrc);