diff --git a/src/include/private/switch_core_pvt.h b/src/include/private/switch_core_pvt.h index 78e067b336..720c0fdfd9 100644 --- a/src/include/private/switch_core_pvt.h +++ b/src/include/private/switch_core_pvt.h @@ -221,6 +221,9 @@ struct switch_media_bug { uint32_t record_pre_buffer_max; switch_frame_t *ping_frame; switch_frame_t *read_demux_frame; + switch_queue_t *read_video_queue; + switch_queue_t *write_video_queue; + switch_thread_t *video_bug_thread; struct switch_media_bug *next; }; diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 7ee80e771a..fa3d3de634 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -488,7 +488,8 @@ typedef enum { SWITCH_ABC_TYPE_TAP_NATIVE_READ, SWITCH_ABC_TYPE_TAP_NATIVE_WRITE, SWITCH_ABC_TYPE_CLOSE, - SWITCH_ABC_TYPE_READ_VIDEO_PING + SWITCH_ABC_TYPE_READ_VIDEO_PING, + SWITCH_ABC_TYPE_STREAM_VIDEO_PING } switch_abc_type_t; typedef struct { @@ -1723,7 +1724,10 @@ typedef enum { SMBF_TAP_NATIVE_READ = (1 << 13), SMBF_TAP_NATIVE_WRITE = (1 << 14), SMBF_ONE_ONLY = (1 << 15), - SMBF_MASK = (1 << 16) + SMBF_MASK = (1 << 16), + SMBF_READ_VIDEO_PING = (1 << 17), + SMBF_READ_VIDEO_STREAM = (1 << 18), + SMBF_WRITE_VIDEO_STREAM = (1 << 19) } switch_media_bug_flag_enum_t; typedef uint32_t switch_media_bug_flag_t; diff --git a/src/switch_core_media.c b/src/switch_core_media.c index cda875ef77..91dd02ad6b 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -10218,6 +10218,39 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor img = dup_img; } + + if (session->bugs) { + switch_media_bug_t *bp; + //switch_bool_t ok = SWITCH_TRUE; + int prune = 0; + switch_thread_rwlock_rdlock(session->bug_rwlock); + for (bp = session->bugs; bp; bp = bp->next) { + if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) { + continue; + } + + if (!switch_channel_test_flag(session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bp, SMBF_ANSWER_REQ)) { + continue; + } + + if (switch_test_flag(bp, SMBF_PRUNE)) { + prune++; + continue; + } + + if (bp->ready && switch_test_flag(bp, SMBF_WRITE_VIDEO_STREAM)) { + switch_image_t *dimg = NULL; + switch_img_copy(img, &dimg); + switch_queue_push(bp->write_video_queue, dimg); + + } + switch_thread_rwlock_unlock(session->bug_rwlock); + if (prune) { + switch_core_media_bug_prune(session); + } + } + } + write_frame = *frame; frame = &write_frame; frame->img = img; @@ -10381,7 +10414,15 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core continue; } - if (bp->ready && switch_test_flag(bp, SMBF_READ_PING)) { + if (bp->ready && switch_test_flag(bp, SMBF_READ_VIDEO_STREAM)) { + if ((*frame)->img) { + switch_image_t *img = NULL; + switch_img_copy((*frame)->img, &img); + switch_queue_push(bp->read_video_queue, img); + } + } + + if (bp->ready && switch_test_flag(bp, SMBF_READ_VIDEO_PING)) { switch_mutex_lock(bp->read_mutex); bp->ping_frame = *frame; if (bp->callback) { diff --git a/src/switch_core_media_bug.c b/src/switch_core_media_bug.c index d6f7009097..3633c77db6 100644 --- a/src/switch_core_media_bug.c +++ b/src/switch_core_media_bug.c @@ -386,6 +386,134 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_read(switch_media_bug_t *b return SWITCH_STATUS_SUCCESS; } +static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *obj) +{ + switch_media_bug_t *bug = (switch_media_bug_t *) obj; + switch_queue_t *main_q = NULL, *other_q = NULL; + switch_image_t *IMG = NULL, *img = NULL, *other_img = NULL; + void *pop; + uint8_t *buf; + switch_size_t buflen = SWITCH_RTP_MAX_BUF_LEN; + switch_frame_t frame = { 0 }; + + buf = switch_core_session_alloc(bug->session, buflen); + frame.packet = buf; + frame.data = buf + 12; + frame.packetlen = buflen; + frame.buflen = buflen - 12; + frame.flags = SFF_RAW_RTP; + + if (switch_test_flag(bug, SMBF_READ_VIDEO_STREAM)) { + main_q = bug->read_video_queue; + + if (switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM)) { + other_q = bug->write_video_queue; + } + } else if (switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM)) { + main_q = bug->write_video_queue; + } else { + return NULL; + } + + while (bug->ready) { + switch_status_t status; + int w = 0, h = 0, ok = 1; + + + if ((status = switch_queue_pop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) { + switch_img_free(&img); + + if (!pop) { + goto end; + } + + img = (switch_image_t *) pop; + + if (other_q) { + while(switch_queue_size(other_q) > 0) { + if ((status = switch_queue_trypop(other_q, &pop)) == SWITCH_STATUS_SUCCESS) { + switch_img_free(&other_img); + if (!(other_img = (switch_image_t *) pop)) { + goto end; + } + } + } + } + + w = img->d_w; + h = img->d_h; + + if (other_q) { + if (other_img) { + if (other_img->d_w != w || other_img->d_h != h) { + switch_image_t *tmp_img = NULL; + + switch_img_scale(other_img, &tmp_img, w, h); + switch_img_free(&other_img); + other_img = tmp_img; + } + } + + w *= 2; + + if (!IMG || IMG->d_h != h || IMG->d_w != w) { + switch_img_free(&IMG); + IMG = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, w, h, 1); + } + + switch_img_patch(IMG, img, 0, 0); + + if (other_img) { + switch_img_patch(IMG, other_img, w / 2, 0); + } + + } else { + IMG = img; + } + + switch_thread_rwlock_rdlock(bug->session->bug_rwlock); + switch_mutex_lock(bug->read_mutex); + frame.img = IMG; + bug->ping_frame = &frame; + if (bug->callback) { + if (bug->callback(bug, bug->user_data, SWITCH_ABC_TYPE_STREAM_VIDEO_PING) == SWITCH_FALSE + || (bug->stop_time && bug->stop_time <= switch_epoch_time_now(NULL))) { + ok = SWITCH_FALSE; + } + } + bug->ping_frame = NULL; + switch_mutex_unlock(bug->read_mutex); + switch_thread_rwlock_unlock(bug->session->bug_rwlock); + + if (!ok) { + switch_set_flag(bug, SMBF_PRUNE); + goto end; + } + } + } + + end: + + switch_img_free(&IMG); + switch_img_free(&img); + switch_img_free(&other_img); + + while (switch_queue_pop(main_q, &pop) == SWITCH_STATUS_SUCCESS && pop) { + img = (switch_image_t *) pop; + switch_img_free(&img); + } + + if (other_q) { + while (switch_queue_pop(other_q, &pop) == SWITCH_STATUS_SUCCESS && pop) { + img = (switch_image_t *) pop; + switch_img_free(&img); + } + } + + return NULL; +} + + #define MAX_BUG_BUFFER 1024 * 512 SWITCH_DECLARE(switch_status_t) switch_core_media_bug_add(switch_core_session_t *session, const char *function, @@ -532,6 +660,24 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_add(switch_core_session_t *new_bug = bug; + + if ((switch_test_flag(bug, SMBF_READ_VIDEO_STREAM) || switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM))) { + switch_threadattr_t *thd_attr = NULL; + switch_memory_pool_t *pool = switch_core_session_get_pool(session); + + if (switch_test_flag(bug, SMBF_READ_VIDEO_STREAM)) { + switch_queue_create(&bug->read_video_queue, SWITCH_CORE_QUEUE_LEN, pool); + } + + if (switch_test_flag(bug, SMBF_WRITE_VIDEO_STREAM)) { + switch_queue_create(&bug->write_video_queue, SWITCH_CORE_QUEUE_LEN, pool); + } + + switch_threadattr_create(&thd_attr, pool); + switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); + switch_thread_create(&bug->video_bug_thread, thd_attr, video_bug_thread, bug, pool); + } + if (tap_only) { switch_set_flag(session, SSF_MEDIA_BUG_TAP_ONLY); } else { @@ -787,6 +933,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_remove_all_function(switch SWITCH_DECLARE(switch_status_t) switch_core_media_bug_close(switch_media_bug_t **bug) { switch_media_bug_t *bp = *bug; + if (bp) { if ((bp->thread_id && bp->thread_id != switch_thread_self()) || switch_test_flag(bp, SMBF_LOCK)) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(switch_core_media_bug_get_session(*bug)), SWITCH_LOG_DEBUG, "BUG is thread locked skipping.\n"); @@ -795,8 +942,24 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_close(switch_media_bug_t * if (bp->callback) { bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_CLOSE); - bp->ready = 0; } + + bp->ready = 0; + + if (bp->video_bug_thread) { + switch_status_t st; + + if (bp->read_video_queue) { + switch_queue_push(bp->read_video_queue, NULL); + } + + if (bp->write_video_queue) { + switch_queue_push(bp->write_video_queue, NULL); + } + + switch_thread_join(&st, bp->video_bug_thread); + } + switch_core_media_bug_destroy(bp); switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(switch_core_media_bug_get_session(*bug)), SWITCH_LOG_DEBUG, "Removing BUG from %s\n", switch_channel_get_name(bp->session->channel)); diff --git a/src/switch_ivr_async.c b/src/switch_ivr_async.c index ccc3c506d0..2204d8a33b 100644 --- a/src/switch_ivr_async.c +++ b/src/switch_ivr_async.c @@ -1184,13 +1184,12 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s switch_threadattr_create(&thd_attr, pool); switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); switch_thread_create(&rh->thread, thd_attr, recording_thread, bug, pool); - + while(--sanity > 0 && !rh->thread_ready) { switch_yield(10000); } } - if (switch_event_create(&event, SWITCH_EVENT_RECORD_START) == SWITCH_STATUS_SUCCESS) { switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Record-File-Path", rh->file); switch_channel_event_set_data(channel, event); @@ -1334,7 +1333,8 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s if (switch_core_file_has_video(rh->fh)) { - switch_core_media_set_video_file(session, NULL, SWITCH_RW_READ); + //switch_core_media_set_video_file(session, NULL, SWITCH_RW_READ); + switch_channel_clear_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); } switch_core_file_close(rh->fh); @@ -1476,11 +1476,11 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s } break; case SWITCH_ABC_TYPE_READ_VIDEO_PING: - + case SWITCH_ABC_TYPE_STREAM_VIDEO_PING: if (rh->fh) { if (!bug->ping_frame) break; - - if (len && switch_core_file_write_video(rh->fh, bug->ping_frame) != SWITCH_STATUS_SUCCESS && rh->hangup_on_error) { + + if ((len || bug->ping_frame->img) && switch_core_file_write_video(rh->fh, bug->ping_frame) != SWITCH_STATUS_SUCCESS && rh->hangup_on_error) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error writing video to %s\n", rh->file); switch_channel_hangup(channel, SWITCH_CAUSE_DESTINATION_OUT_OF_ORDER); switch_core_session_reset(session, SWITCH_TRUE, SWITCH_TRUE); @@ -1488,6 +1488,7 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s } } break; + case SWITCH_ABC_TYPE_WRITE: default: break; @@ -2290,7 +2291,19 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t } if (switch_core_file_has_video(fh)) { - switch_core_media_set_video_file(session, fh, SWITCH_RW_READ); + //switch_core_media_set_video_file(session, fh, SWITCH_RW_READ); + switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); + + if ((vval = switch_channel_get_variable(channel, "record_concat_video")) && switch_true(vval)) { + flags |= SMBF_READ_VIDEO_STREAM; + flags |= SMBF_WRITE_VIDEO_STREAM; + } else { + flags |= SMBF_READ_VIDEO_PING; + } + } else { + flags &= ~SMBF_READ_VIDEO_PING; + flags &= ~SMBF_READ_VIDEO_STREAM; + flags &= ~SMBF_WRITE_VIDEO_STREAM; } } else {