kenmirrors
/
freeswitch
mirror of https://github.com/signalwire/freeswitch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

FS-7505 FS-7514: working towards vid rec

This commit is contained in:
Anthony Minessale 2015-02-27 23:13:17 -06:00 committed by Michael Jerris
parent a42f40f938
commit 59da14542f
5 changed files with 176 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
src/mod/applications/mod_conference/mod_conference.c
View File

@ -407,6 +407,8 @@ typedef struct mcu_canvas_s {
int layers_used;
int layout_floor_id;
int refresh;
int reset_video;
int play_file;
switch_rgb_color_t bgcolor;
switch_mutex_t *mutex;
switch_timer_t timer;
@ -550,6 +552,7 @@ typedef struct conference_obj {
switch_hash_t *layout_hash;
switch_hash_t *layout_group_hash;
struct conf_fps video_fps;
int playing_video_file;
} conference_obj_t;
/* Relationship with another member */
@ -1496,8 +1499,9 @@ static void write_canvas_image_to_codec_group(conference_obj_t *conference, code
if (encode_status == SWITCH_STATUS_SUCCESS || encode_status == SWITCH_STATUS_MORE_DATA) {
switch_assert((encode_status == SWITCH_STATUS_SUCCESS && frame->m) || !frame->m);
switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
if (frame->timestamp) {
switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
}
frame->packetlen = frame->datalen + 12;
switch_mutex_lock(conference->member_mutex);
@ -1530,7 +1534,6 @@ static void write_canvas_image_to_codec_group(conference_obj_t *conference, code
}
#define MAX_MUX_CODECS 10
//#define TRACK_FPS
static video_layout_t *find_best_layout(conference_obj_t *conference, layout_group_t *lg)
{
@ -1574,18 +1577,15 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
int buflen = SWITCH_RECOMMENDED_BUFFER_SIZE * 2;
int i = 0;
int used = 0;
uint32_t video_key_freq = 30000000;
uint32_t video_key_freq = 10000000;
switch_time_t last_key_time = 0;
mcu_layer_t *layer = NULL;
switch_frame_t write_frame = { 0 };
uint8_t *packet = NULL;
layout_group_t *lg = NULL;
switch_image_t *write_img = NULL, *file_img = NULL;
#ifdef TRACK_FPS
uint64_t frames = 0;
switch_time_t started = switch_micro_time_now();
#endif
uint32_t timestamp = 0;
switch_timer_t file_timer = { 0 };
if (conference->video_layout_group) {
lg = switch_core_hash_find(conference->layout_group_hash, conference->video_layout_group);
@ -1606,6 +1606,7 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
conference->video_timer_reset = 1;
packet = switch_core_alloc(conference->pool, SWITCH_RECOMMENDED_BUFFER_SIZE);
switch_core_timer_init(&file_timer, "soft", 1, 90, NULL);
while (globals.running && !switch_test_flag(conference, CFLAG_DESTRUCT) && switch_test_flag(conference, CFLAG_VIDEO_MUXING)) {
switch_bool_t need_refresh = SWITCH_FALSE, need_keyframe = SWITCH_FALSE, need_reset = SWITCH_FALSE;
@ -1614,18 +1615,24 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
if (conference->video_timer_reset) {
conference->video_timer_reset = 0;
if (conference->canvas->timer.interval) {
switch_core_timer_destroy(&conference->canvas->timer);
}
switch_core_timer_init(&conference->canvas->timer, "soft", conference->video_fps.ms, conference->video_fps.samples, NULL);
need_reset = SWITCH_TRUE;
conference->canvas->reset_video = 1;
}
if (!conference->record_fh) {
if (!conference->playing_video_file) {
switch_core_timer_next(&conference->canvas->timer);
}
now = switch_micro_time_now();
switch_mutex_lock(conference->member_mutex);
used = 0;
for (imember = conference->members; imember; imember = imember->next) {
void *pop;
@ -1641,6 +1648,8 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
min_members++;
}
if (conference->playing_video_file) continue;
if (conference->canvas->layout_floor_id > -1 && imember->id == conference->video_floor_holder &&
imember->video_layer_id != conference->canvas->layout_floor_id) {
attach_video_layer(imember, conference->canvas->layout_floor_id);
@ -1701,6 +1710,7 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
size = switch_queue_size(imember->video_queue);
} while(size > 0);
if (img) {
int i;
@ -1788,18 +1798,20 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
}
switch_mutex_unlock(conference->member_mutex);
if (!conference->playing_video_file) {
for (i = 0; i < conference->canvas->total_layers; i++) {
mcu_layer_t *layer = &conference->canvas->layers[i];
for (i = 0; i < conference->canvas->total_layers; i++) {
mcu_layer_t *layer = &conference->canvas->layers[i];
if (layer->member_id > -1 && layer->cur_img && (layer->tagged || layer->geometry.overlap)) {
if (conference->canvas->refresh) {
layer->refresh = 1;
conference->canvas->refresh++;
}
if (layer->member_id > -1 && layer->cur_img && (layer->tagged || layer->geometry.overlap)) {
if (conference->canvas->refresh) {
layer->refresh = 1;
conference->canvas->refresh++;
scale_and_patch(conference, layer, NULL);
layer->tagged = 0;
}
scale_and_patch(conference, layer, NULL);
layer->tagged = 0;
}
}
@ -1807,35 +1819,12 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
conference->canvas->refresh = 0;
}
#if 0
if (1) {
switch_img_txt_handle_t *txthandle = NULL;
switch_rgb_color_t color;
switch_img_txt_handle_create(&txthandle, "/usr/share/fonts/truetype/Microsoft/Verdana.ttf",
"#FFFFFF", "#000000", 24, 0, NULL);
switch_img_txt_handle_render(txthandle, conference->canvas->img, 10, 10, "W00t this works!", NULL, NULL, NULL, 0, 0);
switch_color_set_rgb(&color, "#FF0000");
switch_img_fill(conference->canvas->img, 300, 10, 400, 40, &color);
switch_img_txt_handle_render(txthandle, conference->canvas->img, 300, 22, "W00t this works!", NULL, NULL, "#FF0000", 0, 0);
switch_img_txt_handle_destroy(&txthandle);
if (conference->canvas->reset_video) {
//need_refresh = SWITCH_TRUE;
//need_reset = SWITCH_TRUE;
need_keyframe = SWITCH_TRUE;
conference->canvas->reset_video = 0;
}
#endif
#ifdef TRACK_FPS
{
uint64_t diff = ((now - started) / 1000000);
if (!diff) diff = 1;
frames++;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "fps %ld %ld %ld\n", frames, diff, frames / diff);
}
#endif
if (video_key_freq && (now - last_key_time) > video_key_freq) {
need_keyframe = SWITCH_TRUE;
@ -1843,11 +1832,29 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
}
write_img = conference->canvas->img;
timestamp = conference->canvas->timer.samplecount;
if (conference->fnode) {
if (conference->playing_video_file) {
if (switch_core_file_read_video(&conference->fnode->fh, &write_frame) == SWITCH_STATUS_SUCCESS) {
switch_img_free(&file_img);
if (conference->canvas->play_file) {
conference->canvas->reset_video = 1;
conference->canvas->play_file = 0;
//if (file_timer.interval) {
// switch_core_timer_destroy(&file_timer);
//}
conference->canvas->timer.interval = 1;
conference->canvas->timer.samples = 90;
}
write_img = file_img = write_frame.img;
//switch_core_timer_sync(&file_timer);
//timestamp = file_timer.samplecount;
switch_core_timer_sync(&conference->canvas->timer);
timestamp = conference->canvas->timer.samplecount;
}
} else if (file_img) {
switch_img_free(&file_img);
@ -1862,7 +1869,7 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
for (i = 0; write_codecs[i] && switch_core_codec_ready(&write_codecs[i]->codec) && i < MAX_MUX_CODECS; i++) {
write_codecs[i]->frame.img = write_img;
write_canvas_image_to_codec_group(conference, write_codecs[i], i,
conference->canvas->timer.samplecount, need_refresh, need_keyframe, need_reset);
timestamp, need_refresh, need_keyframe, need_reset);
if (conference->video_write_bandwidth) {
switch_core_codec_control(&write_codecs[i]->codec, SCC_VIDEO_BANDWIDTH, SCCT_INT, &conference->video_write_bandwidth, NULL, NULL);
@ -1874,7 +1881,7 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
switch_mutex_lock(conference->member_mutex);
for (imember = conference->members; imember; imember = imember->next) {
if (switch_test_flag(conference, CFLAG_MINIMIZE_VIDEO_ENCODING) && !switch_test_flag(imember, MFLAG_NO_MINIMIZE_ENCODING)) {
continue;
}
@ -1884,6 +1891,13 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
continue;
}
if (need_refresh) {
switch_core_session_request_video_refresh(imember->session);
}
if (need_keyframe) {
switch_core_media_gen_key_frame(imember->session);
}
switch_set_flag(&write_frame, SFF_RAW_RTP);
write_frame.img = write_img;
@ -1900,10 +1914,11 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
}
}
switch_mutex_unlock(conference->member_mutex);
switch_mutex_unlock(conference->member_mutex);
}
switch_img_free(&file_img);
for (i = 0; i < MCU_MAX_LAYERS; i++) {
layer = &conference->canvas->layers[i];
@ -1929,6 +1944,10 @@ static void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread
switch_core_timer_destroy(&conference->canvas->timer);
if (file_timer.interval) {
switch_core_timer_destroy(&file_timer);
}
destroy_canvas(&conference->canvas);
return NULL;
@ -4130,7 +4149,13 @@ static switch_status_t conference_file_close(conference_obj_t *conference, confe
close_al(node->al);
}
#endif
if (switch_core_file_has_video(&node->fh) && conference->canvas) {
conference->canvas->timer.interval = conference->video_fps.ms;
conference->canvas->timer.samples = conference->video_fps.samples;
switch_core_timer_sync(&conference->canvas->timer);
conference->canvas->reset_video = 1;
conference->playing_video_file = 0;
}
return switch_core_file_close(&node->fh);
}
@ -4372,11 +4397,13 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi
if (switch_test_flag(member->conference, CFLAG_VIDEO_MUXING) && frame->img) {
switch_image_t *img_copy = NULL;
switch_img_copy(frame->img, &img_copy);
switch_queue_push(member->video_queue, img_copy);
switch_thread_rwlock_unlock(member->conference->rwlock);
unlock_member(member);
if (!member->conference->playing_video_file) {
switch_img_copy(frame->img, &img_copy);
switch_queue_push(member->video_queue, img_copy);
switch_thread_rwlock_unlock(member->conference->rwlock);
}
unlock_member(member);
return SWITCH_STATUS_SUCCESS;
}
@ -6512,7 +6539,6 @@ static void *SWITCH_THREAD_FUNC conference_record_thread_run(switch_thread_t *th
switch_size_t data_buf_len;
switch_event_t *event;
switch_size_t len = 0;
char *ext;
int flags = 0;
data_buf_len = samples * sizeof(int16_t);
@ -6576,15 +6602,6 @@ static void *SWITCH_THREAD_FUNC conference_record_thread_run(switch_thread_t *th
fh.pre_buffer_datalen = SWITCH_DEFAULT_FILE_BUFFER_LEN;
/* video recording, only for testing at this time*/
if ((ext = strrchr(rec->path, '.')) != NULL) {
ext++;
if (!strncasecmp(ext, "fsv", 3) || !strncasecmp(ext, "mp4", 3)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Disable buffer for video recording\n");
fh.pre_buffer_datalen = 0;
}
}
flags = SWITCH_FILE_FLAG_WRITE | SWITCH_FILE_DATA_SHORT;
if (conference->members_with_video && switch_test_flag(conference, CFLAG_TRANSCODE_VIDEO)) {
@ -6965,6 +6982,7 @@ static switch_status_t conference_play_file(conference_obj_t *conference, char *
/* Open the file */
fnode->fh.pre_buffer_datalen = SWITCH_DEFAULT_FILE_BUFFER_LEN;
if (switch_core_file_open(&fnode->fh, file, channels, conference->rate, flags, pool) !=
SWITCH_STATUS_SUCCESS) {
switch_event_t *event;
@ -7014,6 +7032,11 @@ static switch_status_t conference_play_file(conference_obj_t *conference, char *
fnode->async = async;
fnode->file = switch_core_strdup(fnode->pool, file);
if (switch_core_file_has_video(&fnode->fh)) {
conference->canvas->play_file = 1;
conference->playing_video_file = 1;
}
/* Queue the node */
switch_mutex_lock(conference->mutex);

74
src/mod/formats/mod_vlc/mod_vlc.c
View File

@ -109,6 +109,7 @@ struct vlc_video_context {
switch_queue_t *video_queue;
int playing;
int ending;
int vid_ready;
uint32_t sync_offset;
switch_mutex_t *video_mutex;
@ -406,10 +407,10 @@ unsigned video_format_setup_callback(void **opaque, char *chroma, unsigned *widt
void video_format_clean_callback(void *opaque)
{
vlc_video_context_t *context = (vlc_video_context_t *)opaque;
switch_safe_free(context->raw_yuyv_data);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "cleanup\n");
context->err = 1;
}
#if 0
@ -501,6 +502,7 @@ static switch_status_t av_init_handle(switch_file_handle_t *handle, switch_image
switch_mutex_init(&vcontext->audio_mutex, SWITCH_MUTEX_NESTED, vcontext->pool);
switch_mutex_init(&vcontext->video_mutex, SWITCH_MUTEX_NESTED, vcontext->pool);
switch_thread_cond_create(&vcontext->cond, vcontext->pool);
switch_thread_cond_create(&acontext->cond, acontext->pool);
switch_core_timer_init(&vcontext->timer, "soft", 1, 1000, vcontext->pool);
@ -787,16 +789,17 @@ static switch_status_t vlc_file_av_read(switch_file_handle_t *handle, void *data
libvlc_state_t status;
if (vcontext->err) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "VLC error\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "VLC ended\n");
return SWITCH_STATUS_GENERR;
}
status = libvlc_media_get_state(vcontext->m);
if (status == libvlc_Error) {
vcontext->err = acontext->err = 1;
return SWITCH_STATUS_GENERR;
}
switch_mutex_lock(vcontext->audio_mutex);
while (vcontext->playing == 0 && status != libvlc_Ended && status != libvlc_Error) {
switch_thread_cond_wait(vcontext->cond, vcontext->audio_mutex);
@ -805,11 +808,21 @@ static switch_status_t vlc_file_av_read(switch_file_handle_t *handle, void *data
if (vcontext->err == 1) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "VLC error\n");
switch_mutex_unlock(vcontext->audio_mutex);
return SWITCH_STATUS_FALSE;
}
switch_mutex_unlock(vcontext->audio_mutex);
if (!vcontext->vid_ready) {
switch_mutex_lock(vcontext->audio_mutex);
if (!vcontext->vid_ready) {
switch_thread_cond_wait(vcontext->cond, vcontext->audio_mutex);
}
switch_mutex_unlock(vcontext->audio_mutex);
}
switch_mutex_lock(vcontext->audio_mutex);
read = switch_buffer_read(vcontext->audio_buffer, data, bytes);
switch_mutex_unlock(vcontext->audio_mutex);
@ -864,12 +877,10 @@ static switch_status_t vlc_file_read(switch_file_handle_t *handle, void *data, s
if (context->err == 1) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "VLC error\n");
switch_mutex_unlock(context->audio_mutex);
return SWITCH_STATUS_FALSE;
}
switch_mutex_unlock(context->audio_mutex);
switch_mutex_lock(context->audio_mutex);
read = switch_buffer_read(context->audio_buffer, data, bytes);
switch_mutex_unlock(context->audio_mutex);
@ -895,7 +906,24 @@ static switch_status_t vlc_file_read_video(switch_file_handle_t *handle, switch_
vlc_video_context_t *vcontext = acontext->vcontext;
void *pop;
if (switch_queue_pop(vcontext->video_queue, &pop) == SWITCH_STATUS_SUCCESS && pop) {
if (vcontext->err) {
return SWITCH_STATUS_FALSE;
}
if (switch_queue_pop(vcontext->video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
if (!vcontext->vid_ready) {
vcontext->vid_ready = 1;
if (switch_mutex_trylock(vcontext->audio_mutex) == SWITCH_STATUS_SUCCESS) {
switch_thread_cond_signal(vcontext->cond);
switch_mutex_unlock(vcontext->audio_mutex);
}
}
if (!pop) {
vcontext->err = 1;
return SWITCH_STATUS_FALSE;
}
frame->img = (switch_image_t *) pop;
return SWITCH_STATUS_SUCCESS;
}
@ -909,6 +937,10 @@ static switch_status_t vlc_file_write_video(switch_file_handle_t *handle, switch
vlc_video_context_t *vcontext = acontext->vcontext;
switch_status_t status = SWITCH_STATUS_SUCCESS;
if (vcontext && vcontext->err) {
return SWITCH_STATUS_FALSE;
}
if (!frame->img) {
return SWITCH_STATUS_SUCCESS;
}
@ -929,6 +961,15 @@ static switch_status_t vlc_file_write_video(switch_file_handle_t *handle, switch
switch_image_t *img_copy = NULL;
vlc_frame_data_t *fdata = NULL;
if (!vcontext->vid_ready) {
vcontext->vid_ready = 1;
if (switch_mutex_trylock(vcontext->audio_mutex) == SWITCH_STATUS_SUCCESS) {
switch_thread_cond_signal(vcontext->cond);
switch_mutex_unlock(vcontext->audio_mutex);
}
}
switch_img_copy(frame->img, &img_copy);
switch_zmalloc(fdata, sizeof(*fdata));
@ -962,6 +1003,14 @@ static switch_status_t vlc_file_av_write(switch_file_handle_t *handle, void *dat
return SWITCH_STATUS_SUCCESS;
}
if (!vcontext->vid_ready) {
switch_mutex_lock(vcontext->audio_mutex);
if (!vcontext->vid_ready) {
switch_thread_cond_wait(vcontext->cond, vcontext->audio_mutex);
}
switch_mutex_unlock(vcontext->audio_mutex);
}
switch_mutex_lock(vcontext->audio_mutex);
if (!switch_buffer_inuse(vcontext->audio_buffer)) {
switch_core_timer_sync(&vcontext->timer);
@ -1009,14 +1058,23 @@ static switch_status_t vlc_file_av_close(switch_file_handle_t *handle)
vlc_video_context_t *vcontext = acontext->vcontext;
vcontext->ending = 1;
if (vcontext && vcontext->video_queue) {
switch_queue_push(vcontext->video_queue, NULL);
}
if (switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE) && switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO)) {
if (switch_mutex_trylock(vcontext->video_mutex) == SWITCH_STATUS_SUCCESS) {
if (vcontext->cond && switch_mutex_trylock(vcontext->video_mutex) == SWITCH_STATUS_SUCCESS) {
switch_thread_cond_signal(vcontext->cond);
switch_mutex_unlock(vcontext->video_mutex);
}
if (acontext->cond && switch_mutex_trylock(vcontext->audio_mutex) == SWITCH_STATUS_SUCCESS) {
switch_thread_cond_signal(acontext->cond);
switch_mutex_unlock(vcontext->audio_mutex);
}
while(switch_buffer_inuse(vcontext->audio_buffer) || switch_queue_size(vcontext->video_queue)) {
libvlc_state_t status = libvlc_media_get_state(vcontext->m);

4
src/switch_core_file.c
View File

@ -222,6 +222,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_perform_file_open(const char *file,
}
}
if (switch_test_flag(fh, SWITCH_FILE_FLAG_VIDEO) && (flags & SWITCH_FILE_FLAG_WRITE)) {
fh->pre_buffer_datalen = 0;
}
if (fh->pre_buffer_datalen) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Prebuffering %d bytes\n", (int)fh->pre_buffer_datalen);
switch_buffer_create_dynamic(&fh->pre_buffer, fh->pre_buffer_datalen * fh->channels, fh->pre_buffer_datalen * fh->channels, 0);

2
src/switch_core_media.c
View File

@ -1553,7 +1553,7 @@ SWITCH_DECLARE(switch_status_t) switch_media_handle_create(switch_media_handle_t
session->media_handle->mparams = params;
if (!session->media_handle->mparams->video_key_freq) {
session->media_handle->mparams->video_key_freq = 30000000;
session->media_handle->mparams->video_key_freq = 10000000;
}
if (!session->media_handle->mparams->video_key_first) {

16
src/switch_rtp.c
View File

@ -6733,6 +6733,8 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
}
if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc) {
//#define USE_DELTA
#ifdef USE_DELTA
if (rtp_session->ts_norm.last_ssrc) {
rtp_session->ts_norm.delta_ct = 1;
rtp_session->ts_norm.delta_ttl = 0;
@ -6740,18 +6742,24 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
rtp_session->ts_norm.ts += rtp_session->ts_norm.delta;
}
}
#endif
rtp_session->ts_norm.last_ssrc = send_msg->header.ssrc;
rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts);
}
if (ntohl(send_msg->header.ts) != rtp_session->ts_norm.last_frame) {
rtp_session->ts_norm.delta = ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame;
#ifdef USE_DELTA
int32_t delta = (int32_t) (ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame);
if (delta > 0 && delta < 90000) {
rtp_session->ts_norm.delta = delta;
}
//printf("WTF %d\n", rtp_session->ts_norm.delta);
rtp_session->ts_norm.ts += rtp_session->ts_norm.delta;
//switch_core_timer_sync(&rtp_session->timer);
//printf("W00t %d\n", rtp_session->timer.samplecount);
//rtp_session->ts_norm.ts = rtp_session->timer.samplecount;
#else
switch_core_timer_sync(&rtp_session->timer);
rtp_session->ts_norm.ts = rtp_session->timer.samplecount;
#endif
}
rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts);