From 8cbe796831d7ebe2d6efec9d3bbc71b9e0341b3a Mon Sep 17 00:00:00 2001 From: Seven Du Date: Tue, 28 Nov 2017 01:59:09 +0800 Subject: [PATCH] FS-11333: [mod_mp4v2] improvements update mod_mp4v2 fix record video and lipsync refactor to use a dedicated video write thread read frame_size from encoder to support mp3 and aac codec fix audio only mode control bandwidth and fps --- src/mod/applications/mod_mp4v2/mod_mp4v2.c | 703 +++++++-------------- 1 file changed, 234 insertions(+), 469 deletions(-) diff --git a/src/mod/applications/mod_mp4v2/mod_mp4v2.c b/src/mod/applications/mod_mp4v2/mod_mp4v2.c index 69b611c2a3..8faccc4e02 100644 --- a/src/mod/applications/mod_mp4v2/mod_mp4v2.c +++ b/src/mod/applications/mod_mp4v2/mod_mp4v2.c @@ -38,6 +38,7 @@ #define TIMESCALE 1000 #define SampleLenFieldSize 4 +#define PTIME 20 // #define HAS_SPS_PARSER SWITCH_MODULE_LOAD_FUNCTION(mod_mp4v2_load); @@ -138,374 +139,6 @@ static void init_video_track(MP4FileHandle mp4, MP4TrackId *video, switch_frame_ MP4SetVideoProfileLevel(mp4, 0x7F); } -static void record_video_thread(switch_core_session_t *session, void *obj) -{ - struct record_helper *eh = obj; - switch_channel_t *channel = switch_core_session_get_channel(session); - switch_status_t status; - switch_frame_t *read_frame; - uint bytes; - MP4FileHandle mp4; - MP4TrackId video; - unsigned char buf[40960]; - int len = 0; - uint8_t iframe = 0; - uint32_t *size = (uint32_t *)buf; - uint8_t *hdr = NULL; - uint8_t fragment_type; - uint8_t nal_type; - uint8_t start_bit; - uint8_t *sps = NULL; - // uint8_t *pps = NULL; - int sps_set = 0; - int pps_set = 0; - - eh->up = 1; - mp4 = eh->fd; - - /* Tell the channel to request a fresh vid frame */ - switch_core_session_request_video_refresh(session); - - len = 0; - while (switch_channel_ready(channel) && eh->up) { - status = switch_core_session_read_video_frame(session, &read_frame, SWITCH_IO_FLAG_NONE, 0); - - if (!SWITCH_READ_ACCEPTABLE(status)) { - break; - } - - if (switch_test_flag(read_frame, SFF_CNG)) { - continue; - } - - bytes = read_frame->datalen; - - if (bytes > 2000) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "xxxxxxxx buffer overflow\n"); - continue; - } - - hdr = read_frame->data; - fragment_type = hdr[0] & 0x1f; - nal_type = hdr[1] & 0x1f; - start_bit = hdr[1] & 0x80; - iframe = (((fragment_type == 28 || fragment_type == 29) && nal_type == 5 && start_bit == 128) || fragment_type == 5 || fragment_type ==7 || fragment_type ==8) ? 1 : 0; - -#if 0 - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%02x %02x %02x | len:%d m:%d st:%d i:%d\n", hdr[0], hdr[1], hdr[2], bytes, read_frame->m, start_bit, iframe); -#endif - - // echo back - switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); - - if (fragment_type == 7 && !sps_set) { //sps - sps = malloc(bytes); - memcpy(sps, read_frame->data, bytes); - sps_set = 1; - - switch_mutex_lock(eh->mutex); - - init_video_track(mp4, &video, read_frame); - if (video == MP4_INVALID_TRACK_ID) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error add video track!\n"); - switch_mutex_unlock(eh->mutex); - goto end; - } - - switch_mutex_unlock(eh->mutex); - continue; - } else if (fragment_type == 8 && !pps_set) { //pps - switch_mutex_lock(eh->mutex); - MP4AddH264PictureParameterSet(mp4, video, read_frame->data, bytes); - switch_mutex_unlock(eh->mutex); - pps_set = 1; - // continue; - } - - if ((!sps_set) && (!pps_set)) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Waiting for SPS/PPS\n"); - // continue; - } - - len += 4 + read_frame->datalen; - - if (len > 40960) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffer overflow!!!! %d\n", len); - len = 0; - size = (uint32_t *)buf; - continue; - } - - *size = htonl(read_frame->datalen); - memcpy(size + 1, read_frame->data, read_frame->datalen); - - size = (uint32_t *)((uint8_t *)size + 4 + read_frame->datalen); - - if (read_frame->m) { - int duration = 0; - - switch_mutex_lock(eh->mutex); - if (!eh->timer.interval) { - switch_core_timer_init(&eh->timer, "soft", 1, 1, switch_core_session_get_pool(session)); - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init timer\n"); - } else { - switch_core_timer_sync(&eh->timer); - } - switch_mutex_unlock(eh->mutex); - - if (eh->last_pts) { - duration = eh->timer.samplecount - eh->last_pts; - } - eh->last_pts = eh->timer.samplecount; - - switch_mutex_lock(eh->mutex); - MP4WriteSample(mp4, video, buf, len, duration, 0, iframe); - switch_mutex_unlock(eh->mutex); - len = 0; - size = (uint32_t *)buf; - } - - } - -end: - eh->up = 0; - return; -} - -SWITCH_STANDARD_APP(record_mp4_function) -{ - switch_status_t status; - switch_frame_t *read_frame; - switch_channel_t *channel = switch_core_session_get_channel(session); - struct record_helper eh = { 0 }; - MP4FileHandle fd; - MP4TrackId audio; - switch_mutex_t *mutex = NULL; - switch_codec_implementation_t read_impl = { 0 }; - switch_dtmf_t dtmf = { 0 }; - int count = 0, sanity = 30; - switch_codec_t codec; - int duration = 0; - switch_event_t *event; - - memset(&codec, 0, sizeof(switch_codec_t)); - - switch_channel_answer(channel); - switch_core_session_get_read_impl(session, &read_impl); - - switch_channel_set_variable(channel, SWITCH_PLAYBACK_TERMINATOR_USED, ""); - - while (switch_channel_up(channel) && !switch_channel_test_flag(channel, CF_VIDEO)) { - switch_yield(10000); - - if (count) count--; - - if (count == 0) { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s waiting for video.\n", switch_channel_get_name(channel)); - count = 100; - if (!--sanity) { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_WARNING, "%s timeout waiting for video.\n", - switch_channel_get_name(channel)); - switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "Got timeout while waiting for video"); - return; - } - } - } - - if (!switch_channel_ready(channel)) { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_CRIT, "%s not ready.\n", switch_channel_get_name(channel)); - switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "Channel not ready"); - return; - } - - if ((fd = MP4CreateEx((char*)data, 0, 1, 1, NULL, 0, NULL, 0)) == MP4_INVALID_FILE_HANDLE) { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_CRIT, "Error opening file %s\n", (char *) data); - switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "Got error while opening file"); - return; - } - - audio = MP4AddULawAudioTrack(fd, read_impl.samples_per_second); - MP4SetTrackIntegerProperty(fd, audio, "mdia.minf.stbl.stsd.ulaw.channels", read_impl.number_of_channels); - MP4SetTrackIntegerProperty(fd, audio, "mdia.minf.stbl.stsd.ulaw.sampleSize", 8); - - // audio = MP4AddAACAudioTrack(fd, 8000); - // audio = MP4AddAudioTrack(fd, 8000, MP4_INVALID_DURATION, MP4_MPEG4_AUDIO_TYPE); - - // MP4SetTrackIntegerProperty(fd, audio, "mdia.minf.stbl.stsd.aac.channels", 1); - // MP4SetTrackIntegerProperty(fd, audio, "mdia.minf.stbl.stsd.aac.sampleSize", 8); - - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "ms: %d, ts: %d\n", read_impl.microseconds_per_packet, read_impl.samples_per_second /(read_impl.microseconds_per_packet / 1000) / 2); - - /* MP4SetAudioProfileLevel sets the minumum profile/level of MPEG-4 audio support necessary to render the contents of the file. - ISO/IEC 14496-1:2001 MPEG-4 Systems defines the following values: - 0x00 Reserved - 0x01 Main Profile @ Level 1 - 0x02 Main Profile @ Level 2 - 0x03 Main Profile @ Level 3 - 0x04 Main Profile @ Level 4 - 0x05 Scalable Profile @ Level 1 - 0x06 Scalable Profile @ Level 2 - 0x07 Scalable Profile @ Level 3 - 0x08 Scalable Profile @ Level 4 - 0x09 Speech Profile @ Level 1 - 0x0A Speech Profile @ Level 2 - 0x0B Synthesis Profile @ Level 1 - 0x0C Synthesis Profile @ Level 2 - 0x0D Synthesis Profile @ Level 3 - 0x0E-0x7F Reserved - 0x80-0xFD User private - 0xFE No audio profile specified - 0xFF No audio required - */ - // MP4SetAudioProfileLevel(fd, 0x7F); - MP4SetAudioProfileLevel(fd, 0x0F); - if (0) { - // uint8_t c[] = {0x15, 0x88}; // 00010 1011 0001 000 - uint8_t c[] = {0x12, 0x08}; // 00010 0100 0001 000 - MP4SetTrackESConfiguration(fd, audio, c, sizeof(c)); - } - - if (read_impl.ianacode != 0) { - if (switch_core_codec_init(&codec, - "PCMU", - NULL, - NULL, - read_impl.samples_per_second, - read_impl.microseconds_per_packet / 1000, - read_impl.number_of_channels, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE, - NULL, switch_core_session_get_pool(session)) == SWITCH_STATUS_SUCCESS) { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Audio Codec Activation Success\n"); - } else { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Audio Codec Activation Fail\n"); - switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "Audio codec activation failed"); - goto end; - } - switch_core_session_set_read_codec(session, &codec); - } - - if (switch_channel_test_flag(channel, CF_VIDEO)) { - switch_codec_implementation_t video_read_impl = { 0 }; - - switch_core_session_get_video_read_impl(session, &video_read_impl); - - if (video_read_impl.iananame && (!strcasecmp(video_read_impl.iananame, "H264"))) { - switch_channel_set_flag(channel, CF_VIDEO_PASSIVE); - switch_channel_set_flag(channel, CF_VIDEO_READY); - - switch_mutex_init(&mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session)); - eh.mutex = mutex; - eh.fd = fd; - switch_core_media_start_video_function(session, record_video_thread, &eh); - } else { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "only H264 video is supported, you have %s\n", video_read_impl.iananame); - switch_channel_set_flag(channel, CF_VIDEO_ECHO); - switch_channel_set_flag(channel, CF_VIDEO_READY); - } - } - - if (switch_event_create(&event, SWITCH_EVENT_RECORD_START) == SWITCH_STATUS_SUCCESS) { - switch_channel_event_set_data(channel, event); - switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Record-File-Path", (char *)data); - switch_event_fire(&event); - } - - while (switch_channel_ready(channel)) { - status = switch_core_session_read_frame(session, &read_frame, SWITCH_IO_FLAG_SINGLE_READ, 0); - - if (switch_channel_test_flag(channel, CF_BREAK)) { - switch_channel_clear_flag(channel, CF_BREAK); - eh.up = 0; - break; - } - - switch_ivr_parse_all_events(session); - - //check for dtmf interrupts - if (switch_channel_has_dtmf(channel)) { - const char * terminators = switch_channel_get_variable(channel, SWITCH_PLAYBACK_TERMINATORS_VARIABLE); - switch_channel_dequeue_dtmf(channel, &dtmf); - - if (terminators && !strcasecmp(terminators, "none")) { - terminators = NULL; - } - - if (terminators && strchr(terminators, dtmf.digit)) { - char sbuf[2] = {dtmf.digit, '\0'}; - switch_channel_set_variable(channel, SWITCH_PLAYBACK_TERMINATOR_USED, sbuf); - eh.up = 0; - break; - } - } - - if (!SWITCH_READ_ACCEPTABLE(status)) { - eh.up = 0; - break; - } - - switch_core_session_write_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); - - if (switch_test_flag(read_frame, SFF_CNG)) { - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cng, datalen:%d\n", read_frame->datalen); - continue; - // memset(read_frame->data, 0x0, 160); - } - - if (mutex) switch_mutex_lock(mutex); - - if (!eh.timer.interval) { - switch_core_timer_init(&eh.timer, "soft", 1, 1, switch_core_session_get_pool(session)); - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init timer\n"); - } - - duration = read_frame->datalen / read_impl.number_of_channels; - - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d %d\n", duration, read_frame->datalen); - - MP4WriteSample(fd, audio, read_frame->data, read_frame->datalen, duration, 0, 1); - - if (mutex) switch_mutex_unlock(mutex); - } - - switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "OK"); - - if (switch_event_create(&event, SWITCH_EVENT_RECORD_STOP) == SWITCH_STATUS_SUCCESS) { - switch_channel_event_set_data(channel, event); - switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Record-File-Path", (char *)data); - switch_event_fire(&event); - } - -end: - - if (eh.up) { - while (eh.up) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "waiting video thread to be done ...\n"); - switch_yield(100000); - switch_cond_next(); - } - } - - if (eh.timer.interval) { - switch_core_timer_destroy(&eh.timer); - } - - switch_core_media_end_video_function(session); - switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE); - - switch_core_session_set_read_codec(session, NULL); - - if (fd != MP4_INVALID_FILE_HANDLE) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "closing file %s\n", (char *)data); - MP4Close(fd, 0); - } - - if (switch_core_codec_ready(&codec)) switch_core_codec_destroy(&codec); -} - -// SWITCH_STANDARD_APP(play_mp4_function) -// { - -// } - static inline char *get_audio_codec_name(uint8_t audio_type) { switch (audio_type) { @@ -540,14 +173,17 @@ static int get_aac_sample_rate_index(unsigned int sampleRate) } struct mp4_file_context { + switch_file_handle_t *handle; switch_memory_pool_t *pool; MP4FileHandle fd; MP4TrackId audio; MP4TrackId video; + uint32_t audio_frame_size; switch_codec_t audio_codec; switch_codec_t video_codec; switch_mutex_t *mutex; switch_buffer_t *buf; + uint32_t last_chunk_size; int sps_set; int pps_set; switch_timer_t timer; @@ -556,11 +192,92 @@ struct mp4_file_context { int audio_start; uint8_t audio_type; // MP4 Audio Type MP4Duration audio_duration; - switch_image_t *last_img; + switch_thread_t *video_thread; + switch_queue_t *video_queue; }; typedef struct mp4_file_context mp4_file_context_t; +static switch_status_t do_write_video(switch_file_handle_t *handle, switch_frame_t *frame); + +static void *SWITCH_THREAD_FUNC video_write_thread_run(switch_thread_t *thread, void *obj) +{ + mp4_file_context_t *context = (mp4_file_context_t *)obj; + void *pop = NULL; + switch_image_t *last_img = NULL; + switch_status_t status = SWITCH_STATUS_SUCCESS; + switch_status_t encode_status = SWITCH_STATUS_SUCCESS; + uint8_t data[SWITCH_DEFAULT_VIDEO_SIZE]; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video_write_thread start\n"); + + while (switch_queue_pop(context->video_queue, &pop) == SWITCH_STATUS_SUCCESS) { + switch_frame_t frame = { 0 }; + + if (!pop) break; + + if (!last_img) { // first img + last_img = (switch_image_t *)pop; + continue; + } + + frame.data = data; + frame.img = last_img; + // switch_set_flag(&frame, SFF_DYNAMIC); + + do { + frame.datalen = SWITCH_DEFAULT_VIDEO_SIZE; + encode_status = switch_core_codec_encode_video(&context->video_codec, &frame); + + if (encode_status == SWITCH_STATUS_SUCCESS || encode_status == SWITCH_STATUS_MORE_DATA) { + switch_assert((encode_status == SWITCH_STATUS_SUCCESS && frame.m) || !frame.m); + + if (frame.datalen == 0) break; + + status = do_write_video(context->handle, &frame); + } + } while(status == SWITCH_STATUS_SUCCESS && encode_status == SWITCH_STATUS_MORE_DATA); + + switch_img_free(&last_img); + last_img = (switch_image_t *)pop; + } + + switch_img_free(&last_img); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video_write_thread done\n"); + + return NULL; +} + +static void launch_video_write_thread(mp4_file_context_t *context, switch_memory_pool_t *pool) +{ + //switch_thread_t *thread; + switch_threadattr_t *thd_attr = NULL; + + switch_threadattr_create(&thd_attr, pool); + switch_threadattr_detach_set(thd_attr, 1); + switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); + // switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME); + switch_thread_create(&context->video_thread, thd_attr, video_write_thread_run, context, pool); +} + + +static int flush_video_queue(switch_queue_t *q, int min) +{ + void *pop; + + if (switch_queue_size(q) > min) { + while (switch_queue_trypop(q, &pop) == SWITCH_STATUS_SUCCESS) { + switch_image_t *img = (switch_image_t *) pop; + switch_img_free(&img); + if (min && switch_queue_size(q) <= min) { + break; + } + } + } + + return switch_queue_size(q); +} + static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *path) { mp4_file_context_t *context; @@ -580,7 +297,9 @@ static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *p memset(context, 0, sizeof(mp4_file_context_t)); - context->offset = -100; + context->handle = handle; + context->offset = 0; + if (handle->params && (tmp = switch_event_get_header(handle->params, "mp4v2_video_offset"))) { context->offset = atoi(tmp); } @@ -614,6 +333,22 @@ static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *p flags |= SWITCH_FOPEN_READ; } + if (handle->mm.samplerate) { + handle->samplerate = handle->mm.samplerate; + } else { + handle->mm.samplerate = handle->samplerate; + } + + if (!handle->mm.ab) { + handle->mm.ab = 128; + } + + if (!handle->mm.vb) { + handle->mm.vb = switch_calc_bitrate(handle->mm.vw, handle->mm.vh, 1, handle->mm.fps); + } + + // MP4_CREATE_64BIT_DATA if file > 4G + if ((context->fd = MP4CreateEx(path, 0, 1, 1, NULL, 0, NULL, 0)) == MP4_INVALID_FILE_HANDLE) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Error opening file %s\n", path); return SWITCH_STATUS_GENERR; @@ -651,7 +386,7 @@ static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *p info |= handle->channels << 3; //(4bit) info = htons(info); - context->audio = MP4AddAudioTrack(context->fd, handle->samplerate, handle->samplerate, MP4_MPEG4_AUDIO_TYPE); + context->audio = MP4AddAudioTrack(context->fd, handle->samplerate, 1024, MP4_MPEG4_AUDIO_TYPE); MP4SetTrackESConfiguration(context->fd, context->audio, (uint8_t *)&info, sizeof(info)); MP4SetTrackIntegerProperty(context->fd, context->audio, "mdia.minf.stbl.stsd.mp4a.channels", handle->channels); } @@ -672,7 +407,7 @@ static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *p NULL, NULL, handle->samplerate, - 20,//ms + PTIME,//ms handle->channels, SWITCH_CODEC_FLAG_ENCODE, NULL, handle->memory_pool) == SWITCH_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Audio Codec Activation Success\n"); @@ -681,7 +416,21 @@ static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *p goto end; } + if (context->audio_type == MP4_MP3_AUDIO_TYPE) { // fetch frame size + uint32_t size; + uint32_t flag = 0xFFFFFFFF; + + switch_core_codec_encode(&context->audio_codec, NULL, &flag, 0, 0, + (void *)&context->audio_frame_size, &size, NULL, &flag); + } else if (context->audio_type == MP4_MPEG4_AUDIO_TYPE) { + context->audio_frame_size = 1024; + } + if (switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO)) { + switch_codec_settings_t codec_settings = {{ 0 }}; + codec_settings.video.bandwidth = handle->mm.vb; + codec_settings.video.fps = handle->mm.fps; + if (switch_core_codec_init(&context->video_codec, "H264", NULL, @@ -689,12 +438,15 @@ static switch_status_t mp4_file_open(switch_file_handle_t *handle, const char *p 90000, 0,//ms 1, SWITCH_CODEC_FLAG_ENCODE, - NULL, handle->memory_pool) == SWITCH_STATUS_SUCCESS) { + &codec_settings, handle->memory_pool) == SWITCH_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Video Codec H264 Activation Success\n"); } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Video Codec H264 Activation Fail\n"); goto end; } + + switch_queue_create(&context->video_queue, 60, handle->memory_pool); + launch_video_write_thread(context, handle->memory_pool); } if (switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) { @@ -729,9 +481,10 @@ static switch_status_t mp4_file_truncate(switch_file_handle_t *handle, int64_t o static switch_status_t mp4_file_close(switch_file_handle_t *handle) { mp4_file_context_t *context = handle->private_info; + switch_status_t status; if (context->fd) { - MP4Close(context->fd, 0); + MP4Close(context->fd, MP4_CLOSE_DO_NOT_COMPUTE_BITRATE); context->fd = NULL; } @@ -742,7 +495,15 @@ static switch_status_t mp4_file_close(switch_file_handle_t *handle) switch_core_timer_destroy(&context->timer); } - switch_img_free(&context->last_img); + if (context->video_queue) { + switch_queue_term(context->video_queue); + flush_video_queue(context->video_queue, 0); + } + + if (context->video_thread) { + switch_thread_join(&status, context->video_thread); + } + switch_buffer_destroy(&context->buf); return SWITCH_STATUS_SUCCESS; @@ -768,9 +529,7 @@ static switch_status_t mp4_file_write(switch_file_handle_t *handle, void *data, uint32_t encoded_rate; mp4_file_context_t *context = handle->private_info; uint32_t size = 0; - - context->audio_duration += *len; - + uint32_t flag = 0; if (context->audio_type == MP4_PCM16_LITTLE_ENDIAN_AUDIO_TYPE) { size = datalen; @@ -779,7 +538,7 @@ static switch_status_t mp4_file_write(switch_file_handle_t *handle, void *data, switch_core_codec_encode(&context->audio_codec, NULL, data, datalen, handle->samplerate, - buf, &size, &encoded_rate, NULL); + buf, &size, &encoded_rate, &flag); } switch_mutex_lock(context->mutex); @@ -787,40 +546,14 @@ static switch_status_t mp4_file_write(switch_file_handle_t *handle, void *data, if (!context->timer.interval) { switch_core_timer_init(&context->timer, "soft", 1, 1, context->pool); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init timer\n"); - } else if(!context->audio_start) { // try make up some sampels if the video already start - int i, count; - uint8_t buf0[SWITCH_RECOMMENDED_BUFFER_SIZE] = { 0 }; - - context->audio_start++; - switch_core_timer_sync(&context->timer); - - count = context->timer.samplecount - context->offset; - - if (count > 0) { - count /= *len; - } - - if (context->audio_type != MP4_ULAW_AUDIO_TYPE) { - count = 0; // todo: make this feature work for mp3/aac - } - - if (count){ - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video is short, make up %lu samples\n", count * (*len)); - MP4WriteSample(context->fd, context->audio, buf0, size, 0, 0, 1); - } - - for (i = 1; i < count; i++) { - MP4WriteSample(context->fd, context->audio, buf0, size, *len, 0, 1); - } } - if (context->audio_type == MP4_MPEG4_AUDIO_TYPE && size == 0) { - // don't write 0 - } else { - MP4WriteSample(context->fd, context->audio, buf, size, context->audio_duration, 0, 1); - context->audio_duration = 0; + if (size > 0) { + MP4WriteSample(context->fd, context->audio, buf, size, context->audio_frame_size ? context->audio_frame_size : *len, 0, 1); } + context->audio_duration += *len; + switch_mutex_unlock(context->mutex); return status; @@ -841,115 +574,150 @@ static switch_status_t do_write_video(switch_file_handle_t *handle, switch_frame uint8_t fragment_type; uint8_t nal_type; uint8_t start_bit; + uint8_t end_bit; mp4_file_context_t *context = handle->private_info; hdr = (uint8_t *)frame->data; fragment_type = hdr[0] & 0x1f; nal_type = hdr[1] & 0x1f; start_bit = hdr[1] & 0x80; - is_iframe = (((fragment_type == 28 || fragment_type == 29) && nal_type == 5 && start_bit == 128) || fragment_type == 5 || fragment_type ==7 || fragment_type ==8) ? 1 : 0; + end_bit = hdr[1] & 0x40; + + is_iframe = fragment_type == 5 || (fragment_type == 28 && nal_type == 5); // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%02x %02x %02x | len:%d m:%d st:%d i:%d\n", hdr[0], hdr[1], hdr[2], datalen, frame->m, start_bit, is_iframe); - size = htonl(datalen); - switch_buffer_write(context->buf, &size, 4); - switch_buffer_write(context->buf, hdr, datalen); - - switch_mutex_lock(context->mutex); - if (fragment_type == 7 && !context->sps_set) { //sps context->sps_set = 1; init_video_track(context->fd, &context->video, frame); if (context->video == MP4_INVALID_TRACK_ID) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error add video track!\n"); - switch_goto_status(SWITCH_STATUS_FALSE, end); + return SWITCH_STATUS_FALSE; } } else if (fragment_type == 8 && context->sps_set && !context->pps_set) { //pps MP4AddH264PictureParameterSet(context->fd, context->video, hdr, datalen); context->pps_set = 1; } - if (nal_type == 7 || nal_type == 8 || frame->m == 0) { - } else if (context->sps_set && context->pps_set) { + if (fragment_type == 28) { + if (start_bit && end_bit) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "WTF?\n"); + } + + if (start_bit) { + nal_type |= (hdr[0] & 0x60); + + size = htonl(datalen); + switch_buffer_write(context->buf, &size, 4); + switch_buffer_write(context->buf, &nal_type, 1); + switch_buffer_write(context->buf, hdr + 2, datalen - 2); + context->last_chunk_size = datalen - 1; + } else if (end_bit) { + uint32_t used; + const void *data; + uint32_t *chunk_size = NULL; + + switch_buffer_write(context->buf, hdr + 2, datalen - 2); + context->last_chunk_size += datalen - 2; + used = switch_buffer_inuse(context->buf); + switch_buffer_peek_zerocopy(context->buf, &data); + chunk_size = (uint32_t *)((uint8_t *)data + used - context->last_chunk_size - 4); + *chunk_size = htonl(context->last_chunk_size); + } else { + switch_buffer_write(context->buf, hdr + 2, datalen - 2); + context->last_chunk_size += datalen - 2; + } + } else { + size = htonl(datalen); + switch_buffer_write(context->buf, &size, 4); + switch_buffer_write(context->buf, hdr, datalen); + } + + if (!frame->m) { + return SWITCH_STATUS_SUCCESS; + } + + switch_mutex_lock(context->mutex); + + if (context->sps_set && context->pps_set) { uint32_t used = switch_buffer_inuse(context->buf); const void *data; int duration = 0; + int ts = 0; - if (!context->timer.interval) { - switch_core_timer_init(&context->timer, "soft", 1, 1, context->pool); - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init timer\n"); + if (frame->img && frame->img->user_priv) { + ts = *(int *)frame->img->user_priv; } else { switch_core_timer_sync(&context->timer); + ts = context->timer.samplecount; } - duration = context->timer.samplecount - context->last_pts; + duration = ts - context->last_pts; + + if (duration <= 0) duration = 1; // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "samplecount: %u, duration: %u\n", context->timer.samplecount, duration); switch_buffer_peek_zerocopy(context->buf, &data); - if (context->last_pts == 0) { // first img, write at the very beginning so we don't see blank screen - duration /= 2; - MP4WriteSample(context->fd, context->video, data, used, duration, 0, is_iframe); + context->last_pts = ts; - if (duration > context->offset) { - duration -= context->offset; - } else { - duration = 0; - } - } - - context->last_pts = context->timer.samplecount; - - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "samplecount: %u, duration: %u\n", context->timer.samplecount, duration); - - if (duration) { - MP4WriteSample(context->fd, context->video, data, used, duration, 0, is_iframe); - } + MP4WriteSample(context->fd, context->video, data, used, duration, 0, is_iframe); switch_buffer_zero(context->buf); } -end: switch_mutex_unlock(context->mutex); + { + int delta = context->timer.samplecount * (handle->samplerate / 1000) - context->audio_duration; + + if (delta > (int)handle->samplerate) { + uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE] = { 0 }; + size_t samples = handle->samplerate / 1000 * PTIME; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "missed audio %d samples at %d\n", delta, (int)context->audio_duration / (handle->samplerate / 1000)); + + while ((delta -= samples) > 0) { + mp4_file_write(handle, data, &samples); + samples = handle->samplerate / 1000 * PTIME; + } + } + } + return status; } static switch_status_t mp4_file_write_video(switch_file_handle_t *handle, switch_frame_t *frame) { - switch_status_t status = SWITCH_STATUS_SUCCESS; - switch_status_t encode_status = SWITCH_STATUS_SUCCESS; mp4_file_context_t *context = handle->private_info; if (!frame->img) { return do_write_video(handle, frame); } else { - switch_frame_t eframe = { 0 }; - uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; + switch_image_t *img = NULL; - - if (!context->last_img) { - switch_img_copy(frame->img, &context->last_img); - return status; + if (!context->timer.interval) { + switch_mutex_lock(context->mutex); + switch_core_timer_init(&context->timer, "soft", 1, 1, context->pool); + switch_mutex_unlock(context->mutex); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init timer\n"); + } else { + switch_mutex_lock(context->mutex); + switch_core_timer_sync(&context->timer); + switch_mutex_unlock(context->mutex); } - eframe.data = data + 12; - eframe.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE - 12; - eframe.img = context->last_img; - do { - frame->datalen = SWITCH_DEFAULT_VIDEO_SIZE; - encode_status = switch_core_codec_encode_video(&context->video_codec, &eframe); + switch_img_copy(frame->img, &img); + switch_assert(img); + img->user_priv = malloc(sizeof(int)); + *(int *)img->user_priv = context->timer.samplecount; - if (encode_status == SWITCH_STATUS_SUCCESS || encode_status == SWITCH_STATUS_MORE_DATA) { - switch_assert((encode_status == SWITCH_STATUS_SUCCESS && eframe.m) || !eframe.m); - if (eframe.datalen > 0) status = do_write_video(handle, &eframe); - } - } while(status == SWITCH_STATUS_SUCCESS && encode_status == SWITCH_STATUS_MORE_DATA); + if (switch_queue_trypush(context->video_queue, img) != SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "video queue full, discard one frame\n"); + } } - switch_img_copy(frame->img, &context->last_img); - - return status; + return SWITCH_STATUS_SUCCESS; } static switch_status_t mp4_file_set_string(switch_file_handle_t *handle, switch_audio_col_t col, const char *string) @@ -962,14 +730,14 @@ static switch_status_t mp4_file_get_string(switch_file_handle_t *handle, switch_ return SWITCH_STATUS_FALSE; } -static char *supported_formats[2] = { 0 }; +static char *supported_formats[3] = { 0 }; SWITCH_MODULE_LOAD_FUNCTION(mod_mp4v2_load) { - switch_application_interface_t *app_interface; switch_file_interface_t *file_interface; - supported_formats[0] = "mp4"; + supported_formats[0] = "mp4v2"; + supported_formats[1] = "mp4"; /* connect my internal structure to the blank pointer passed to me */ *module_interface = switch_loadable_module_create_module_interface(pool, modname); @@ -988,9 +756,6 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_mp4v2_load) file_interface->file_set_string = mp4_file_set_string; file_interface->file_get_string = mp4_file_get_string; - // SWITCH_ADD_APP(app_interface, "play_mp4", "play an mp4 file", "play an mp4 file", play_mp4_function, "", SAF_NONE); - SWITCH_ADD_APP(app_interface, "record_mp4", "record an mp4 file", "record an mp4 file", record_mp4_function, "", SAF_NONE); - /* indicate that the module should continue to be loaded */ return SWITCH_STATUS_SUCCESS; }