diff --git a/src/mod/applications/mod_av/.gitignore b/src/mod/applications/mod_av/.gitignore new file mode 100644 index 0000000000..35829e418c --- /dev/null +++ b/src/mod/applications/mod_av/.gitignore @@ -0,0 +1,3 @@ +test/test_BT7.mp4 +test/test_RGB.mp4 +test/test_packetizer diff --git a/src/mod/applications/mod_av/Makefile.am b/src/mod/applications/mod_av/Makefile.am index ebf4c93929..39cb581611 100644 --- a/src/mod/applications/mod_av/Makefile.am +++ b/src/mod/applications/mod_av/Makefile.am @@ -13,7 +13,7 @@ endif noinst_LTLIBRARIES = libavmod.la -libavmod_la_SOURCES = mod_av.c avformat.c avcodec.c +libavmod_la_SOURCES = mod_av.c avformat.c avcodec.c switch_packetizer.c libavmod_la_CFLAGS = $(AM_CFLAGS) $(AVFORMAT_CFLAGS) $(AVCODEC_CFLAGS) $(SWSCALE_CFLAGS) $(AVUTIL_CFLAGS) $(RESAMPLE_CFLAGS) mod_LTLIBRARIES = mod_av.la @@ -22,13 +22,16 @@ mod_av_la_LIBADD = libavmod.la $(switch_builddir)/libfreeswitch.la $(AVFORMAT_ mod_av_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lm -lz -noinst_PROGRAMS = test/test_mod_av test/test_avformat +noinst_PROGRAMS = test/test_mod_av test/test_avformat test/test_packetizer test_test_mod_av_CFLAGS = $(SWITCH_AM_CFLAGS) -I../ -DSWITCH_TEST_BASE_DIR_FOR_CONF=\"${abs_builddir}/test\" -DSWITCH_TEST_BASE_DIR_OVERRIDE=\"${abs_builddir}/test\" $(AVFORMAT_CFLAGS) $(AVCODEC_CFLAGS) $(SWSCALE_CFLAGS) $(AVUTIL_CFLAGS) $(RESAMPLE_CFLAGS) test_test_mod_av_LDFLAGS = $(AVFORMAT_LIBS) $(AVCODEC_LIBS) $(SWSCALE_LIBS) $(AVUTIL_LIBS) $(RESAMPLE_LIBS) -avoid-version -no-undefined $(SWITCH_AM_LDFLAGS) test_test_mod_av_LDADD = libavmod.la $(switch_builddir)/libfreeswitch.la test_test_avformat_CFLAGS = $(SWITCH_AM_CFLAGS) -I../ -DSWITCH_TEST_BASE_DIR_FOR_CONF=\"${abs_builddir}/test\" -DSWITCH_TEST_BASE_DIR_OVERRIDE=\"${abs_builddir}/test\" $(AVFORMAT_CFLAGS) $(AVCODEC_CFLAGS) $(SWSCALE_CFLAGS) $(AVUTIL_CFLAGS) $(RESAMPLE_CFLAGS) test_test_avformat_LDFLAGS = $(AVFORMAT_LIBS) $(AVCODEC_LIBS) $(SWSCALE_LIBS) $(AVUTIL_LIBS) $(RESAMPLE_LIBS) -avoid-version -no-undefined $(SWITCH_AM_LDFLAGS) test_test_avformat_LDADD = libavmod.la $(switch_builddir)/libfreeswitch.la +test_test_packetizer_CFLAGS = $(SWITCH_AM_CFLAGS) -I../ -DSWITCH_TEST_BASE_DIR_FOR_CONF=\"${abs_builddir}/test\" -DSWITCH_TEST_BASE_DIR_OVERRIDE=\"${abs_builddir}/test\" $(AVFORMAT_CFLAGS) $(AVCODEC_CFLAGS) $(SWSCALE_CFLAGS) $(AVUTIL_CFLAGS) $(RESAMPLE_CFLAGS) +test_test_packetizer_LDFLAGS = $(AVFORMAT_LIBS) $(AVCODEC_LIBS) $(SWSCALE_LIBS) $(AVUTIL_LIBS) $(RESAMPLE_LIBS) -avoid-version -no-undefined $(SWITCH_AM_LDFLAGS) +test_test_packetizer_LDADD = libavmod.la $(switch_builddir)/libfreeswitch.la TESTS = $(noinst_PROGRAMS) diff --git a/src/mod/applications/mod_av/avformat.c b/src/mod/applications/mod_av/avformat.c index c979e8cce6..b8ed73321f 100644 --- a/src/mod/applications/mod_av/avformat.c +++ b/src/mod/applications/mod_av/avformat.c @@ -54,6 +54,9 @@ GCC_DIAG_OFF(deprecated-declarations) #include #endif +#include +#define SLICE_SIZE (SWITCH_DEFAULT_VIDEO_SIZE + 100) + GCC_DIAG_ON(deprecated-declarations) #define SCALE_FLAGS SWS_BICUBIC #define DFT_RECORD_OFFSET 0 @@ -116,8 +119,6 @@ typedef struct record_helper_s { uint64_t last_ts; } record_helper_t; - - /* file interface */ struct av_file_context { @@ -161,6 +162,11 @@ struct av_file_context { switch_time_t last_vid_write; int audio_timer; + + switch_bool_t no_video_decode; + switch_queue_t *video_pkt_queue; + switch_packetizer_t *packetizer; + AVPacket *last_read_pkt; }; typedef struct av_file_context av_file_context_t; @@ -772,6 +778,15 @@ static int flush_video_queue(switch_queue_t *q, int min) return switch_queue_size(q); } +static void flush_video_pkt_queue(switch_queue_t *q) +{ + AVPacket *pkt; + + while (switch_queue_trypop(q, (void **)&pkt) == SWITCH_STATUS_SUCCESS) { + av_packet_unref(pkt); + } +} + static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *obj) { av_file_context_t *context = (av_file_context_t *) obj; @@ -1387,6 +1402,34 @@ GCC_DIAG_ON(deprecated-declarations) if (context->has_video && pkt.stream_index == context->video_st.st->index) { AVFrame *vframe; switch_image_t *img; + + if (context->no_video_decode) { + if (eof) { + break; + } else { + switch_status_t status; + AVPacket *new_pkt = malloc(sizeof(AVPacket)); + + if (0) { // debug + uint8_t *p = pkt.data; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %u %x %x %x %x %x %x\n", pkt.size, *p, *(p+1), *(p+2), *(p+3), *(p+4), *(p+5)); + } + + av_init_packet(new_pkt); + av_packet_ref(new_pkt, &pkt); + status = switch_queue_push(context->video_pkt_queue, new_pkt); + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %4u flag=%x pts=%" SWITCH_INT64_T_FMT " dts=%" SWITCH_INT64_T_FMT "\n", pkt.size, pkt.flags, pkt.pts, pkt.dts); + + context->vid_ready = 1; + if (status != SWITCH_STATUS_SUCCESS) { + av_packet_unref(new_pkt); + free(new_pkt); + } + av_packet_unref(&pkt); + continue; + } + } + if (!sync) { switch_buffer_zero(context->audio_buffer); sync = 1; @@ -1680,6 +1723,11 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa if (context->has_video) { switch_queue_create(&context->eh.video_queue, context->read_fps, handle->memory_pool); + context->no_video_decode = handle->params && switch_true(switch_event_get_header(handle->params, "no_video_decode")); + if (context->no_video_decode) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Opening video in no decode mode\n"); + switch_queue_create(&context->video_pkt_queue, 120 * 5, handle->memory_pool); + } switch_mutex_init(&context->eh.mutex, SWITCH_MUTEX_NESTED, handle->memory_pool); switch_core_timer_init(&context->video_timer, "soft", (int)(1000.0f / context->read_fps), 1, context->pool); } @@ -2177,6 +2225,15 @@ static switch_status_t av_file_close(switch_file_handle_t *handle) context->file_read_thread_running = 0; } + if (context->video_pkt_queue) { + flush_video_pkt_queue(context->video_pkt_queue); + switch_queue_term(context->video_pkt_queue); + } + + if (context->packetizer) { + switch_packetizer_close(&context->packetizer); + } + if (context->file_read_thread) { switch_thread_join(&status, context->file_read_thread); context->file_read_thread = NULL; @@ -2249,6 +2306,7 @@ static switch_status_t av_file_read(switch_file_handle_t *handle, void *data, si } switch_mutex_lock(context->mutex); + while (!context->file_read_thread_started) { switch_thread_cond_wait(context->cond, context->mutex); } @@ -2334,6 +2392,82 @@ static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_f } #else +static switch_status_t no_video_decode_packets(switch_file_handle_t *handle, switch_frame_t *frame, switch_video_read_flag_t flags) +{ + av_file_context_t *context = (av_file_context_t *)handle->private_info; + MediaStream *mst = &context->video_st; + AVStream *st = mst->st; + // AVCodecContext *ctx = st->codec; + // int ticks = 0; + // int64_t max_delta = 1 * AV_TIME_BASE; // 1 second + switch_status_t status = SWITCH_STATUS_SUCCESS; + AVPacket *pkt; + int64_t pts; + + + if (!context->packetizer) { + // uint8_t *p = st->codecpar->extradata; + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %u %x %x %x %x %x %x\n", st->codecpar->extradata_size, *p, *(p+1), *(p+2), *(p+3), *(p+4), *(p+5)); + + context->packetizer = switch_packetizer_create(SPT_H264_SIZED_BITSTREAM, SLICE_SIZE); + if (!context->packetizer) return SWITCH_STATUS_FALSE; + + switch_packetizer_feed_extradata(context->packetizer, st->codecpar->extradata, st->codecpar->extradata_size); + } + + if (context->last_read_pkt) { + status = switch_packetizer_read(context->packetizer, frame); + if (status == SWITCH_STATUS_SUCCESS) { + av_packet_unref(context->last_read_pkt); + free(context->last_read_pkt); + context->last_read_pkt = NULL; + } + return status; + } + + status = switch_queue_trypop(context->video_pkt_queue, (void **)&pkt); + + if (status != SWITCH_STATUS_SUCCESS || !pkt) { + switch_cond_next(); + return SWITCH_STATUS_BREAK; + } + + context->last_read_pkt = pkt; + status = switch_packetizer_feed(context->packetizer, pkt->data, pkt->size); + status = switch_packetizer_read(context->packetizer, frame); + pts = av_rescale_q(pkt->pts, st->time_base, AV_TIME_BASE_Q); + frame->timestamp = pts * 9 / 100; // scale to sample 900000 + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "pts=%" SWITCH_INT64_T_FMT " status = %d\n", pts, status); + + if (status == SWITCH_STATUS_SUCCESS) { + av_packet_unref(context->last_read_pkt); + free(context->last_read_pkt); + context->last_read_pkt = NULL; + } + + + if (status == SWITCH_STATUS_SUCCESS || status == SWITCH_STATUS_MORE_DATA) { + if (!context->video_start_time) { + context->video_start_time = switch_time_now() - pts; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "set start time: %" SWITCH_INT64_T_FMT " now: %" SWITCH_INT64_T_FMT " pts: %" SWITCH_INT64_T_FMT "\n", context->video_start_time, switch_time_now(), pts); + } else if (flags & SVR_BLOCK) { + int64_t sleep = pts - (switch_time_now() - context->video_start_time); + if (sleep > 0) { + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "zzZ... %" SWITCH_INT64_T_FMT "\n", sleep); + if (sleep > 1000000) { + sleep = 1000000; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "but zzZ... %" SWITCH_INT64_T_FMT " at most\n", sleep); + } + switch_yield(sleep); + } else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "video is late %" SWITCH_INT64_T_FMT "\n", sleep); + } + } + } + + return status; +} + static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_frame_t *frame, switch_video_read_flag_t flags) { av_file_context_t *context = (av_file_context_t *)handle->private_info; @@ -2353,6 +2487,13 @@ static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_f return SWITCH_STATUS_BREAK; } + if (context->no_video_decode) { + switch_set_flag(frame, SFF_ENCODED); + status = no_video_decode_packets(handle, frame, flags); + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "return len=%4u nalu=%02x m=%d ts=%u\n", frame->datalen, *(uint8_t *)frame->data, frame->m, frame->timestamp); + return status; + } + if (handle->mm.fps > 0 && handle->mm.fps < smaller_ts) { smaller_ts = handle->mm.fps; } diff --git a/src/mod/applications/mod_av/switch_packetizer.c b/src/mod/applications/mod_av/switch_packetizer.c new file mode 100644 index 0000000000..e539bd9b59 --- /dev/null +++ b/src/mod/applications/mod_av/switch_packetizer.c @@ -0,0 +1,394 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2020, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Seven Du + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Seven Du + * + * switch_packetizer H264 packetizer + * + */ + +#include +#define MAX_NALUS 256 + +typedef struct our_h264_nalu_s { + const uint8_t *start; + const uint8_t *eat; + uint32_t len; +} our_h264_nalu_t; + +typedef struct h264_packetizer_s { + switch_packetizer_bitstream_t type; + uint32_t slice_size; + int nalu_current_index; + our_h264_nalu_t nalus[MAX_NALUS]; + uint8_t *extradata; + uint32_t extradata_size; + uint8_t *sps; + uint8_t *pps; + uint32_t sps_len; + uint32_t pps_len; + int sps_sent; + int pps_sent; +} h264_packetizer_t; + +/* ff_avc_find_startcode is not exposed in the ffmpeg lib but you can use it + Either include the avc.h which available in the ffmpeg source, or + just add the declaration like we does following to avoid include that whole avc.h + The function is implemented in avc.h, guess we'll get rid of this later if we can directly use libx264 + +#include +*/ + +const uint8_t *ff_avc_find_startcode(const uint8_t *p, const uint8_t *end); + +static const uint8_t *fs_avc_find_startcode_internal(const uint8_t *p, const uint8_t *end) +{ + const uint8_t *a = p + 4 - ((intptr_t)p & 3); + + for (end -= 3; p < a && p < end; p++) { + if (p[0] == 0 && p[1] == 0 && p[2] == 1) + return p; + } + + for (end -= 3; p < end; p += 4) { + uint32_t x = *(const uint32_t*)p; + if ((x - 0x01010101) & (~x) & 0x80808080) { + if (p[1] == 0) { + if (p[0] == 0 && p[2] == 1) + return p; + if (p[2] == 0 && p[3] == 1) + return p+1; + } + if (p[3] == 0) { + if (p[2] == 0 && p[4] == 1) + return p+2; + if (p[4] == 0 && p[5] == 1) + return p+3; + } + } + } + + for (end += 3; p < end; p++) { + if (p[0] == 0 && p[1] == 0 && p[2] == 1) + return p; + } + + return end + 3; +} + +static const uint8_t *fs_avc_find_startcode(const uint8_t *p, const uint8_t *end) +{ + const uint8_t *out= fs_avc_find_startcode_internal(p, end); + + if (p < out && out < end && !out[-1]) { + out--; + } + + return out; +} + +SWITCH_DECLARE(switch_packetizer_t *) switch_packetizer_create(switch_packetizer_bitstream_t type, uint32_t slice_size) +{ + h264_packetizer_t *context = malloc(sizeof(h264_packetizer_t)); + memset(context, 0, sizeof(h264_packetizer_t)); + context->slice_size = slice_size; + context->type = type; + return (switch_packetizer_t *)context; +} + +// for H264 +SWITCH_DECLARE(switch_status_t) switch_packetizer_feed_extradata(switch_packetizer_t *packetizer, void *data, uint32_t size) +{ + h264_packetizer_t *context = (h264_packetizer_t *)packetizer; + uint8_t *p; + int left = size; + int n_sps = 0; + int n_pps = 0; + int sps_len; + int pps_len; + int i; + + if (left < 10) return SWITCH_STATUS_FALSE; + + if (context->extradata) { + context->sps = NULL; + context->pps = NULL; + context->sps_len = 0; + context->pps_len = 0; + free(context->extradata); + context->extradata = NULL; + } + + context->extradata = malloc(size); + if (!context->extradata) return SWITCH_STATUS_MEMERR; + memcpy(context->extradata, data, size); + +/* +0x0000 | 01 64 00 1E FF E1 00 1F 67 64 00 1E AC C8 60 33 // E1: 1SPS 00 1F: SPS 31byte +0x0010 | 0E F9 E6 FF C1 C6 01 C4 44 00 00 03 00 04 00 00 +0x0020 | 03 00 B8 3C 58 B6 68 01 00 05 68 E9 78 47 2C // 01: 1PPS 00 05: PPS 5byte +*/ + + p = context->extradata; + + if (*p != 1) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "NOT supported version: %d\n", *p); + return SWITCH_STATUS_FALSE; + } + p += 5; + left -= 5; + + if (left < 0) return SWITCH_STATUS_FALSE; + + //sps + n_sps = *p & 0x1f; + p += 1; + left -= 1; + + for (i = 0; i < n_sps; i++) { + sps_len = ntohs(*(uint16_t *)p); + p += sizeof(uint16_t); + left -= sizeof(uint16_t); + + if (left < sps_len) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "corrupted data %d < %u\n", left, sps_len); + return SWITCH_STATUS_FALSE; + } + + if (!context->sps) { // we only need the first one + context->sps = p; + context->sps_len = sps_len; + } + + p += sps_len; + left -= sps_len; + } + + if (left < 0) return SWITCH_STATUS_FALSE; + + n_pps = *p & 0x1f; + p += 1; + left -= 1; + + for (i = 0; i < n_pps; i++) { + pps_len = ntohs(*(uint16_t *)p); + p += sizeof(uint16_t); + left -= sizeof(uint16_t); + + if (left < pps_len) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "corrupted data %d < %u\n", left, pps_len); + return SWITCH_STATUS_FALSE; + } + + if (!context->pps) { // we only need the first one + context->pps = p; + context->pps_len = pps_len; + } + p += pps_len; + left -= pps_len; + } + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_DECLARE(switch_status_t) switch_packetizer_feed(switch_packetizer_t *packetizer, void *data, uint32_t size) +{ + h264_packetizer_t *context = (h264_packetizer_t *)packetizer; + const uint8_t *p = data; + const uint8_t *end = p + size; + int i = 0; + + // reset everytime + memset(context->nalus, 0, MAX_NALUS * sizeof(our_h264_nalu_t)); + context->nalu_current_index = 0; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %u %x %x %x %x %x\n", size, *p, *(p+1), *(p+2), *(p+3), *(p+4)); + + if (context->type == SPT_H264_SIZED_BITSTREAM) { + int left = size; + uint32_t len; + + while (left > 0) { + if (left < sizeof(uint32_t)) return SWITCH_STATUS_MORE_DATA; + len = htonl(*(uint32_t *)p); + left -= sizeof(uint32_t); + left -= len; + if (left < 0) return SWITCH_STATUS_MORE_DATA; + p += sizeof(uint32_t); + + context->nalus[i].start = p; + context->nalus[i].eat = p; + context->nalus[i].len = len; + + p += len; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "#%d %x len=%u\n", i, *context->nalus[i].start, context->nalus[i].len); + i++; + } + + return SWITCH_STATUS_SUCCESS; + } + + if (context->type == SPT_H264_SIGNALE_NALU) { + context->nalus[0].start = data; + context->nalus[0].eat = data; + context->nalus[0].len = size; + + return SWITCH_STATUS_SUCCESS; + } + + // SPT_H264_BITSTREAM + while ((p = fs_avc_find_startcode(p, end)) < end) { + if (!context->nalus[i].start) { + while (!(*p++)) ; /* eat the sync bytes, what ever 0 0 1 or 0 0 0 1 */ + context->nalus[i].start = p; + context->nalus[i].eat = p; + } else { + context->nalus[i].len = p - context->nalus[i].start; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "#%d %x len=%u\n", i, *context->nalus[i].start, context->nalus[i].len); + while (!(*p++)) ; /* eat the sync bytes, what ever 0 0 1 or 0 0 0 1 */ + i++; + context->nalus[i].start = p; + context->nalus[i].eat = p; + } + if (i >= MAX_NALUS - 2) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "TOO MANY SLICES!\n"); + break; + } + + } + + context->nalus[i].len = p - context->nalus[i].start; + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_DECLARE(switch_status_t) switch_packetizer_read(switch_packetizer_t *packetizer, switch_frame_t *frame) +{ + h264_packetizer_t *context = (h264_packetizer_t *)packetizer; + uint32_t slice_size = context->slice_size; + our_h264_nalu_t *nalu = &context->nalus[context->nalu_current_index]; + uint8_t nalu_hdr = 0; + uint8_t nalu_type = 0; + uint8_t nri = 0; + int left = nalu->len - (nalu->eat - nalu->start); + uint8_t *p = frame->data; + uint8_t start = nalu->start == nalu->eat ? 0x80 : 0; + int n = nalu->len / slice_size + 1; + int real_slice_size = nalu->len / n + 1 + 2; + + if (nalu->start == NULL) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "INVALID BITSTREAM\n"); + return SWITCH_STATUS_FALSE; + } + + nalu_hdr = *(uint8_t *)(nalu->start); + nalu_type = nalu_hdr & 0x1f; + nri = nalu_hdr & 0x60; + + if (real_slice_size > slice_size) real_slice_size = slice_size; + if (frame->buflen < slice_size) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "frame buffer too small %u < %u\n", frame->buflen, slice_size); + return SWITCH_STATUS_FALSE; + } + + if (context->type == SPT_H264_BITSTREAM || SPT_H264_SIZED_BITSTREAM) { + if (nalu_type == 0x05) { + // insert SPS/PPS before + if (context->sps && !context->sps_sent) { + memcpy(frame->data, context->sps, context->sps_len); + frame->datalen = context->sps_len; + frame->m = 0; + context->sps_sent = 1; + return SWITCH_STATUS_MORE_DATA; + } else if (context->pps && !context->pps_sent) { + memcpy(frame->data, context->pps, context->pps_len); + frame->datalen = context->pps_len; + frame->m = 0; + context->pps_sent = 1; + return SWITCH_STATUS_MORE_DATA; + } + } else if (nalu_type == 0x07) { + context->sps_sent = 1; + } else if (nalu_type == 0x08) { + context->pps_sent = 1; + } + } + + if (nalu->len <= slice_size) { + memcpy(frame->data, nalu->start, nalu->len); + frame->datalen = nalu->len; + context->nalu_current_index++; + + switch_clear_flag(frame, SFF_CNG); + + if (context->nalus[context->nalu_current_index].len) { + frame->m = 0; + return SWITCH_STATUS_MORE_DATA; + } + + frame->m = 1; + + if (nalu_type == 0x05) { + context->sps_sent = 0; + context->pps_sent = 0; + } + + return SWITCH_STATUS_SUCCESS; + } + + if (left <= (real_slice_size - 2)) { + p[0] = nri | 28; // FU-A + p[1] = 0x40 | nalu_type; + memcpy(p+2, nalu->eat, left); + nalu->eat += left; + frame->datalen = left + 2; + context->nalu_current_index++; + + if (!context->nalus[context->nalu_current_index].len) { + frame->m = 1; + return SWITCH_STATUS_SUCCESS; + } + + return SWITCH_STATUS_MORE_DATA; + } + + p[0] = nri | 28; // FU-A + p[1] = start | nalu_type; + if (start) nalu->eat++; + memcpy(p+2, nalu->eat, real_slice_size - 2); + nalu->eat += (real_slice_size - 2); + frame->datalen = real_slice_size; + frame->m = 0; + return SWITCH_STATUS_MORE_DATA; +} + +SWITCH_DECLARE(void) switch_packetizer_close(switch_packetizer_t **packetizer) +{ + h264_packetizer_t *context = (h264_packetizer_t *)(*packetizer); + if (context->extradata) free(context->extradata); + free(context); + *packetizer = NULL; +} diff --git a/src/mod/applications/mod_av/switch_packetizer.h b/src/mod/applications/mod_av/switch_packetizer.h new file mode 100644 index 0000000000..39cdf4f4d6 --- /dev/null +++ b/src/mod/applications/mod_av/switch_packetizer.h @@ -0,0 +1,62 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2020, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Seven Du + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Seven Du + * + * switch_packetizer H264 packetizer + * + */ + +#include + +#ifndef SWITCH_PACKETIZER_H +#define SWITCH_PACKETIZER_H + +typedef void switch_packetizer_t; + +typedef enum { + SPT_H264_BITSTREAM, // with separator 0 0 0 1 or 0 0 1 + SPT_H264_SIZED_BITSTREAM, + SPT_H264_SIGNALE_NALU, + SPT_VP8_BITSTREAM, + SPT_VP9_BITSTREAM, + + // no more beyond this line + SPT_INVALID_STREAM +} switch_packetizer_bitstream_t; + +/* + + create a packetizer and feed data, to avoid data copy, data MUST be valid before the next feed, or before close. + + */ + +SWITCH_DECLARE(switch_packetizer_t *) switch_packetizer_create(switch_packetizer_bitstream_t type, uint32_t slice_size); +SWITCH_DECLARE(switch_status_t) switch_packetizer_feed(switch_packetizer_t *packetizer, void *data, uint32_t size); +SWITCH_DECLARE(switch_status_t) switch_packetizer_feed_extradata(switch_packetizer_t *packetizer, void *data, uint32_t size); +SWITCH_DECLARE(switch_status_t) switch_packetizer_read(switch_packetizer_t *packetizer, switch_frame_t *frame); +SWITCH_DECLARE(void) switch_packetizer_close(switch_packetizer_t **packetizer); + +#endif diff --git a/src/mod/applications/mod_av/test/test_avformat.c b/src/mod/applications/mod_av/test/test_avformat.c index af364afd66..8520a558e6 100644 --- a/src/mod/applications/mod_av/test/test_avformat.c +++ b/src/mod/applications/mod_av/test/test_avformat.c @@ -162,6 +162,39 @@ FST_CORE_BEGIN("conf") } FST_TEST_END() + FST_TEST_BEGIN(avformat_test_play_no_decode) + { + char path[1024]; + switch_status_t status; + switch_file_handle_t fh = { 0 }; + uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE] = { 0 }; + switch_frame_t frame = { 0 }; + switch_size_t len = SAMPLES; + uint32_t flags = SWITCH_FILE_FLAG_READ | SWITCH_FILE_DATA_SHORT | SWITCH_FILE_FLAG_VIDEO; + int i = 0; + + sprintf(path, "{no_video_decode=true}%s%s%s", SWITCH_GLOBAL_dirs.conf_dir, SWITCH_PATH_SEPARATOR, "../test_RGB.mp4"); + // switch_set_string(path, "{no_video_decode=true}/usr/local/freeswitch/storage/bingbing.mp4"); + status = switch_core_file_open(&fh, path, 1, 8000, flags, fst_pool); + fst_requires(status == SWITCH_STATUS_SUCCESS); + fst_requires(switch_test_flag(&fh, SWITCH_FILE_OPEN)); + frame.data = data; + + do { + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_core_file_read(&fh, data, &len); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "status: %d len: %d\n", status, (int)len); + fst_check(frame.img == NULL); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_core_file_read_video(&fh, &frame, 0); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "status: %d len: %d %02x\n", status, frame.datalen, *(uint8_t *)frame.data); + } while (status == SWITCH_STATUS_MORE_DATA); + + switch_core_file_close(&fh); + } + FST_TEST_END() + + FST_TEARDOWN_BEGIN() { //const char *err = NULL; diff --git a/src/mod/applications/mod_av/test/test_packetizer.c b/src/mod/applications/mod_av/test/test_packetizer.c new file mode 100644 index 0000000000..9d34f98d51 --- /dev/null +++ b/src/mod/applications/mod_av/test/test_packetizer.c @@ -0,0 +1,256 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2018, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Seven Du + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * + * test_avformat -- avformat tests + * + */ + +#include +#include + +#define SLICE_SIZE 4 + +FST_CORE_BEGIN("conf") +{ + FST_MODULE_BEGIN(mod_av, mod_av_test) + { + FST_SETUP_BEGIN() + { + fst_requires_module("mod_av"); + } + FST_SETUP_END() + + FST_TEST_BEGIN(test_packetizer_bitstream) + { + switch_packetizer_t *packetizer = switch_packetizer_create(SPT_H264_BITSTREAM, SLICE_SIZE); + uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; + switch_frame_t frame = {0}; + switch_status_t status; + uint8_t h264data[] = {0, 0, 0, 1, 0x67, 1, 2, 0, 0, 0, 1, 0x68, 1, 2, 0, 0, 0, 1, 0x65, 1, 2, 3, 4, 5, 6}; + + frame.data = data; + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + switch_set_flag(&frame, SFF_ENCODED); + + status = switch_packetizer_feed(packetizer, h264data, sizeof(h264data)); + fst_requires(status == SWITCH_STATUS_SUCCESS); + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_SUCCESS); + fst_requires(frame.datalen == 4); + switch_packetizer_close(&packetizer); + } + FST_TEST_END() + + FST_TEST_BEGIN(test_packetizer_sized_bitstream_has_sps_pps) + { + switch_packetizer_t *packetizer = switch_packetizer_create(SPT_H264_SIZED_BITSTREAM, SLICE_SIZE); + uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; + switch_frame_t frame = {0}; + switch_status_t status; + uint8_t h264data[] = {0, 0, 0, 3, 0x67, 1, 2, 0, 0, 0, 3, 0x68, 1, 2, 0, 0, 0, 7, 0x65, 1, 2, 3, 4, 5, 6}; + + frame.data = data; + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + switch_set_flag(&frame, SFF_ENCODED); + + status = switch_packetizer_feed(packetizer, h264data, sizeof(h264data)); + fst_requires(status == SWITCH_STATUS_SUCCESS); + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_SUCCESS); + fst_requires(frame.datalen == 4); + fst_check(frame.m == 1); + switch_packetizer_close(&packetizer); + } + FST_TEST_END() + + FST_TEST_BEGIN(test_packetizer_sized_bitstream_no_sps_pps) + { + switch_packetizer_t *packetizer = switch_packetizer_create(SPT_H264_SIZED_BITSTREAM, SLICE_SIZE); + uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; + switch_frame_t frame = {0}; + switch_status_t status; + uint8_t h264data[] = {0, 0, 0, 3, 0x06, 1, 2, 0, 0, 0, 3, 0x09, 1, 2, 0, 0, 0, 7, 0x65, 1, 2, 3, 4, 5, 6}; + uint8_t extradata[] = {0x01, 0x64, 0x00, 0x1e, 0xff, 0xe1, 0x00, 0x03, 0x67, 0x64, 0x00, 0xe1, 0x00, 0x03, 0x68, 0x01, 0x02}; +// 1 fps 3 bytes 1pps 3 bytes + frame.data = data; + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + switch_set_flag(&frame, SFF_ENCODED); + + status = switch_packetizer_feed_extradata(packetizer, extradata, sizeof(extradata)); + fst_requires(status == SWITCH_STATUS_SUCCESS); + + status = switch_packetizer_feed(packetizer, h264data, sizeof(h264data)); + fst_requires(status == SWITCH_STATUS_SUCCESS); + + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x06); + + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u %x\n", frame.datalen, *(uint8_t *)frame.data); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x09); + + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u %x\n", frame.datalen, *(uint8_t *)frame.data); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x07); + + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u %x\n", frame.datalen, *(uint8_t *)frame.data); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x08); + + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u %x\n", frame.datalen, *(uint8_t *)frame.data); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x1c); + fst_check(frame.m == 0); + + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u %x\n", frame.datalen, *(uint8_t *)frame.data); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x1c); + fst_check(frame.m == 0); + + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u %x\n", frame.datalen, *(uint8_t *)frame.data); + fst_requires(status == SWITCH_STATUS_SUCCESS); + fst_requires(frame.datalen == 4); + fst_check((*(uint8_t *)frame.data & 0x1f) == 0x1c); + fst_check(frame.m == 1); + switch_packetizer_close(&packetizer); + } + FST_TEST_END() + + FST_TEST_BEGIN(test_packetizer_invalid) + { + switch_packetizer_t *packetizer = switch_packetizer_create(SPT_H264_BITSTREAM, SLICE_SIZE); + uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE]; + switch_frame_t frame = {0}; + switch_status_t status; + uint8_t h264data[] = {0, 0, 2, 9, 0x67, 1, 2, 0, 0, 0, 0, 0x68, 1, 2, 0, 0, 0, 0, 0x65, 1, 2, 3, 4, 5, 6}; + + frame.data = data; + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + switch_set_flag(&frame, SFF_ENCODED); + + status = switch_packetizer_feed(packetizer, h264data, sizeof(h264data)); + fst_requires(status == SWITCH_STATUS_SUCCESS); + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 3); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_MORE_DATA); + fst_requires(frame.datalen == 4); + frame.datalen = SWITCH_RECOMMENDED_BUFFER_SIZE; + status = switch_packetizer_read(packetizer, &frame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%u\n", frame.datalen); + fst_requires(status == SWITCH_STATUS_SUCCESS); + fst_requires(frame.datalen == 4); + switch_packetizer_close(&packetizer); + } + FST_TEST_END() + + FST_TEARDOWN_BEGIN() + { + //const char *err = NULL; + switch_sleep(1000000); + //fst_check(switch_loadable_module_unload_module(SWITCH_GLOBAL_dirs.mod_dir, (char *)"mod_av", SWITCH_TRUE, &err) == SWITCH_STATUS_SUCCESS); + } + FST_TEARDOWN_END() + } + FST_MODULE_END() +} +FST_CORE_END()