diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index 340f18f0b6..46167543af 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -326,6 +326,7 @@ SWITCH_DECLARE(void) switch_core_session_stop_media(switch_core_session_t *sessi SWITCH_DECLARE(switch_media_flow_t) switch_core_session_media_flow(switch_core_session_t *session, switch_media_type_t type); SWITCH_DECLARE(switch_status_t) switch_core_media_get_vid_params(switch_core_session_t *session, switch_vid_params_t *vid_params); SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_file(switch_core_session_t *session, switch_file_handle_t *fh, switch_rw_t rw); +SWITCH_DECLARE(switch_bool_t) switch_core_session_in_video_thread(switch_core_session_t *session); SWITCH_END_EXTERN_C #endif diff --git a/src/mod/applications/mod_fsv/mod_fsv.c b/src/mod/applications/mod_fsv/mod_fsv.c index 6791f2a95e..77ac9caeb7 100644 --- a/src/mod/applications/mod_fsv/mod_fsv.c +++ b/src/mod/applications/mod_fsv/mod_fsv.c @@ -670,10 +670,6 @@ static void decode_video_thread(switch_core_session_t *session, void *obj) goto done; } - switch_channel_set_flag(channel, CF_VIDEO_DECODED_READ); - switch_channel_set_flag(channel, CF_VIDEO_DEBUG_READ); - switch_channel_set_flag(channel, CF_VIDEO_DEBUG_WRITE); - while (switch_channel_ready(channel)) { switch_status_t status = switch_core_session_read_video_frame(session, &frame, SWITCH_IO_FLAG_NONE, 0); @@ -687,6 +683,7 @@ static void decode_video_thread(switch_core_session_t *session, void *obj) } if (frame && frame->datalen > 0) { + printf("WTF %p\n", (void *)frame->img); switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); } else { continue; @@ -733,6 +730,11 @@ SWITCH_STANDARD_APP(decode_video_function) switch_channel_answer(channel); switch_core_session_request_video_refresh(session); + + switch_channel_set_flag(channel, CF_VIDEO_DECODED_READ); + switch_channel_set_flag(channel, CF_VIDEO_DEBUG_READ); + switch_channel_set_flag(channel, CF_VIDEO_DEBUG_WRITE); + switch_core_media_start_video_function(session, decode_video_thread, &max_pictures); switch_ivr_play_file(session, NULL, moh, NULL); diff --git a/src/mod/codecs/mod_vpx/mod_vpx.c b/src/mod/codecs/mod_vpx/mod_vpx.c index eb82f9fb7b..417828dcbb 100644 --- a/src/mod/codecs/mod_vpx/mod_vpx.c +++ b/src/mod/codecs/mod_vpx/mod_vpx.c @@ -44,12 +44,11 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load); SWITCH_MODULE_DEFINITION(mod_vpx, mod_vpx_load, NULL, NULL); - -#define encoder_interface (vpx_codec_vp8_cx()) -#define decoder_interface (vpx_codec_vp8_dx()) - struct vpx_context { switch_codec_t *codec; + int is_vp9; + vpx_codec_iface_t *encoder_interface; + vpx_codec_iface_t *decoder_interface; unsigned int flags; switch_codec_settings_t codec_settings; unsigned int bandwidth; @@ -88,6 +87,9 @@ typedef struct vpx_context vpx_context_t; static switch_status_t init_decoder(switch_codec_t *codec) { vpx_context_t *context = (vpx_context_t *)codec->private_info; + vpx_codec_dec_cfg_t cfg = {0, 0, 0}; + vpx_codec_flags_t dec_flags = 0; + if (context->flags & SWITCH_CODEC_FLAG_DECODE && !context->decoder_init) { vp8_postproc_cfg_t ppcfg; @@ -96,7 +98,13 @@ static switch_status_t init_decoder(switch_codec_t *codec) // context->decoder_init = 0; //} - if (vpx_codec_dec_init(&context->decoder, decoder_interface, NULL, VPX_CODEC_USE_POSTPROC) != VPX_CODEC_OK) { + cfg.threads = switch_core_cpu_count(); + + if (!context->is_vp9) { // vp8 only + dec_flags = VPX_CODEC_USE_POSTPROC; + } + + if (vpx_codec_dec_init(&context->decoder, context->decoder_interface, &cfg, dec_flags) != VPX_CODEC_OK) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", context->encoder.err, context->encoder.err_detail); return SWITCH_STATUS_FALSE; } @@ -226,7 +234,7 @@ static switch_status_t init_encoder(switch_codec_t *codec) } } else if (context->flags & SWITCH_CODEC_FLAG_ENCODE) { - if (vpx_codec_enc_init(&context->encoder, encoder_interface, config, 0 & VPX_CODEC_USE_OUTPUT_PARTITION) != VPX_CODEC_OK) { + if (vpx_codec_enc_init(&context->encoder, context->encoder_interface, config, 0 & VPX_CODEC_USE_OUTPUT_PARTITION) != VPX_CODEC_OK) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", context->encoder.err, context->encoder.err_detail); return SWITCH_STATUS_FALSE; } @@ -258,7 +266,6 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ vpx_context_t *context = NULL; int encoding, decoding; - encoding = (flags & SWITCH_CODEC_FLAG_ENCODE); decoding = (flags & SWITCH_CODEC_FLAG_DECODE); @@ -274,11 +281,23 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ context->codec_settings = *codec_settings; } + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s\n", codec->implementation->iananame); + + if (!strcmp(codec->implementation->iananame, "VP9")) { + context->is_vp9 = 1; + context->encoder_interface = vpx_codec_vp9_cx(); + context->decoder_interface = vpx_codec_vp9_dx(); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "is vp9\n"); + } else { + context->encoder_interface = vpx_codec_vp8_cx(); + context->decoder_interface = vpx_codec_vp8_dx(); + } + if (codec->fmtp_in) { codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in); } - if (vpx_codec_enc_config_default(encoder_interface, &context->config, 0) != VPX_CODEC_OK) { + if (vpx_codec_enc_config_default(context->encoder_interface, &context->config, 0) != VPX_CODEC_OK) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder config Error\n"); return SWITCH_STATUS_FALSE; } @@ -328,6 +347,16 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ +-+-+-+-+-+-+-+-+ */ + typedef struct { + unsigned extended:1; + unsigned reserved1:1; + unsigned non_referenced:1; + unsigned start:1; + unsigned reserved2:1; + unsigned pid:1; + } vp8_payload_descriptor_t; + + static switch_status_t consume_partition(vpx_context_t *context, switch_frame_t *frame) { if (!context->pkt) { @@ -353,9 +382,14 @@ static switch_status_t consume_partition(vpx_context_t *context, switch_frame_t } if (context->pkt->data.frame.sz < SLICE_SIZE) { - uint8_t hdr = 0x10; + uint8_t hdr = 0x10; + memcpy(frame->data, &hdr, 1); + // ((uint8_t *)frame->data)[0] = 0x10; + //vp8_payload_descriptor_t *hdr = (vp8_payload_descriptor_t *)frame->data; + //hdr->start = 1; + + //printf("WTF [%x] [%x]\n", *(unsigned char *)hdr, 0x10); - memcpy(frame->data, &hdr, 1); memcpy((uint8_t *)frame->data + 1, context->pkt->data.frame.buf, context->pkt->data.frame.sz); frame->datalen = context->pkt->data.frame.sz + 1; frame->m = 1; @@ -494,7 +528,7 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_frame_t * return consume_partition(context, frame); } -static switch_status_t buffer_vpx_packets(vpx_context_t *context, switch_frame_t *frame) +static switch_status_t buffer_vp8_packets(vpx_context_t *context, switch_frame_t *frame) { uint8_t *data = frame->data; uint8_t S; @@ -552,6 +586,27 @@ static switch_status_t buffer_vpx_packets(vpx_context_t *context, switch_frame_t } +static switch_status_t buffer_vp9_packets(vpx_context_t *context, switch_frame_t *frame) +{ + uint8_t *data = frame->data; + uint8_t *vp9; + int len = 0; + + if (!frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no frame in codec!!\n"); + return SWITCH_STATUS_RESTART; + } + + vp9 = data + 1; + + len = frame->datalen - (vp9 - data); + switch_buffer_write(context->vpx_packet_buffer, vp9, len); + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffered %d bytes, buffer size: %" SWITCH_SIZE_T_FMT "\n", len, switch_buffer_inuse(context->vpx_packet_buffer)); + + return SWITCH_STATUS_SUCCESS; +} + static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *frame) { vpx_context_t *context = (vpx_context_t *)codec->private_info; @@ -560,6 +615,8 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t * switch_status_t status = SWITCH_STATUS_SUCCESS; int is_keyframe = ((*(unsigned char *)frame->data) & 0x01) ? 0 : 1; + if (context->is_vp9) is_keyframe = 1; // don't know how to get it yet + if (context->need_decoder_reset != 0) { vpx_codec_destroy(&context->decoder); context->decoder_init = 0; @@ -578,7 +635,7 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t * decoder = &context->decoder; - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %" SWITCH_SIZE_T_FMT " mark:%d\n", frame->datalen, frame->timestamp, frame->m); + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %u mark:%d\n", frame->datalen, frame->timestamp, frame->m); if (!is_keyframe && context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && (!frame->m) && (!context->last_received_complete_picture)) { @@ -592,7 +649,7 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t * context->last_received_timestamp = frame->timestamp; context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE; - status = buffer_vpx_packets(context, frame); + status = context->is_vp9 ? buffer_vp9_packets(context, frame) : buffer_vp8_packets(context, frame); //printf("READ buf:%ld got_key:%d st:%d m:%d\n", switch_buffer_inuse(context->vpx_packet_buffer), context->got_key_frame, status, frame->m); @@ -621,7 +678,8 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t * err = vpx_codec_decode(decoder, data, (unsigned int)len, NULL, 0); if (err != VPX_CODEC_OK) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error decoding %" SWITCH_SIZE_T_FMT " bytes, [%d:%d:%s]\n", len, err, decoder->err, decoder->err_detail); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error decoding %" SWITCH_SIZE_T_FMT " bytes, [%d:%s:%s]\n", + len, err, vpx_codec_error(decoder), vpx_codec_error_detail(decoder)); switch_goto_status(SWITCH_STATUS_RESTART, end); } @@ -749,6 +807,9 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load) SWITCH_ADD_CODEC(codec_interface, "VP8 Video"); switch_core_codec_add_video_implementation(pool, codec_interface, 99, "VP8", NULL, switch_vpx_init, switch_vpx_encode, switch_vpx_decode, switch_vpx_control, switch_vpx_destroy); + SWITCH_ADD_CODEC(codec_interface, "VP9 Video"); + switch_core_codec_add_video_implementation(pool, codec_interface, 99, "VP9", NULL, + switch_vpx_init, switch_vpx_encode, switch_vpx_decode, switch_vpx_control, switch_vpx_destroy); /* indicate that the module should continue to be loaded */ return SWITCH_STATUS_SUCCESS; diff --git a/src/switch_channel.c b/src/switch_channel.c index fe7df086c2..10a2af61ec 100644 --- a/src/switch_channel.c +++ b/src/switch_channel.c @@ -1831,7 +1831,9 @@ SWITCH_DECLARE(void) switch_channel_set_flag_value(switch_channel_t *channel, sw } if (flag == CF_VIDEO_DECODED_READ) { - switch_channel_wait_for_flag(channel, CF_VIDEO_READY, SWITCH_TRUE, 10000, NULL); + if (!switch_core_session_in_video_thread(channel->session)) { + switch_channel_wait_for_flag(channel, CF_VIDEO_READY, SWITCH_TRUE, 10000, NULL); + } } } diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 519c20828d..557ba8365a 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -159,6 +159,7 @@ typedef struct switch_rtp_engine_s { switch_codec_settings_t codec_settings; switch_media_flow_t rmode; switch_media_flow_t smode; + switch_thread_id_t thread_id; } switch_rtp_engine_t; struct switch_media_handle_s { @@ -4634,7 +4635,7 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi switch_status_t status; switch_frame_t *read_frame; switch_media_handle_t *smh; - uint32_t loops = 0, xloops = 0, vloops = 0; + uint32_t loops = 0, xloops = 0, vloops = 0, viloops = 0; switch_frame_t fr = { 0 }; unsigned char *buf = NULL; switch_image_t *blank_img = NULL; @@ -4645,13 +4646,12 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi blank_img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, 320, 240, 1); switch_img_fill(blank_img, 0, 0, blank_img->d_w, blank_img->d_h, &bgcolor); - - if (!(smh = session->media_handle)) { return NULL; } v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + v_engine->thread_id = switch_thread_self(); switch_core_session_read_lock(session); @@ -4731,9 +4731,9 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi if (switch_test_flag(read_frame, SFF_CNG)) { continue; } - + if (read_frame->img) { - if (vloops > 10) { + if (++viloops > 10) { switch_channel_set_flag(channel, CF_VIDEO_READY); smh->vid_params.width = read_frame->img->d_w; smh->vid_params.height = read_frame->img->d_h; @@ -4884,6 +4884,23 @@ SWITCH_DECLARE(void) switch_core_media_end_video_function(switch_core_session_t } } +SWITCH_DECLARE(switch_bool_t) switch_core_session_in_video_thread(switch_core_session_t *session) +{ + switch_rtp_engine_t *v_engine; + switch_media_handle_t *smh; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return SWITCH_FALSE; + } + + v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + + return switch_thread_equal(switch_thread_self(), v_engine->thread_id) ? SWITCH_TRUE : SWITCH_FALSE; +} + + //? #define RA_PTR_LEN 512 SWITCH_DECLARE(switch_status_t) switch_core_media_proxy_remote_addr(switch_core_session_t *session, const char *sdp_str) diff --git a/src/switch_rtp.c b/src/switch_rtp.c index f2008b98f7..60ad95a7ef 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -3467,8 +3467,8 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session } rtp_session->pool = pool; - rtp_session->te = 101; - rtp_session->recv_te = 101; + rtp_session->te = 0; + rtp_session->recv_te = 0; rtp_session->session = session; switch_mutex_init(&rtp_session->flag_mutex, SWITCH_MUTEX_NESTED, pool);