From 0cd5658caab8ef8161214b71f60c454daffe42b9 Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Tue, 18 Nov 2014 16:39:32 -0600 Subject: [PATCH] FS-7500: another refactoring pass, temp code still in place, WORK IN PROGRESS --- src/include/private/switch_core_pvt.h | 2 +- src/include/switch_core.h | 61 +--- src/include/switch_core_media.h | 12 +- src/include/switch_frame.h | 1 + src/include/switch_types.h | 14 +- .../mod_conference/mod_conference.c | 85 +----- src/mod/applications/mod_fsv/mod_fsv.c | 28 +- src/mod/codecs/mod_openh264/mod_openh264.cpp | 155 +++++++---- src/mod/codecs/mod_vpx/mod_vpx.c | 197 +++++++------ src/mod/codecs/mod_yuv/mod_yuv.c | 16 +- src/mod/formats/mod_vlc/mod_vlc.c | 25 +- src/switch_core_codec.c | 19 +- src/switch_core_media.c | 261 ++++++++++++------ src/switch_core_session.c | 26 +- src/switch_ivr_bridge.c | 60 ++-- src/switch_rtp.c | 5 +- 16 files changed, 482 insertions(+), 485 deletions(-) diff --git a/src/include/private/switch_core_pvt.h b/src/include/private/switch_core_pvt.h index e166f9fa5a..8beb502f87 100644 --- a/src/include/private/switch_core_pvt.h +++ b/src/include/private/switch_core_pvt.h @@ -191,7 +191,7 @@ struct switch_core_session { switch_core_video_thread_callback_func_t *_video_thread_callback; void *_video_thread_user_data; //switch_time_t last_video_write_time; - + switch_image_write_callback_t image_write_callback; void *image_write_callback_user_data; }; diff --git a/src/include/switch_core.h b/src/include/switch_core.h index ca48d8118c..ba1c877919 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -169,10 +169,6 @@ typedef enum { DS_INVALID, } dtls_state_t; -typedef switch_status_t (switch_core_video_thread_callback_func_t) (switch_core_session_t *session, switch_frame_t *frame, void *user_data); - - - #define MESSAGE_STAMP_FFL(_m) _m->_file = __FILE__; _m->_func = __SWITCH_FUNC__; _m->_line = __LINE__ #define MESSAGE_STRING_ARG_MAX 10 @@ -1274,18 +1270,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_frame(_In_ switch_core_ */ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(_In_ switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, int stream_id); - -/*! - \brief Write a video image to a session using a video frame - \param session the session to write to - \param frame a pointer to a frame to use for write with proper codec - \param img the image structure with the image data - \param the size for packetization - \param flag pointer to frame flags to pass in / out - \return SWITCH_STATUS_SUCCESS a if the image was written -*/ -SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, - switch_image_t *img, switch_size_t size, uint32_t *flag); /*! \brief set a callback to be called after each frame of an image is written \param session the session to write to @@ -1643,16 +1627,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode(switch_codec_t *codec, /*! \brief Encode video data using a codec handle \param codec the codec handle to use - \param img the img in I420 format - \param encoded_data the buffer to write the encoded data to - \param encoded_data_len the size of the encoded_data buffer - \param flag flags to exchange - \return SWITCH_STATUS_SUCCESS if the data was encoded - \note encoded_data_len will be rewritten to the in-use size of encoded_data + \param frame the frame to encode */ -SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec, - switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); +SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec, switch_frame_t *frame); + /*! \brief send control data using a codec handle @@ -1679,10 +1657,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_control(switch_codec_t *codec, \param flag flags to exchange \return SWITCH_STATUS_SUCCESS if the data was decoded, and a non-NULL img */ -SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec, - switch_frame_t *frame, - switch_image_t **img, unsigned int *flag); - +SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec, switch_frame_t *frame); /*! \brief Destroy an initalized codec handle @@ -2340,34 +2315,6 @@ SWITCH_DECLARE(uint8_t) switch_core_session_compare(switch_core_session_t *a, sw */ SWITCH_DECLARE(uint8_t) switch_core_session_check_interface(switch_core_session_t *session, const switch_endpoint_interface_t *endpoint_interface); -/*! - \brief Set a callback to let the core video thread call us - \param session the session - \param func to callback - \param private user data - \return SWITCH_STATUS_CONTINUE | SWITCH_STATUS_SUCCESS | SWITCH_STATUS_BREAK | SWITCH_STATUS_* - - If returns SWITCH_STATUS_CONTINUE, it will continues to run furthur code (read/write) in the core video thread, - that is to say, if the callback func to nothing and just returns SWITCH_STATUS_CONTINUE, it remains the default behaviour, - Return SWITCH_STATUS_SUCCESS to skip the default behaviour - Return SWITCH_STATUS_BREAK will break the loop and end the video thread -*/ - -SWITCH_DECLARE(switch_status_t) switch_core_session_set_video_thread_callback(switch_core_session_t *session, switch_core_video_thread_callback_func_t *func, void *user_data); - -/*! - \brief Set a callback to let the core video thread call us - \param session the session - \param the current video frame - \param private user data - \return SWITCH_STATUS_CONTINUE or SWITCH_STATUS_SUCCESS - - If returns SWITCH_STATUS_CONTINUE, it will continues to run furthur code (read/write) in the core video thread, - that is to say, if the callback func to nothing and just returns SWITCH_STATUS_CONTINUE, it remains the default behaviour, - Return SWITCH_STATUS_SUCCESS to skip the default behaviour -*/ -SWITCH_DECLARE(switch_status_t) switch_core_session_video_thread_callback(switch_core_session_t *session, switch_frame_t *frame); - SWITCH_DECLARE(switch_hash_index_t *) switch_core_mime_index(void); SWITCH_DECLARE(const char *) switch_core_mime_ext2type(const char *ext); SWITCH_DECLARE(const char *) switch_core_mime_type2ext(const char *type); diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index 8e4f0b74a1..f193482063 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -150,7 +150,7 @@ typedef struct switch_core_media_params_s { /* a core_video_thread will be started automatically when uses rtp based media, external_video_source should be set to SWITCH_TRUE and - switch_core_media_start_video_thread() + switch_core_session_start_video_thread() should be explicitly called to start the video thread if uses the media handle for non-rtp based media */ @@ -266,11 +266,6 @@ SWITCH_DECLARE(void) switch_core_media_deinit(void); SWITCH_DECLARE(void) switch_core_media_set_stats(switch_core_session_t *session); SWITCH_DECLARE(void) switch_core_session_wake_video_thread(switch_core_session_t *session); SWITCH_DECLARE(void) switch_core_session_clear_crypto(switch_core_session_t *session); -<<<<<<< HEAD -SWITCH_DECLARE(switch_status_t) switch_core_media_start_video_thread(switch_core_session_t *session); -======= -SWITCH_DECLARE(switch_status_t) start_core_video_thread(switch_core_session_t *session); ->>>>>>> allow using the video thread externally - e.g. non rtp based video SWITCH_DECLARE(switch_status_t) switch_core_session_get_payload_code(switch_core_session_t *session, switch_media_type_t type, @@ -314,7 +309,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype); - +SWITCH_DECLARE(void) switch_core_media_start_video_function(switch_core_session_t *session, switch_video_function_t video_function, void *user_data); +SWITCH_DECLARE(void) switch_core_media_end_video_function(switch_core_session_t *session); +SWITCH_DECLARE(switch_status_t) switch_core_session_start_video_thread(switch_core_session_t *session); +SWITCH_DECLARE(int) switch_core_media_check_video_function(switch_core_session_t *session); SWITCH_END_EXTERN_C #endif /* For Emacs: diff --git a/src/include/switch_frame.h b/src/include/switch_frame.h index 51800df6c1..b264e51de6 100644 --- a/src/include/switch_frame.h +++ b/src/include/switch_frame.h @@ -74,6 +74,7 @@ SWITCH_BEGIN_EXTERN_C switch_frame_flag_t flags; void *user_data; payload_map_t *pmap; + switch_image_t *img; }; SWITCH_END_EXTERN_C diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 508170239e..93071b8406 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -1450,6 +1450,7 @@ typedef enum { CF_BYPASS_MEDIA_AFTER_HOLD, CF_HANGUP_HELD, CF_CONFERENCE_RESET_MEDIA, + CF_VIDEO_DECODED_READ, /* WARNING: DO NOT ADD ANY FLAGS BELOW THIS LINE */ /* IF YOU ADD NEW ONES CHECK IF THEY SHOULD PERSIST OR ZERO THEM IN switch_core_session.c switch_core_session_request_xml() */ CF_FLAG_MAX @@ -1496,7 +1497,8 @@ typedef enum { SFF_MARKER = (1 << 11), SFF_WAIT_KEY_FRAME = (1 << 12), SFF_RAW_RTP_PARSE_FRAME = (1 << 13), - SFF_PICTURE_RESET = (1 << 14) + SFF_PICTURE_RESET = (1 << 14), + SFF_SAME_IMAGE = (1 << 15) } switch_frame_flag_enum_t; typedef uint32_t switch_frame_flag_t; @@ -2154,6 +2156,8 @@ typedef switch_bool_t (*switch_media_bug_callback_t) (switch_media_bug_t *, void typedef switch_bool_t (*switch_tone_detect_callback_t) (switch_core_session_t *, const char *, const char *); typedef struct switch_xml_binding switch_xml_binding_t; +typedef void (*switch_video_function_t) (switch_core_session_t *session, void *user_data); + typedef switch_status_t (*switch_core_codec_encode_func_t) (switch_codec_t *codec, switch_codec_t *other_codec, void *decoded_data, @@ -2169,13 +2173,9 @@ typedef switch_status_t (*switch_core_codec_decode_func_t) (switch_codec_t *code uint32_t encoded_rate, void *decoded_data, uint32_t *decoded_data_len, uint32_t *decoded_rate, unsigned int *flag); -typedef switch_status_t (*switch_core_codec_video_encode_func_t) (switch_codec_t *codec, - switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); +typedef switch_status_t (*switch_core_codec_video_encode_func_t) (switch_codec_t *codec, switch_frame_t *frame); -typedef switch_status_t (*switch_core_codec_video_decode_func_t) (switch_codec_t *codec, - switch_frame_t *frame, - switch_image_t **img, unsigned int *flag); +typedef switch_status_t (*switch_core_codec_video_decode_func_t) (switch_codec_t *codec, switch_frame_t *frame); typedef enum { SCC_VIDEO_REFRESH = 0 diff --git a/src/mod/applications/mod_conference/mod_conference.c b/src/mod/applications/mod_conference/mod_conference.c index 46a3707c28..3787f107ec 100644 --- a/src/mod/applications/mod_conference/mod_conference.c +++ b/src/mod/applications/mod_conference/mod_conference.c @@ -198,8 +198,7 @@ typedef enum { MFLAG_JOIN_ONLY = (1 << 25), MFLAG_POSITIONAL = (1 << 26), MFLAG_NO_POSITIONAL = (1 << 27), - MFLAG_JOIN_VID_FLOOR = (1 << 28), - MFLAG_RECEIVING_VIDEO = (1 << 29) + MFLAG_JOIN_VID_FLOOR = (1 << 28) } member_flag_t; typedef enum { @@ -231,8 +230,7 @@ typedef enum { typedef enum { RFLAG_CAN_SPEAK = (1 << 0), - RFLAG_CAN_HEAR = (1 << 1), - RFLAG_CAN_SEND_VIDEO = (1 << 2) + RFLAG_CAN_HEAR = (1 << 1) } relation_flag_t; typedef enum { @@ -2394,11 +2392,6 @@ static switch_status_t conference_add_member(conference_obj_t *conference, confe } - if (member->conference->video_floor_holder && member->conference->video_floor_holder != member && member->channel) { - // there's already someone hold the floor, tell the core thread start to read video - switch_channel_clear_flag(member->channel, CF_VIDEO_PASSIVE); - } - unlock_member(member); switch_mutex_unlock(member->audio_out_mutex); switch_mutex_unlock(member->audio_in_mutex); @@ -2917,29 +2910,6 @@ static void *SWITCH_THREAD_FUNC conference_video_bridge_thread_run(switch_thread return NULL; } -switch_status_t video_thread_callback(switch_core_session_t *session, switch_frame_t *frame, void *user_data) -{ - switch_channel_t *channel = switch_core_session_get_channel(session); - char *name = switch_channel_get_name(channel); - conference_member_t *member = (conference_member_t *)user_data; - conference_relationship_t *rel = NULL; - - if (!member || member->relationships == NULL) return SWITCH_STATUS_SUCCESS; - - lock_member(member); - - for (rel = member->relationships; rel; rel = rel->next) { - conference_member_t *imember = conference_member_get(member->conference, rel->id); - if (imember && switch_test_flag(imember, MFLAG_RECEIVING_VIDEO)) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s %d->%d %d\n", name, member->id, imember->id, frame->datalen); - switch_core_session_write_video_frame(imember->session, frame, SWITCH_IO_FLAG_NONE, 0); - switch_thread_rwlock_unlock(imember->rwlock); - } - } - - unlock_member(member); - return SWITCH_STATUS_SUCCESS; -} /* Main video monitor thread (1 per distinct conference room) */ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thread, void *obj) @@ -3031,10 +3001,8 @@ static void *SWITCH_THREAD_FUNC conference_video_thread_run(switch_thread_t *thr } if (isession && switch_channel_test_flag(ichannel, CF_VIDEO)) { - if (!switch_test_flag(imember, MFLAG_RECEIVING_VIDEO)) { - memcpy(vid_frame->packet, buf, vid_frame->packetlen); - switch_core_session_write_video_frame(imember->session, vid_frame, SWITCH_IO_FLAG_NONE, 0); - } + memcpy(vid_frame->packet, buf, vid_frame->packetlen); + switch_core_session_write_video_frame(imember->session, vid_frame, SWITCH_IO_FLAG_NONE, 0); } switch_core_session_rwunlock(isession); @@ -7559,7 +7527,7 @@ static switch_status_t conf_api_sub_stop(conference_obj_t *conference, switch_st static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_stream_handle_t *stream, int argc, char **argv) { - uint8_t nospeak = 0, nohear = 0, sendvideo = 0, clear = 0; + uint8_t nospeak = 0, nohear = 0, clear = 0; switch_assert(conference != NULL); switch_assert(stream != NULL); @@ -7580,10 +7548,9 @@ static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_ if (member_id > 0 && member->id != member_id) continue; for (rel = member->relationships; rel; rel = rel->next) { - stream->write_function(stream, "%d -> %d %s%s%s\n", member->id, rel->id, + stream->write_function(stream, "%d -> %d %s%s\n", member->id, rel->id, (rel->flags & RFLAG_CAN_SPEAK) ? "SPEAK " : "NOSPEAK ", - (rel->flags & RFLAG_CAN_HEAR) ? "HEAR " : "NOHEAR ", - (rel->flags & RFLAG_CAN_SEND_VIDEO) ? "SENDVIDEO " : "NOSENDVIDEO "); + (rel->flags & RFLAG_CAN_HEAR) ? "HEAR" : "NOHEAR"); } } } else { @@ -7598,35 +7565,22 @@ static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_ nospeak = strstr(argv[4], "nospeak") ? 1 : 0; nohear = strstr(argv[4], "nohear") ? 1 : 0; - sendvideo = strstr(argv[4], "sendvideo") ? 1 : 0; if (!strcasecmp(argv[4], "clear")) { clear = 1; } - if (!(clear || nospeak || nohear || sendvideo)) { + if (!(clear || nospeak || nohear)) { return SWITCH_STATUS_GENERR; } if (clear) { - conference_member_t *member = NULL, *other_member = NULL; + conference_member_t *member = NULL; uint32_t id = atoi(argv[2]); uint32_t oid = atoi(argv[3]); if ((member = conference_member_get(conference, id))) { member_del_relationship(member, oid); - other_member = conference_member_get(conference, oid); - - if (other_member) { - if (switch_test_flag(other_member, MFLAG_RECEIVING_VIDEO)) { - switch_clear_flag(other_member, MFLAG_RECEIVING_VIDEO); - if (conference->floor_holder) { - switch_core_session_refresh_video(conference->floor_holder->session); - } - } - switch_thread_rwlock_unlock(other_member->rwlock); - } - stream->write_function(stream, "relationship %u->%u cleared.\n", id, oid); switch_thread_rwlock_unlock(member->rwlock); } else { @@ -7635,7 +7589,7 @@ static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_ return SWITCH_STATUS_SUCCESS; } - if (nospeak || nohear || sendvideo) { + if (nospeak || nohear) { conference_member_t *member = NULL, *other_member = NULL; uint32_t id = atoi(argv[2]); uint32_t oid = atoi(argv[3]); @@ -7647,11 +7601,6 @@ static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_ if (member && other_member) { conference_relationship_t *rel = NULL; - if (sendvideo && switch_test_flag(other_member, MFLAG_RECEIVING_VIDEO) && (! (nospeak || nohear))) { - stream->write_function(stream, "member %d already receiving video", oid); - goto skip; - } - if ((rel = member_get_relationship(member, other_member))) { rel->flags = 0; } else { @@ -7667,13 +7616,7 @@ static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_ if (nohear) { switch_clear_flag(rel, RFLAG_CAN_HEAR); } - if (sendvideo) { - switch_set_flag(rel, RFLAG_CAN_SEND_VIDEO); - switch_set_flag(other_member, MFLAG_RECEIVING_VIDEO); - switch_core_session_refresh_video(member->session); - } - - stream->write_function(stream, "ok %u->%u %s set\n", id, oid, argv[4]); + stream->write_function(stream, "ok %u->%u set\n", id, oid); } else { stream->write_function(stream, "error!\n"); } @@ -7681,7 +7624,6 @@ static switch_status_t conf_api_sub_relate(conference_obj_t *conference, switch_ stream->write_function(stream, "relationship %u->%u not found.\n", id, oid); } -skip: if (member) { switch_thread_rwlock_unlock(member->rwlock); } @@ -9785,16 +9727,11 @@ SWITCH_STANDARD_APP(conference_function) msg.message_id = SWITCH_MESSAGE_INDICATE_BRIDGE; switch_core_session_receive_message(session, &msg); - /* Chime in the core video thread */ - switch_core_session_set_video_thread_callback(session, video_thread_callback, (void *)&member); - /* Run the conference loop */ do { conference_loop_output(&member); } while (member.loop_loop); - switch_core_session_set_video_thread_callback(session, NULL, NULL); - switch_channel_set_private(channel, "_conference_autocall_list_", NULL); /* Tell the channel we are no longer going to be in a bridge */ diff --git a/src/mod/applications/mod_fsv/mod_fsv.c b/src/mod/applications/mod_fsv/mod_fsv.c index f3a807c065..e49c517fae 100644 --- a/src/mod/applications/mod_fsv/mod_fsv.c +++ b/src/mod/applications/mod_fsv/mod_fsv.c @@ -673,7 +673,6 @@ SWITCH_STANDARD_APP(play_yuv_function) if (read_frame) switch_core_session_write_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); { /* video part */ - uint32_t flag = 0; uint32_t encoded_data_len = 1500; switch_frame_t *frame = &vid_frame; switch_time_t now = switch_micro_time_now() / 1000; @@ -691,14 +690,12 @@ SWITCH_STANDARD_APP(play_yuv_function) sprintf(ts_str, "%u", timestamp); text(img->planes[SWITCH_PLANE_PACKED], width, 20, 20, ts_str); - switch_core_codec_encode_video(codec, img, vid_frame.data, &encoded_data_len, &flag); + vid_frame.img = img; + switch_core_codec_encode_video(codec, &vid_frame); while(encoded_data_len) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%lld\n", codec->implementation->iananame, encoded_data_len, flag, last_video_ts); - frame->datalen = encoded_data_len; - frame->packetlen = frame->datalen + 12; - frame->m = flag & SFF_MARKER ? 1 : 0; frame->timestamp = timestamp; if (1) { // we can remove this when ts and marker full passed in core @@ -715,8 +712,8 @@ SWITCH_STANDARD_APP(play_yuv_function) switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); - encoded_data_len = 1500; - switch_core_codec_encode_video(codec, NULL, vid_frame.data, &encoded_data_len, &flag); + vid_frame.datalen = 1500; + switch_core_codec_encode_video(codec, &vid_frame); } } } @@ -817,10 +814,7 @@ SWITCH_STANDARD_APP(decode_video_function) } if ( 1 ) { /* video part */ - uint32_t flag = 0; - switch_image_t *img = NULL; - - switch_core_codec_decode_video(codec, frame, &img, &flag); + switch_core_codec_decode_video(codec, frame); if ((switch_test_flag(frame, SFF_WAIT_KEY_FRAME))) { switch_time_t now = switch_micro_time_now(); @@ -831,19 +825,19 @@ SWITCH_STANDARD_APP(decode_video_function) continue; } - if (img) { - if (img->d_w > 0 && !width) { - width = img->d_w; + if (frame->img) { + if (frame->img->d_w > 0 && !width) { + width = frame->img->d_w; switch_channel_set_variable_printf(channel, "video_width", "%d", width); } - if (img->d_h > 0 && !height) { - height = img->d_h; + if (frame->img->d_h > 0 && !height) { + height = frame->img->d_h; switch_channel_set_variable_printf(channel, "video_height", "%d", height); } decoded_pictures++; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "picture#%d %dx%d\n", decoded_pictures, img->d_w, img->d_h); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "picture#%d %dx%d\n", decoded_pictures, frame->img->d_w, frame->img->d_h); if (max_pictures && (decoded_pictures >= max_pictures)) { break; diff --git a/src/mod/codecs/mod_openh264/mod_openh264.cpp b/src/mod/codecs/mod_openh264/mod_openh264.cpp index 1b88dbce6c..b561eb2466 100644 --- a/src/mod/codecs/mod_openh264/mod_openh264.cpp +++ b/src/mod/codecs/mod_openh264/mod_openh264.cpp @@ -39,7 +39,7 @@ #include "codec_api.h" //#include "inc/logging.h" // for debug -#define FPS 15.0f // frame rate +#define FPS 20.0f // frame rate #define H264_NALU_BUFFER_SIZE 65536 #define MAX_NALUS 100 #define SLICE_SIZE 1200 //NALU Slice Size @@ -66,15 +66,16 @@ typedef struct h264_codec_context_s { switch_image_t *img; int got_sps; int64_t pts; + int need_key_frame; switch_size_t last_received_timestamp; switch_bool_t last_received_complete_picture; } h264_codec_context_t; int FillSpecificParameters(SEncParamExt& param) { /* Test for temporal, spatial, SNR scalability */ - param.iPicWidth = 352; // width of picture in samples - param.iPicHeight = 288; // height of picture in samples - param.iTargetBitrate = 384000; // target bitrate desired + param.iPicWidth = 1280; // width of picture in samples + param.iPicHeight = 720; // height of picture in samples + param.iTargetBitrate = 1280 * 720 * 8; // target bitrate desired param.iRCMode = RC_QUALITY_MODE; // rc mode control param.uiMaxNalSize = SLICE_SIZE * 20; param.iTemporalLayerNum = 1; // layer number at temporal level @@ -91,11 +92,11 @@ int FillSpecificParameters(SEncParamExt& param) { param.bPrefixNalAddingCtrl = 0; int iIndexLayer = 0; - param.sSpatialLayers[iIndexLayer].iVideoWidth = 352; - param.sSpatialLayers[iIndexLayer].iVideoHeight = 288; - param.sSpatialLayers[iIndexLayer].fFrameRate = 15.0f; + param.sSpatialLayers[iIndexLayer].iVideoWidth = 1280; + param.sSpatialLayers[iIndexLayer].iVideoHeight = 720; + param.sSpatialLayers[iIndexLayer].fFrameRate = (double) (FPS * 1.0f); // param.sSpatialLayers[iIndexLayer].iQualityLayerNum = 1; - param.sSpatialLayers[iIndexLayer].iSpatialBitrate = 384000; + param.sSpatialLayers[iIndexLayer].iSpatialBitrate = 1280 * 720 * 8; #ifdef MT_ENABLED param.sSpatialLayers[iIndexLayer].sSliceCfg.uiSliceMode = SM_DYN_SLICE; @@ -137,27 +138,21 @@ static switch_size_t buffer_h264_nalu(h264_codec_context_t *context, switch_fram switch_buffer_t *buffer = context->nalu_buffer; switch_size_t size = 0; - if (!frame) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "No frame in codec!!\n"); - return size; - } - + switch_assert(frame); + nalu_idc = (nalu_hdr & 0x60) >> 5; nalu_type = nalu_hdr & 0x1f; if (!context->got_sps && nalu_type != 7) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Waiting SPS/PPS\n"); - switch_set_flag(frame, SFF_WAIT_KEY_FRAME); - return size; + return 0; } if (!context->got_sps) context->got_sps = 1; size = switch_buffer_write(buffer, sync_bytes, sizeof(sync_bytes)); - if (size == 0 ) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Buffer Memory Error!\n"); - size = switch_buffer_write(buffer, frame->data, frame->datalen); - if (size == 0 ) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Buffer Memory Error!\n"); + #ifdef DEBUG_H264 switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "ts: %ld len: %4d %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x mark=%d size=%d\n", @@ -173,13 +168,13 @@ static switch_size_t buffer_h264_nalu(h264_codec_context_t *context, switch_fram return size; } -static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uint32_t *len, uint32_t *flag) +static switch_status_t nalu_slice(h264_codec_context_t *context, switch_frame_t *frame) { int nalu_len; uint8_t *buffer; switch_status_t status = SWITCH_STATUS_SUCCESS; - *flag &= ~SFF_MARKER; + frame->m = SWITCH_FALSE; if (context->cur_nalu_index >= context->bit_stream_info.sLayerInfo[context->cur_layer].iNalCount) { context->cur_nalu_index = 0; @@ -198,8 +193,8 @@ static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uin if (context->last_frame_type == videoFrameTypeSkip || context->cur_layer >= context->bit_stream_info.iLayerNum) { - *len = 0; - *flag |= SFF_MARKER; + frame->datalen = 0; + frame->m = SWITCH_TRUE; context->cur_layer = 0; context->cur_nalu_index = 0; return status; @@ -225,12 +220,12 @@ static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uin // if (nalu_type == 7) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Got SPS\n"); - memcpy(data, (buffer + context->last_nalu_data_pos), nalu_len); - *len = nalu_len; + memcpy(frame->data, (buffer + context->last_nalu_data_pos), nalu_len); + frame->datalen = nalu_len; // *flag |= (nalu_type == 6 || nalu_type == 7 || nalu_type == 8 || (nalu_type == 0xe && context->last_nalu_type == 8)) ? 0 : SFF_MARKER; if ((context->cur_nalu_index == context->bit_stream_info.sLayerInfo[context->cur_layer].iNalCount - 1) && (context->cur_layer == context->bit_stream_info.iLayerNum - 1)) { - *flag |= SFF_MARKER; + frame->m = SWITCH_TRUE; } else { status = SWITCH_STATUS_MORE_DATA; } @@ -240,7 +235,7 @@ static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uin goto end; } else { int left = nalu_len; - uint8_t *p = (uint8_t *)data; + uint8_t *p = (uint8_t *) frame->data; if (context->nalu_eat) { left = nalu_len + 4 - context->nalu_eat; @@ -267,7 +262,7 @@ static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uin memcpy(p + 2, buffer + context->last_nalu_data_pos, SLICE_SIZE - 2); context->last_nalu_data_pos += (SLICE_SIZE - 2); context->nalu_eat += (SLICE_SIZE - 2); - *len = SLICE_SIZE; + frame->datalen = SLICE_SIZE; status = SWITCH_STATUS_MORE_DATA; goto end; } else { @@ -275,8 +270,8 @@ static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uin p[1] = 0x40 | context->last_nalu_type; memcpy(p + 2, buffer + context->last_nalu_data_pos, left); context->last_nalu_data_pos += left; - *len = left + 2; - *flag |= SFF_MARKER; + frame->datalen = left + 2; + frame->m = SWITCH_TRUE; context->nalu_eat = 0; context->cur_nalu_index++; status = SWITCH_STATUS_MORE_DATA; @@ -361,6 +356,7 @@ static switch_status_t init_encoder(h264_codec_context_t *context, uint32_t widt context->encoder_params.iPicWidth = width; context->encoder_params.iPicHeight = height; + for (int i=0; iencoder_params.iSpatialLayerNum; i++) { context->encoder_params.sSpatialLayers[i].iVideoWidth = width; context->encoder_params.sSpatialLayers[i].iVideoHeight = height; @@ -371,14 +367,12 @@ static switch_status_t init_encoder(h264_codec_context_t *context, uint32_t widt switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder Init Error\n"); return SWITCH_STATUS_FALSE; } + context->encoder_initialized = SWITCH_TRUE; return SWITCH_STATUS_SUCCESS; } -static switch_status_t switch_h264_encode(switch_codec_t *codec, - switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, - unsigned int *flag) +static switch_status_t switch_h264_encode(switch_codec_t *codec, switch_frame_t *frame) { h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; int width = 0; @@ -387,17 +381,24 @@ static switch_status_t switch_h264_encode(switch_codec_t *codec, SSourcePicture* pic = NULL; long result; - if (*flag & SFF_WAIT_KEY_FRAME) { + if (context->need_key_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "H264 KEYFRAME GENERATED\n"); context->encoder->ForceIntraFrame(1); + context->need_key_frame = 0; } - if (img == NULL) { - return nalu_slice(context, encoded_data, encoded_data_len, flag); + if (frame->flags & SFF_SAME_IMAGE) { + return nalu_slice(context, frame); } - //d_w and d_h are corrupt - width = img->w; - height = img->h; + + if (frame->img->d_h > 1) { + width = frame->img->d_w; + height = frame->img->d_h; + } else { + width = frame->img->w; + height = frame->img->h; + } //switch_assert(width > 0 && (width % 2 == 0)); //switch_assert(height > 0 && (height % 2 == 0)); @@ -419,12 +420,12 @@ static switch_status_t switch_h264_encode(switch_codec_t *codec, pic->iColorFormat = videoFormatI420; pic->iPicHeight = height; pic->iPicWidth = width; - pic->iStride[0] = img->stride[0]; - pic->iStride[1] = img->stride[1]; // = img->stride[2]; + pic->iStride[0] = frame->img->stride[0]; + pic->iStride[1] = frame->img->stride[1]; // = frame->img->stride[2]; - pic->pData[0] = img->planes[0]; - pic->pData[1] = img->planes[1]; - pic->pData[2] = img->planes[2]; + pic->pData[0] = frame->img->planes[0]; + pic->pData[1] = frame->img->planes[1]; + pic->pData[2] = frame->img->planes[2]; result = (EVideoFrameType)context->encoder->EncodeFrame(pic, &context->bit_stream_info); if (result != cmResultSuccess ) { @@ -436,45 +437,48 @@ static switch_status_t switch_h264_encode(switch_codec_t *codec, context->cur_nalu_index = 0; context->last_nalu_data_pos = 0; - if(pic){ + if (pic){ delete pic; pic = NULL; } - return nalu_slice(context, encoded_data, encoded_data_len, flag); + + return nalu_slice(context, frame); error: + if(pic){ delete pic; pic = NULL; } - *encoded_data_len = 0; - *flag |= SFF_MARKER; + frame->datalen = 0; + frame->m = SWITCH_TRUE; + return SWITCH_STATUS_FALSE; } -static switch_status_t switch_h264_decode(switch_codec_t *codec, - switch_frame_t *frame, - switch_image_t **img, - unsigned int *flag) +static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t *frame) { h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; switch_size_t size = 0; uint32_t error_code; + switch_status_t status = SWITCH_STATUS_SUCCESS; // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %u mark:%d\n", frame->datalen, ntohl(frame->timestamp), frame->m); - if (context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && + if (0 && context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && (!frame->m) && (!context->last_received_complete_picture)) { // possible packet loss switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Packet Loss, skip privousely received packets\n"); - switch_buffer_zero(context->nalu_buffer); + switch_goto_status(SWITCH_STATUS_RESTART, end); } context->last_received_timestamp = frame->timestamp; context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE; size = buffer_h264_nalu(context, frame); + printf("READ buf:%ld got_key:%d st:%d m:%d\n", size, context->got_sps, status, frame->m); + if (frame->m && size) { int got_picture = 0; @@ -485,6 +489,11 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, SBufferInfo dest_buffer_info; switch_buffer_peek_zerocopy(context->nalu_buffer, &nalu); uint8_t* pData[3] = { 0 }; + + + frame->m = SWITCH_FALSE; + frame->flags = 0; + pData[0] = NULL; pData[1] = NULL; @@ -515,22 +524,40 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, context->img->stride[1] = dest_buffer_info.UsrData.sSystemBuffer.iStride[1]; context->img->stride[2] = dest_buffer_info.UsrData.sSystemBuffer.iStride[1]; - *img = context->img; + frame->img = context->img; // TODO: keep going and see if more picture available // pDecoder->DecodeFrame (NULL, 0, pData, &sDstBufInfo); } else { if (error_code) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decode error: 0x%x\n", error_code); - context->got_sps = 0; + switch_goto_status(SWITCH_STATUS_RESTART, end); } } switch_buffer_zero(context->nalu_buffer); - return SWITCH_STATUS_SUCCESS; + status = SWITCH_STATUS_SUCCESS; } end: - return SWITCH_STATUS_SUCCESS; + + if (size == 0) { + status == SWITCH_STATUS_MORE_DATA; + } + + if (status == SWITCH_STATUS_RESTART) { + context->got_sps = 0; + switch_buffer_zero(context->nalu_buffer); + } + + if (!context->got_sps) { + switch_set_flag(frame, SFF_WAIT_KEY_FRAME); + } + + if (!frame->img) { + status = SWITCH_STATUS_MORE_DATA; + } + + return status; } static switch_status_t switch_h264_control(switch_codec_t *codec, @@ -540,6 +567,18 @@ static switch_status_t switch_h264_control(switch_codec_t *codec, switch_codec_control_type_t *rtype, void **ret_data) { + + + h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; + + switch(cmd) { + case SCC_VIDEO_REFRESH: + context->need_key_frame = 1; + break; + default: + break; + } + return SWITCH_STATUS_SUCCESS; } diff --git a/src/mod/codecs/mod_vpx/mod_vpx.c b/src/mod/codecs/mod_vpx/mod_vpx.c index 2c0028cde5..7806ecfe46 100644 --- a/src/mod/codecs/mod_vpx/mod_vpx.c +++ b/src/mod/codecs/mod_vpx/mod_vpx.c @@ -38,7 +38,7 @@ #include #include -#define FPS 15 +#define FPS 20 #define SLICE_SIZE 1200 SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load); @@ -84,6 +84,24 @@ static switch_status_t init_codec(switch_codec_t *codec) vpx_context_t *context = (vpx_context_t *)codec->private_info; vpx_codec_enc_cfg_t *config = &context->config; + if (!context->codec_settings.video.width) { + context->codec_settings.video.width = 1280; + } + + if (!context->codec_settings.video.height) { + context->codec_settings.video.height = 720; + } + + if (context->codec_settings.video.bandwidth) { + context->bandwidth = context->codec_settings.video.bandwidth; + } else { + context->bandwidth = context->codec_settings.video.width * context->codec_settings.video.height * 8; + } + + if (context->bandwidth > 1250000) { + context->bandwidth = 1250000; + } + // settings config->g_profile = 1; config->g_w = context->codec_settings.video.width; @@ -93,7 +111,8 @@ static switch_status_t init_codec(switch_codec_t *codec) config->g_timebase.den = 1000; config->g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS; config->g_lag_in_frames = 0; // 0- no frame lagging - config->g_threads = 1; + config->g_threads = (switch_core_cpu_count() > 1) ? 2 : 1; + // rate control settings config->rc_dropframe_thresh = 0; config->rc_end_usage = VPX_CBR; @@ -198,6 +217,8 @@ static switch_status_t init_codec(switch_codec_t *codec) vpx_codec_control(&context->decoder, VP8_SET_POSTPROC, &ppcfg); switch_buffer_create_dynamic(&context->vpx_packet_buffer, 512, 512, 1024000); + + printf("WTF CREATE ??? %p\n", (void *)context->vpx_packet_buffer); } return SWITCH_STATUS_SUCCESS; @@ -274,11 +295,11 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ +-+-+-+-+-+-+-+-+ */ -static switch_status_t consume_partition(vpx_context_t *context, void *data, uint32_t *len, uint32_t *flag) +static switch_status_t consume_partition(vpx_context_t *context, switch_frame_t *frame) { if (!context->pkt) context->pkt = vpx_codec_get_cx_data(&context->encoder, &context->iter); - *flag &= ~SFF_MARKER; + frame->m = 0; if (context->pkt) { // if (context->pkt->kind == VPX_CODEC_CX_FRAME_PKT && (context->pkt->data.frame.flags & VPX_FRAME_IS_KEY) && context->pkt_pos == 0) { @@ -290,8 +311,8 @@ static switch_status_t consume_partition(vpx_context_t *context, void *data, uin } if (!context->pkt || context->pkt_pos >= context->pkt->data.frame.sz - 1 || context->pkt->kind != VPX_CODEC_CX_FRAME_PKT) { - *len = 0; - *flag |= SFF_MARKER; + frame->datalen = 0; + frame->m = 1; context->pkt_pos = 0; context->pkt = NULL; return SWITCH_STATUS_SUCCESS; @@ -300,24 +321,24 @@ static switch_status_t consume_partition(vpx_context_t *context, void *data, uin if (context->pkt->data.frame.sz < SLICE_SIZE) { uint8_t hdr = 0x10; - memcpy(data, &hdr, 1); - memcpy((uint8_t *)data + 1, context->pkt->data.frame.buf, context->pkt->data.frame.sz); - *len = context->pkt->data.frame.sz + 1; - *flag |= SFF_MARKER; + memcpy(frame->data, &hdr, 1); + memcpy((uint8_t *)frame->data + 1, context->pkt->data.frame.buf, context->pkt->data.frame.sz); + frame->datalen = context->pkt->data.frame.sz + 1; + frame->m = 1; context->pkt = NULL; context->pkt_pos = 0; return SWITCH_STATUS_SUCCESS; } else { int left = context->pkt->data.frame.sz - context->pkt_pos; - uint8_t *p = data; + uint8_t *p = frame->data; if (left < SLICE_SIZE) { p[0] = 0; memcpy(p+1, (uint8_t *)context->pkt->data.frame.buf + context->pkt_pos, left); context->pkt_pos = 0; context->pkt = NULL; - *len = left + 1; - *flag |= SFF_MARKER; + frame->datalen = left + 1; + frame->m = 1; return SWITCH_STATUS_SUCCESS; } else { uint8_t hdr = context->pkt_pos == 0 ? 0x10 : 0; @@ -325,15 +346,13 @@ static switch_status_t consume_partition(vpx_context_t *context, void *data, uin p[0] = hdr; memcpy(p+1, (uint8_t *)context->pkt->data.frame.buf + context->pkt_pos, SLICE_SIZE - 1); context->pkt_pos += (SLICE_SIZE - 1); - *len = SLICE_SIZE; + frame->datalen = SLICE_SIZE; return SWITCH_STATUS_MORE_DATA; } } } -static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, - unsigned int *flag) +static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_frame_t *frame) { vpx_context_t *context = (vpx_context_t *)codec->private_info; uint32_t duration = 90000 / FPS; @@ -341,20 +360,21 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t * int height = 0; vpx_enc_frame_flags_t vpx_flags = 0; - if (*flag & SFF_WAIT_KEY_FRAME) { - context->need_key_frame = 1; - } - if (img == NULL) { - return consume_partition(context, encoded_data, encoded_data_len, flag); + if (frame->flags & SFF_SAME_IMAGE) { + return consume_partition(context, frame); } //d_w and d_h are messed up + //printf("WTF %d %d\n", frame->img->d_w, frame->img->d_h); - //printf("WTF %d %d\n", img->d_w, img->d_h); - - width = img->w; - height = img->h; + if (frame->img->d_h > 1) { + width = frame->img->d_w; + height = frame->img->d_h; + } else { + width = frame->img->w; + height = frame->img->h; + } //switch_assert(width > 0 && (width % 4 == 0)); //switch_assert(height > 0 && (height % 4 == 0)); @@ -362,45 +382,43 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t * if (context->config.g_w != width || context->config.g_h != height) { context->codec_settings.video.width = width; context->codec_settings.video.height = height; - if (context->codec_settings.video.bandwidth) { - context->bandwidth = context->codec_settings.video.bandwidth; - } else { - context->bandwidth = width * height * 8; - } - - if (context->bandwidth > 1250000) { - context->bandwidth = 1250000; - } switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(codec->session), SWITCH_LOG_NOTICE, "VPX reset encoder picture from %dx%d to %dx%d %u BW\n", context->config.g_w, context->config.g_h, width, height, context->bandwidth); init_codec(codec); - *flag |= SFF_PICTURE_RESET; + frame->flags |= SFF_PICTURE_RESET; context->need_key_frame = 1; } - if (context->need_key_frame > 0) { - // force generate a key frame - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX KEYFRAME REQ\n"); - vpx_flags |= VPX_EFLAG_FORCE_KF; - context->need_key_frame--; + + if (!context->encoder_init) { + init_codec(codec); } - if (vpx_codec_encode(&context->encoder, (vpx_image_t *)img, context->pts, duration, vpx_flags, VPX_DL_REALTIME) != VPX_CODEC_OK) { + if (context->need_key_frame != 0) { + // force generate a key frame + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX KEYFRAME GENERATED\n"); + vpx_flags |= VPX_EFLAG_FORCE_KF; + context->need_key_frame = 0; + } + + if (vpx_codec_encode(&context->encoder, (vpx_image_t *) frame->img, context->pts, duration, vpx_flags, VPX_DL_REALTIME) != VPX_CODEC_OK) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "VP8 encode error %d:%s\n", context->encoder.err, context->encoder.err_detail); + + frame->datalen = 0; return SWITCH_STATUS_FALSE; } context->pts += duration; context->iter = NULL; - return consume_partition(context, encoded_data, encoded_data_len, flag); + return consume_partition(context, frame); } -static void buffer_vpx_packets(vpx_context_t *context, switch_frame_t *frame) +static switch_status_t buffer_vpx_packets(vpx_context_t *context, switch_frame_t *frame) { uint8_t *data = frame->data; uint8_t S; @@ -410,7 +428,7 @@ static void buffer_vpx_packets(vpx_context_t *context, switch_frame_t *frame) if (!frame) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no frame in codec!!\n"); - return; + return SWITCH_STATUS_RESTART; } DES = *data; @@ -431,106 +449,119 @@ static void buffer_vpx_packets(vpx_context_t *context, switch_frame_t *frame) } len = frame->datalen - (data - (uint8_t *)frame->data); + if (len <= 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid packet %d\n", len); - switch_buffer_zero(context->vpx_packet_buffer); - return; + return SWITCH_STATUS_RESTART; } - + if (S && (PID == 0)) { - uint8_t keyframe; + int is_keyframe = ((*data) & 0x01) ? 0 : 1; - keyframe = ((*data) & 0x01) ? 0 : 1; - - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "[%d] PID: %d K:%d P:%d Inv:%d len: %d size:%d\n", frame->datalen, PID, keyframe, profile, invisible, len, size); - - if (keyframe) { - if (!context->got_key_frame) context->got_key_frame = 1; + if (is_keyframe && !context->got_key_frame) { + context->got_key_frame = 1; } } if (!context->got_key_frame) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Waiting for key frame\n"); - switch_set_flag(frame, SFF_WAIT_KEY_FRAME); - return; + return SWITCH_STATUS_RESTART; } switch_buffer_write(context->vpx_packet_buffer, data, len); + + return SWITCH_STATUS_SUCCESS; + } -static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *frame, switch_image_t **img, unsigned int *flag) +static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *frame) { vpx_context_t *context = (vpx_context_t *)codec->private_info; - vpx_codec_ctx_t *decoder = &context->decoder; switch_size_t len; + vpx_codec_ctx_t *decoder = NULL; + switch_status_t status = SWITCH_STATUS_SUCCESS; - if (!decoder) { + if (!context->decoder_init) { + init_codec(codec); + } + + if (!context->decoder_init) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX decoder is not initialized!\n"); return SWITCH_STATUS_FALSE; } + decoder = &context->decoder; + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %" SWITCH_SIZE_T_FMT " mark:%d\n", frame->datalen, frame->timestamp, frame->m); - if (context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && + if (context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && (!frame->m) && (!context->last_received_complete_picture)) { // possible packet loss - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Packet Loss, skip previouse received frame (to avoid crash?)\n"); - switch_buffer_zero(context->vpx_packet_buffer); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Packet Loss, skip previous received frame (to avoid crash?)\n"); + switch_goto_status(SWITCH_STATUS_RESTART, end); } context->last_received_timestamp = frame->timestamp; context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE; - buffer_vpx_packets(context, frame); + status = buffer_vpx_packets(context, frame); - len = switch_buffer_inuse(context->vpx_packet_buffer); + printf("READ buf:%ld got_key:%d st:%d m:%d\n", switch_buffer_inuse(context->vpx_packet_buffer), context->got_key_frame, status, frame->m); - if (frame->m && len) { + if (status == SWITCH_STATUS_SUCCESS && frame->m && (len = switch_buffer_inuse(context->vpx_packet_buffer))) { uint8_t *data; vpx_codec_iter_t iter = NULL; int corrupted = 0; int err; - // int keyframe = 0; + //int keyframe = 0; + + printf("WTF %d %ld\n", frame->m, len); switch_buffer_peek_zerocopy(context->vpx_packet_buffer, (void *)&data); - // keyframe = (*data & 0x01) ? 0 : 1; + //keyframe = (*data & 0x01) ? 0 : 1; - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffered: %" SWITCH_SIZE_T_FMT ", key: %d\n", len, keyframe); + //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffered: %" SWITCH_SIZE_T_FMT ", key: %d\n", len, keyframe); err = vpx_codec_decode(decoder, data, (unsigned int)len, NULL, 0); if (err != VPX_CODEC_OK) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error decoding %" SWITCH_SIZE_T_FMT " bytes, [%d:%d:%s]\n", len, err, decoder->err, decoder->err_detail); - switch_set_flag(frame, SFF_WAIT_KEY_FRAME); - context->got_key_frame = 0; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "require key frame %d\n", context->got_key_frame); - goto error; + switch_goto_status(SWITCH_STATUS_RESTART, end); } if (vpx_codec_control(decoder, VP8D_GET_FRAME_CORRUPTED, &corrupted) != VPX_CODEC_OK) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX control error!\n"); - goto error; + switch_goto_status(SWITCH_STATUS_RESTART, end); } - *img = (switch_image_t *)vpx_codec_get_frame(decoder, &iter); + frame->img = (switch_image_t *) vpx_codec_get_frame(decoder, &iter); - if (!(*img) || corrupted) { + if (!(frame->img) || corrupted) { switch_buffer_zero(context->vpx_packet_buffer); - goto ok; + switch_goto_status(SWITCH_STATUS_SUCCESS, end); } - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%dx%d %dx%d\n", (*img)->w,(*img)->h, (*img)->d_w, (*img)->d_h); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%dx%d %dx%d\n", frame->img->w,frame->img->h, frame->img->d_w, frame->img->d_h); switch_buffer_zero(context->vpx_packet_buffer); } -ok: - return SWITCH_STATUS_SUCCESS; +end: -error: - switch_buffer_zero(context->vpx_packet_buffer); + if (status == SWITCH_STATUS_RESTART) { + context->got_key_frame = 0; + switch_buffer_zero(context->vpx_packet_buffer); + } - return SWITCH_STATUS_FALSE; + if (!frame->img) { + status = SWITCH_STATUS_MORE_DATA; + } + + if (!context->got_key_frame) { + switch_set_flag(frame, SFF_WAIT_KEY_FRAME); + } + + return status; } diff --git a/src/mod/codecs/mod_yuv/mod_yuv.c b/src/mod/codecs/mod_yuv/mod_yuv.c index a1f38a929b..a9026d697b 100644 --- a/src/mod/codecs/mod_yuv/mod_yuv.c +++ b/src/mod/codecs/mod_yuv/mod_yuv.c @@ -50,25 +50,19 @@ static switch_status_t switch_yuv_init(switch_codec_t *codec, switch_codec_flag_ } } -static switch_status_t switch_yuv_encode(switch_codec_t *codec, - switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, - unsigned int *flag) +static switch_status_t switch_yuv_encode(switch_codec_t *codec, switch_frame_t *frame) { /* yuv encode is unclear, so return 0 for now */ - *encoded_data_len = 0; - *flag |= SFF_MARKER; + frame->datalen = 0; + frame->m = 1; return SWITCH_STATUS_SUCCESS; } -static switch_status_t switch_yuv_decode(switch_codec_t *codec, - switch_frame_t *frame, - switch_image_t **img, - unsigned int *flag) +static switch_status_t switch_yuv_decode(switch_codec_t *codec, switch_frame_t *frame) { switch_assert(frame); - *img = (switch_image_t *)frame->user_data; + frame->img = (switch_image_t *)frame->user_data; return SWITCH_STATUS_SUCCESS; } diff --git a/src/mod/formats/mod_vlc/mod_vlc.c b/src/mod/formats/mod_vlc/mod_vlc.c index 28c5e576b8..ce1f9bd553 100644 --- a/src/mod/formats/mod_vlc/mod_vlc.c +++ b/src/mod/formats/mod_vlc/mod_vlc.c @@ -114,7 +114,6 @@ struct vlc_video_context { int height; int force_width; int force_height; - int video_refresh_req; int channels; }; @@ -242,6 +241,7 @@ static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixel switch_assert(context->img); yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height); + switch_mutex_unlock(context->video_mutex); } @@ -267,14 +267,12 @@ static void vlc_video_channel_unlock_callback(void *data, void *id, void *const { vlc_video_context_t *context = (vlc_video_context_t *)data; - switch_assert(id == NULL); /* picture identifier, not needed here */ if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return; - if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0); - switch_assert(context->img); - + if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0); switch_assert(context->img); + yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height); switch_mutex_unlock(context->video_mutex); @@ -282,19 +280,15 @@ static void vlc_video_channel_unlock_callback(void *data, void *id, void *const static void vlc_video_display_callback(void *data, void *id) { - vlc_video_context_t *context = (vlc_video_context_t *) data; - int32_t flag = 0; - /* VLC wants to display the video */ + vlc_video_context_t *context = (vlc_video_context_t *) data; + if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return; - if (context->video_refresh_req > 0) { - flag |= SFF_WAIT_KEY_FRAME; - context->video_refresh_req--; - } + context->vid_frame->img = context->img; - switch_core_session_write_video_image(context->session, context->vid_frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL); + switch_core_session_write_video_frame(context->session, context->vid_frame, SWITCH_IO_FLAG_NONE, 0); } unsigned video_format_setup_callback(void **opaque, char *chroma, unsigned *width, unsigned *height, unsigned *pitches, unsigned *lines) @@ -321,6 +315,7 @@ unsigned video_format_setup_callback(void **opaque, char *chroma, unsigned *widt frame_size = (*width) * (*height) * 4 * 2; context->raw_yuyv_data = malloc(frame_size); + if (context->raw_yuyv_data == NULL) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "memory error\n"); return 0; @@ -1200,7 +1195,7 @@ static switch_call_cause_t vlc_outgoing_channel(switch_core_session_t *session, goto fail; } - start_core_video_thread(*new_session); + switch_core_session_start_video_thread(*new_session); switch_channel_set_state(channel, CS_INIT); if (switch_core_session_thread_launch(*new_session) != SWITCH_STATUS_SUCCESS) { @@ -1390,7 +1385,7 @@ static switch_status_t vlc_receive_message(switch_core_session_t *session, switc case SWITCH_MESSAGE_INDICATE_JITTER_BUFFER: break; case SWITCH_MESSAGE_INDICATE_VIDEO_REFRESH_REQ: - tech_pvt->context->video_refresh_req = 1; + switch_core_media_gen_key_frame(session); break; default: break; diff --git a/src/switch_core_codec.c b/src/switch_core_codec.c index 0e392edcbb..480a5a046f 100644 --- a/src/switch_core_codec.c +++ b/src/switch_core_codec.c @@ -788,9 +788,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode(switch_codec_t *codec, return status; } -SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec, - switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag) +SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec, switch_frame_t *frame) { switch_status_t status = SWITCH_STATUS_FALSE; @@ -809,7 +807,14 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *c if (codec->mutex) switch_mutex_lock(codec->mutex); if (codec->implementation->encode_video) { - status = codec->implementation->encode_video(codec, img, encoded_data, encoded_data_len, flag); + status = codec->implementation->encode_video(codec, frame); + + if (frame->datalen) { + frame->packetlen = frame->datalen + 12; + frame->flags |= SFF_SAME_IMAGE; + } else { + frame->flags &= ~SFF_SAME_IMAGE; + } } if (codec->mutex) switch_mutex_unlock(codec->mutex); @@ -818,9 +823,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *c } -SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec, - switch_frame_t *frame, - switch_image_t **img, unsigned int *flag) +SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec, switch_frame_t *frame) { switch_status_t status = SWITCH_STATUS_FALSE; @@ -840,7 +843,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *c if (codec->mutex) switch_mutex_lock(codec->mutex); if (codec->implementation->decode_video) { - status = codec->implementation->decode_video(codec, frame, img, flag); + status = codec->implementation->decode_video(codec, frame); } if (codec->mutex) switch_mutex_unlock(codec->mutex); diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 683fab8073..2df5d3b5ec 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -176,6 +176,7 @@ struct switch_media_handle_s { switch_mutex_t *mutex; switch_mutex_t *sdp_mutex; + switch_mutex_t *control_mutex; const switch_codec_implementation_t *negotiated_codecs[SWITCH_MAX_CODECS]; int num_negotiated_codecs; @@ -196,6 +197,9 @@ struct switch_media_handle_s { switch_time_t video_last_key_time; switch_time_t video_init; switch_timer_t video_timer; + switch_video_function_t video_function; + void *video_user_data; + int8_t video_function_running; }; @@ -1510,7 +1514,7 @@ SWITCH_DECLARE(switch_status_t) switch_media_handle_create(switch_media_handle_t session->media_handle->mparams = params; if (!session->media_handle->mparams->video_key_freq) { - session->media_handle->mparams->video_key_freq = 15000000; + session->media_handle->mparams->video_key_freq = 30000000; } if (!session->media_handle->mparams->video_key_first) { @@ -1524,6 +1528,7 @@ SWITCH_DECLARE(switch_status_t) switch_media_handle_create(switch_media_handle_t switch_mutex_init(&session->media_handle->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session)); switch_mutex_init(&session->media_handle->sdp_mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session)); + switch_mutex_init(&session->media_handle->control_mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session)); session->media_handle->engines[SWITCH_MEDIA_TYPE_AUDIO].ssrc = (uint32_t) ((intptr_t) &session->media_handle->engines[SWITCH_MEDIA_TYPE_AUDIO] + (uint32_t) time(NULL)); @@ -1804,14 +1809,12 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session engine = &smh->engines[type]; - engine->read_frame.datalen = 0; - if (!engine->read_codec.implementation || !switch_core_codec_ready(&engine->read_codec)) { return SWITCH_STATUS_FALSE; } switch_assert(engine->rtp_session != NULL); - engine->read_frame.datalen = 0; + if (!switch_channel_up_nosig(session->channel) || !switch_rtp_ready(engine->rtp_session) || switch_channel_test_flag(session->channel, CF_NOT_READY)) { return SWITCH_STATUS_FALSE; @@ -1824,7 +1827,12 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session return SWITCH_STATUS_INUSE; } - + + engine->read_frame.datalen = 0; + engine->read_frame.flags = SFF_NONE; + engine->read_frame.m = SWITCH_FALSE; + engine->read_frame.img = NULL; + while (smh->media_flags[SCMF_RUNNING] && engine->read_frame.datalen == 0) { engine->read_frame.flags = SFF_NONE; @@ -4449,6 +4457,25 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi continue; } + if (smh->video_function) { + int run = 0; + + switch_mutex_lock(smh->control_mutex); + if (smh->video_function_running == 0) { + smh->video_function_running = 1; + run = 1; + } + switch_mutex_unlock(smh->control_mutex); + + if (run) { + smh->video_function(session, smh->video_user_data); + switch_mutex_lock(smh->control_mutex); + smh->video_function = NULL; + smh->video_user_data = NULL; + smh->video_function_running = 0; + switch_mutex_unlock(smh->control_mutex); + } + } status = switch_core_session_read_video_frame(session, &read_frame, SWITCH_IO_FLAG_NONE, 0); @@ -4456,7 +4483,6 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi switch_cond_next(); continue; } - if (switch_channel_test_flag(channel, CF_VIDEO_REFRESH_REQ)) { switch_core_session_refresh_video(session); @@ -4467,14 +4493,6 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi continue; } - status = switch_core_session_video_thread_callback(session, read_frame); - - if (status != SWITCH_STATUS_CONTINUE) { - if (status == SWITCH_STATUS_SUCCESS) continue; - - break; - } - if (switch_channel_test_flag(channel, CF_VIDEO_ECHO)) { switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); } @@ -4491,8 +4509,7 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi return NULL; } - -SWITCH_DECLARE(switch_status_t) switch_core_media_start_video_thread(switch_core_session_t *session) +SWITCH_DECLARE(switch_status_t) switch_core_session_start_video_thread(switch_core_session_t *session) { switch_threadattr_t *thd_attr = NULL; switch_memory_pool_t *pool = switch_core_session_get_pool(session); @@ -4527,6 +4544,61 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_start_video_thread(switch_core return SWITCH_STATUS_SUCCESS; } +SWITCH_DECLARE(void) switch_core_media_start_video_function(switch_core_session_t *session, switch_video_function_t video_function, void *user_data) +{ + switch_media_handle_t *smh; + + if (!(smh = session->media_handle)) { + return; + } + + switch_mutex_lock(smh->control_mutex); + if (!smh->video_function_running) { + smh->video_function = video_function; + smh->video_user_data = user_data; + switch_core_session_video_reset(session); + switch_core_session_start_video_thread(session); + } + switch_mutex_unlock(smh->control_mutex); +} + +SWITCH_DECLARE(int) switch_core_media_check_video_function(switch_core_session_t *session) +{ + switch_media_handle_t *smh; + int r; + + if (!(smh = session->media_handle)) { + return 0; + } + + switch_mutex_lock(smh->control_mutex); + r = (smh->video_function_running > 0); + switch_mutex_unlock(smh->control_mutex); + + return r; +} + +SWITCH_DECLARE(void) switch_core_media_end_video_function(switch_core_session_t *session) +{ + switch_media_handle_t *smh; + + if (!(smh = session->media_handle)) { + return; + } + + switch_mutex_lock(smh->control_mutex); + if (smh->video_function_running > 0) { + smh->video_function_running = -1; + } + switch_mutex_unlock(smh->control_mutex); + + while(smh->video_function_running != 0) { + switch_yield(10000); + } +} + + +>>>>>>> another refactoring pass, temp code still in place, WORK IN PROGRESS //? #define RA_PTR_LEN 512 SWITCH_DECLARE(switch_status_t) switch_core_media_proxy_remote_addr(switch_core_session_t *session, const char *sdp_str) @@ -4658,8 +4730,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_proxy_remote_addr(switch_core_ !switch_channel_test_flag(session->channel, CF_WEBRTC)) { /* Reactivate the NAT buster flag. */ switch_rtp_set_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_AUTOADJ); - switch_core_media_start_video_thread(session); - + switch_core_session_start_video_thread(session); } if (switch_media_handle_test_media_flag(smh, SCMF_AUTOFIX_TIMING)) { v_engine->check_frames = 0; @@ -5631,8 +5702,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi v_engine->local_sdp_ip, v_engine->local_sdp_port, v_engine->cur_payload_map->remote_sdp_ip, v_engine->cur_payload_map->remote_sdp_port, v_engine->cur_payload_map->agreed_pt); - - switch_core_media_start_video_thread(session); + switch_core_session_start_video_thread(session); switch_rtp_set_default_payload(v_engine->rtp_session, v_engine->cur_payload_map->agreed_pt); } } @@ -5665,7 +5735,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi !((val = switch_channel_get_variable(session->channel, "disable_rtp_auto_adjust")) && switch_true(val))) { /* Reactivate the NAT buster flag. */ switch_rtp_set_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_AUTOADJ); - switch_core_media_start_video_thread(session); + switch_core_session_start_video_thread(session); } } @@ -5764,8 +5834,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi } switch_rtp_set_payload_map(v_engine->rtp_session, &v_engine->payload_map); - - switch_core_media_start_video_thread(session); + switch_core_session_start_video_thread(session); switch_channel_set_flag(session->channel, CF_VIDEO); if ((ssrc = switch_channel_get_variable(session->channel, "rtp_use_video_ssrc"))) { @@ -7911,7 +7980,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se case SWITCH_MESSAGE_INDICATE_VIDEO_REFRESH_REQ: { if (v_engine->rtp_session) { - switch_rtp_video_refresh(v_engine->rtp_session); + if (switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_FIR)) { + switch_rtp_video_refresh(v_engine->rtp_session); + } else if (switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_PLI)) { + switch_rtp_video_loss(v_engine->rtp_session); + } } } @@ -9454,77 +9527,34 @@ SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_se session->image_write_callback_user_data = user_data; } -SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, - switch_image_t *img, switch_size_t size, uint32_t *flag) +static switch_status_t raw_write_video(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, int stream_id) { - uint32_t encoded_data_len = size, lflag = 0, *flagp = flag; - switch_codec_t *codec = switch_core_session_get_video_write_codec(session); - switch_timer_t *timer; - switch_media_handle_t *smh; - //switch_rtp_engine_t *v_engine; + switch_io_event_hook_video_write_frame_t *ptr; + switch_status_t status = SWITCH_STATUS_FALSE; - switch_assert(session); - - if (!(smh = session->media_handle)) { - return SWITCH_STATUS_FALSE; + if (session->endpoint_interface->io_routines->write_video_frame) { + if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) { + for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) { + if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) { + break; + } + } + } } - //v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; - - - if (!flag) { - flagp = &lflag; - } - - if (!(timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO))) { - - if (!smh->video_timer.timer_interface) { - switch_core_timer_init(&smh->video_timer, "soft", 1, 90, switch_core_session_get_pool(session)); - } - - timer = &smh->video_timer; - } - - - do { - encoded_data_len = size; - switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp); - - if (*flagp & SFF_PICTURE_RESET) { - smh->video_init = 0; - smh->video_last_key_time = 0; - *flagp &= ~SFF_PICTURE_RESET; - } - - frame->datalen = encoded_data_len; - frame->packetlen = frame->datalen + 12; - frame->m = (*flagp & SFF_MARKER) ? 1 : 0; - frame->timestamp = timer->samplecount; - - switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME); - - switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); - - if (session->image_write_callback) { - session->image_write_callback(session, frame, img, session->image_write_callback_user_data); - } - - img = NULL; - - } while(encoded_data_len); - - return SWITCH_STATUS_SUCCESS; + return status; } SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, int stream_id) { - switch_io_event_hook_video_write_frame_t *ptr; switch_status_t status = SWITCH_STATUS_FALSE; - switch_media_handle_t *smh; - switch_time_t now = switch_micro_time_now(); + switch_codec_t *codec = switch_core_session_get_video_write_codec(session); + switch_timer_t *timer; + switch_media_handle_t *smh; + switch_image_t *img = frame->img; switch_assert(session); @@ -9555,15 +9585,43 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor smh->video_last_key_time = now; } - if (session->endpoint_interface->io_routines->write_video_frame) { - if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) { - for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) { - if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) { - break; - } - } - } + if (!img) { + return raw_write_video(session, frame, flags, stream_id); } + + if (!(timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO))) { + + if (!smh->video_timer.timer_interface) { + switch_core_timer_init(&smh->video_timer, "soft", 1, 90, switch_core_session_get_pool(session)); + } + + timer = &smh->video_timer; + } + + frame->timestamp = timer->samplecount; + frame->flags &= ~SFF_SAME_IMAGE; + + do { + frame->datalen = SWITCH_DEFAULT_VIDEO_SIZE; + switch_core_codec_encode_video(codec, frame); + + if (frame->flags & SFF_PICTURE_RESET) { + smh->video_init = 0; + smh->video_last_key_time = 0; + frame->flags &= ~SFF_PICTURE_RESET; + } + + switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME); + + status = raw_write_video(session, frame, flags, stream_id); + + if (status == SWITCH_STATUS_SUCCESS && session->image_write_callback) { + session->image_write_callback(session, frame, img, session->image_write_callback_user_data); + } + + } while(frame->datalen); + + return status; } @@ -9572,10 +9630,15 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core { switch_status_t status = SWITCH_STATUS_FALSE; switch_io_event_hook_video_read_frame_t *ptr; + uint32_t loops = 0; switch_assert(session != NULL); - if (switch_channel_down(session->channel)) { + top: + + loops++; + + if (switch_channel_down_nosig(session->channel)) { return SWITCH_STATUS_FALSE; } @@ -9657,6 +9720,20 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core } } + if (switch_channel_test_flag(session->channel, CF_VIDEO_DECODED_READ)) { + switch_status_t decode_status = switch_core_codec_decode_video((*frame)->codec, *frame); + + if (switch_test_flag((*frame), SFF_WAIT_KEY_FRAME)) { + switch_core_session_refresh_video(session); + switch_clear_flag((*frame), SFF_WAIT_KEY_FRAME); + } + + if (decode_status == SWITCH_STATUS_MORE_DATA) { + printf("mo data\n"); + goto top; + } + } + done: return status; diff --git a/src/switch_core_session.c b/src/switch_core_session.c index 3381852565..44aff99849 100644 --- a/src/switch_core_session.c +++ b/src/switch_core_session.c @@ -2676,6 +2676,7 @@ SWITCH_DECLARE(void) switch_core_session_video_reset(switch_core_session_t *sess { switch_channel_set_flag(session->channel, CF_VIDEO_ECHO); switch_channel_clear_flag(session->channel, CF_VIDEO_PASSIVE); + switch_channel_clear_flag(session->channel, CF_VIDEO_DECODED_READ); switch_core_session_refresh_video(session); session->image_write_callback = NULL; session->image_write_callback_user_data = NULL; @@ -3065,31 +3066,6 @@ SWITCH_DECLARE(void) switch_core_session_debug_pool(switch_stream_handle_t *stre session_manager.running, session_manager.busy, session_manager.popping); } -SWITCH_DECLARE(switch_status_t) switch_core_session_set_video_thread_callback(switch_core_session_t *session, switch_core_video_thread_callback_func_t *func, void *user_data) -{ - if (!func) { - session->_video_thread_callback = NULL; - session->_video_thread_user_data = NULL; - return SWITCH_STATUS_SUCCESS; - } else if (session->_video_thread_callback) { - return SWITCH_STATUS_FALSE; - } else { - session->_video_thread_callback = func; - session->_video_thread_user_data = user_data; - return SWITCH_STATUS_SUCCESS; - } -} - -SWITCH_DECLARE(switch_status_t) switch_core_session_video_thread_callback(switch_core_session_t *session, switch_frame_t *frame) -{ - if (session->_video_thread_callback) { - return session->_video_thread_callback(session, frame, session->_video_thread_user_data); - } - - return SWITCH_STATUS_CONTINUE; -} - - /* For Emacs: * Local Variables: * mode:c diff --git a/src/switch_ivr_bridge.c b/src/switch_ivr_bridge.c index 360223c30f..5f68032120 100644 --- a/src/switch_ivr_bridge.c +++ b/src/switch_ivr_bridge.c @@ -46,38 +46,47 @@ struct vid_helper { int up; }; -static void *SWITCH_THREAD_FUNC video_bridge_thread(switch_thread_t *thread, void *obj) +static void video_bridge_thread(switch_core_session_t *session, void *obj) { struct vid_helper *vh = obj; switch_channel_t *channel = switch_core_session_get_channel(vh->session_a); switch_channel_t *b_channel = switch_core_session_get_channel(vh->session_b); switch_status_t status; switch_frame_t *read_frame = 0; - const char *source = switch_channel_get_variable(channel, "source"); - const char *b_source = switch_channel_get_variable(b_channel, "source"); vh->up = 1; switch_core_session_read_lock(vh->session_a); switch_core_session_read_lock(vh->session_b); - if (!switch_stristr("loopback", source) && !switch_stristr("loopback", b_source)) { - switch_channel_set_flag(channel, CF_VIDEO_PASSIVE); - //switch_channel_set_flag(b_channel, CF_VIDEO_PASSIVE); - } - switch_core_session_refresh_video(vh->session_a); switch_core_session_refresh_video(vh->session_b); while (switch_channel_up_nosig(channel) && switch_channel_up_nosig(b_channel) && vh->up == 1) { if (switch_channel_media_up(channel)) { + switch_codec_t *a_codec = switch_core_session_get_video_read_codec(vh->session_a); + switch_codec_t *b_codec = switch_core_session_get_video_write_codec(vh->session_b); + + if (!b_codec || !a_codec || a_codec->implementation->impl_id == b_codec->implementation->impl_id) { + //switch_channel_clear_flag(channel, CF_VIDEO_DECODED_READ); + } else { + //switch_channel_set_flag(channel, CF_VIDEO_DECODED_READ); + } + + switch_channel_set_flag(channel, CF_VIDEO_DECODED_READ); + status = switch_core_session_read_video_frame(vh->session_a, &read_frame, SWITCH_IO_FLAG_NONE, 0); if (!SWITCH_READ_ACCEPTABLE(status)) { switch_cond_next(); continue; } + + if (!switch_test_flag(read_frame, SFF_CNG)) { + switch_core_session_write_video_frame(vh->session_a, read_frame, SWITCH_IO_FLAG_NONE, 0); + continue; + } } if (switch_test_flag(read_frame, SFF_CNG)) { @@ -93,9 +102,6 @@ static void *SWITCH_THREAD_FUNC video_bridge_thread(switch_thread_t *thread, voi } - switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE); - //switch_channel_clear_flag(b_channel, CF_VIDEO_PASSIVE); - switch_core_session_kill_channel(vh->session_b, SWITCH_SIG_BREAK); switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(vh->session_a), SWITCH_LOG_DEBUG, "%s video thread ended.\n", switch_channel_get_name(channel)); @@ -106,18 +112,12 @@ static void *SWITCH_THREAD_FUNC video_bridge_thread(switch_thread_t *thread, voi switch_core_session_rwunlock(vh->session_b); vh->up = 0; - return NULL; + return; } -static switch_thread_t *launch_video(struct vid_helper *vh) +static void launch_video(struct vid_helper *vh) { - switch_thread_t *thread; - switch_threadattr_t *thd_attr = NULL; - - switch_threadattr_create(&thd_attr, switch_core_session_get_pool(vh->session_a)); - switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); - switch_thread_create(&thread, thd_attr, video_bridge_thread, vh, switch_core_session_get_pool(vh->session_a)); - return thread; + switch_core_media_start_video_function(vh->session_a, video_bridge_thread, vh); } #endif @@ -229,7 +229,6 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) const char *exec_data = NULL; #ifdef SWITCH_VIDEO_IN_THREADS - switch_thread_t *vid_thread = NULL; struct vid_helper vh = { 0 }; uint32_t vid_launch = 0; #endif @@ -246,6 +245,9 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) chan_a = switch_core_session_get_channel(session_a); chan_b = switch_core_session_get_channel(session_b); + + switch_channel_clear_flag(chan_a, CF_VIDEO_ECHO); + switch_channel_clear_flag(chan_b, CF_VIDEO_ECHO); if ((exec_app = switch_channel_get_variable(chan_a, "bridge_pre_execute_app"))) { @@ -410,7 +412,10 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) vid_launch++; vh.session_a = session_a; vh.session_b = session_b; - vid_thread = launch_video(&vh); + //DFF TEMP + if (switch_channel_test_flag(chan_a, CF_BRIDGE_ORIGINATOR)) { + launch_video(&vh); + } } #endif @@ -590,7 +595,7 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) end_of_bridge_loop: #ifdef SWITCH_VIDEO_IN_THREADS - if (vid_thread) { + if (vh.up > 0) { vh.up = -1; switch_channel_set_flag(chan_a, CF_NOT_READY); //switch_channel_set_flag(chan_b, CF_NOT_READY); @@ -634,9 +639,7 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) end: #ifdef SWITCH_VIDEO_IN_THREADS - if (vid_thread) { - switch_status_t st; - + if (switch_core_media_check_video_function(session_a)) { if (vh.up == 1) { vh.up = -1; } @@ -647,7 +650,7 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) switch_core_session_kill_channel(session_b, SWITCH_SIG_BREAK); switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session_a), SWITCH_LOG_DEBUG, "Ending video thread.\n"); - switch_thread_join(&st, vid_thread); + switch_core_media_end_video_function(session_a); switch_channel_clear_flag(chan_a, CF_NOT_READY); switch_channel_clear_flag(chan_b, CF_NOT_READY); } @@ -689,6 +692,9 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) switch_core_session_kill_channel(session_b, SWITCH_SIG_BREAK); data->done = 1; + switch_core_session_video_reset(session_a); + switch_core_session_video_reset(session_b); + switch_core_session_rwunlock(session_b); return NULL; } diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 19cd856428..57cc417cf4 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -1444,7 +1444,7 @@ static uint8_t get_next_write_ts(switch_rtp_t *rtp_session, uint32_t timestamp) static void send_fir(switch_rtp_t *rtp_session) { - if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->ice.ice_user) { + if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { return; } @@ -1557,7 +1557,7 @@ static void send_fir(switch_rtp_t *rtp_session) static void send_pli(switch_rtp_t *rtp_session) { - if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->ice.ice_user) { + if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { return; } @@ -2035,7 +2035,6 @@ static int check_rtcp_and_ice(switch_rtp_t *rtp_session) } if (rtp_session->pli_countdown) { - if (rtp_session->pli_countdown == PLI_COUNTDOWN || (rtp_session->pli_countdown == PLI_COUNTDOWN / 2) || rtp_session->pli_countdown == 1) { send_pli(rtp_session); }