diff --git a/src/include/private/switch_core_pvt.h b/src/include/private/switch_core_pvt.h index 4b8be156b8..9acc2a2741 100644 --- a/src/include/private/switch_core_pvt.h +++ b/src/include/private/switch_core_pvt.h @@ -134,6 +134,8 @@ struct switch_core_session { switch_mutex_t *resample_mutex; switch_mutex_t *codec_read_mutex; switch_mutex_t *codec_write_mutex; + switch_mutex_t *video_codec_read_mutex; + switch_mutex_t *video_codec_write_mutex; switch_thread_cond_t *cond; switch_mutex_t *frame_read_mutex; @@ -166,6 +168,16 @@ struct switch_core_session { switch_frame_t enc_read_frame; uint8_t raw_read_buf[SWITCH_RECOMMENDED_BUFFER_SIZE]; uint8_t enc_read_buf[SWITCH_RECOMMENDED_BUFFER_SIZE]; + + /* video frame.data being trated differently than audio, allocate a dynamic data buffer if necessary*/ + switch_buffer_t *video_raw_write_buffer; + switch_frame_t video_raw_write_frame; + // switch_frame_t video_enc_write_frame; + + switch_buffer_t *video_raw_read_buffer; + switch_frame_t video_raw_read_frame; + // switch_frame_t video_enc_read_frame; + switch_codec_t bug_codec; uint32_t read_frame_count; uint32_t track_duration; diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index e70e663dd9..84c4b135d6 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -611,6 +611,13 @@ struct switch_codec_fmtp { }; +struct switch_picture { + uint32_t width; /* the picture width */ + uint32_t height; /* the picture height */ + uint8_t *planes[4]; /* pointer to the top left pixel for each plane */ + uint32_t stride[4]; /* stride between rows for each plane */ +}; + /*! an abstract handle to a codec module */ struct switch_codec { /*! the codec interface table this handle uses */ @@ -632,6 +639,10 @@ struct switch_codec { struct switch_codec *next; switch_core_session_t *session; switch_frame_t *cur_frame; + /*! raw picture for encode */ + switch_picture_t enc_picture; + /*! decoded picture */ + switch_picture_t dec_picture; }; /*! \brief A table of settings and callbacks that define a paticular implementation of a codec */ diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 2b19038907..50c841ae44 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -541,6 +541,7 @@ SWITCH_DECLARE_DATA extern switch_filenames SWITCH_GLOBAL_filenames; #define SWITCH_MAX_SAMPLE_LEN 48 #define SWITCH_BYTES_PER_SAMPLE 2 /* slin is 2 bytes per sample */ #define SWITCH_RECOMMENDED_BUFFER_SIZE 8192 +#define SWITCH_RECOMMENDED_VIDEO_BUFFER_SIZE 4096 * 1024 /* Fixme: Just Make sure it's big enough for now */ #define SWITCH_MAX_CODECS 50 #define SWITCH_MAX_STATE_HANDLERS 30 #define SWITCH_CORE_QUEUE_LEN 100000 @@ -1472,10 +1473,12 @@ typedef enum { 
 SFF_CNG        = (1 <<  0) - Frame represents comfort noise
 SFF_RAW_RTP    = (1 <<  1) - Frame has raw rtp accessible
-SFF_RTP_HEADER = (1 << 2)  - Get the rtp header from the frame header
-SFF_PLC        = (1 << 3)  - Frame has generated PLC data
-SFF_RFC2833    = (1 << 4)  - Frame has rfc2833 dtmf data
-SFF_DYNAMIC    = (1 << 5)  - Frame is dynamic and should be freed
+SFF_RTP_HEADER = (1 <<  2) - Get the rtp header from the frame header
+SFF_PLC        = (1 <<  3) - Frame has generated PLC data
+SFF_RFC2833    = (1 <<  4) - Frame has rfc2833 dtmf data
+SFF_DYNAMIC    = (1 <<  5) - Frame is dynamic and should be freed
+SFF_MARKER     = (1 << 11) - Frame flag has Marker set, only set by encoder
+SFF_WAIT_KEY_FRAME = (1 << 12) - Need a key from before could decode
*/ typedef enum { @@ -1490,7 +1493,9 @@ typedef enum { SFF_ZRTP = (1 << 7), SFF_UDPTL_PACKET = (1 << 8), SFF_NOT_AUDIO = (1 << 9), - SFF_RTCP = (1 << 10) + SFF_RTCP = (1 << 10), + SFF_MARKER = (1 << 11), + SFF_WAIT_KEY_FRAME = (1 << 12) } switch_frame_flag_enum_t; typedef uint32_t switch_frame_flag_t; @@ -2090,6 +2095,7 @@ typedef struct switch_caller_extension switch_caller_extension_t; typedef struct switch_caller_application switch_caller_application_t; typedef struct switch_state_handler_table switch_state_handler_table_t; typedef struct switch_timer switch_timer_t; +typedef struct switch_picture switch_picture_t; typedef struct switch_codec switch_codec_t; typedef struct switch_core_thread_session switch_core_thread_session_t; typedef struct switch_codec_implementation switch_codec_implementation_t; diff --git a/src/switch_core_io.c b/src/switch_core_io.c index 35d348e751..7bda3d05d3 100644 --- a/src/switch_core_io.c +++ b/src/switch_core_io.c @@ -104,8 +104,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core goto done; } - switch_assert(*frame != NULL); - if (switch_test_flag(*frame, SFF_CNG)) { status = SWITCH_STATUS_SUCCESS; goto done; diff --git a/src/switch_core_media.c b/src/switch_core_media.c index f4f970ed4a..de57966395 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -4346,9 +4346,6 @@ SWITCH_DECLARE(int) switch_core_media_toggle_hold(switch_core_session_t *session #define BUF_SIZE (352 * 288 * 3 / 2 * 4) // big enough for 4CIF, looks like C doesn't like huge array #define FPS 15 -#define WIDTH 352 -#define HEIGHT 288 -#define SIZE WIDTH * HEIGHT static switch_status_t video_bridge_callback(switch_core_session_t *session, switch_bool_t video_transcoding, uint32_t *ts) { @@ -4397,7 +4394,7 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi uint32_t flag = 0; uint32_t encoded_data_len = 1500; uint32_t encoded_rate = 0; - switch_frame_t write_frame; + switch_frame_t write_frame = { 0 }; #if 0 switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d/%s != %d/%s need transcoding!!!\n", @@ -4413,13 +4410,13 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi if (decoded_data_len < 3) return SWITCH_STATUS_SUCCESS; - decoded_data_len = 152064; - - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "decoded_data_len: %d %s\n", decoded_data_len, codec->implementation->iananame); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s decoded_data_len: %d size: %dx%d\n", codec->implementation->iananame, decoded_data_len, codec->dec_picture.width, codec->dec_picture.height); write_frame.packet = rtp_buff; write_frame.data = rtp_buff + 12; + other_codec->enc_picture.width = codec->dec_picture.width; + other_codec->enc_picture.height = codec->dec_picture.height; encoded_data_len = 1500; switch_core_codec_encode(other_codec, NULL, raw_buff, decoded_data_len, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag); @@ -4428,7 +4425,7 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi write_frame.datalen = encoded_data_len; write_frame.packetlen = write_frame.datalen + 12; - write_frame.m = flag; + write_frame.m = flag & SFF_MARKER ? 1 : 0; write_frame.timestamp = *ts; if (write_frame.m) *ts += 90000 / FPS; diff --git a/src/switch_core_session.c b/src/switch_core_session.c index 4e02b735c7..8d77bd4433 100644 --- a/src/switch_core_session.c +++ b/src/switch_core_session.c @@ -1317,6 +1317,17 @@ SWITCH_DECLARE(void) switch_core_session_reset(switch_core_session_t *session, s switch_buffer_destroy(&session->raw_read_buffer); switch_mutex_unlock(session->codec_read_mutex); + switch_mutex_lock(session->video_codec_write_mutex); + switch_buffer_destroy(&session->video_raw_write_buffer); + switch_mutex_unlock(session->video_codec_write_mutex); + + switch_mutex_lock(session->video_codec_read_mutex); + switch_buffer_destroy(&session->video_raw_read_buffer); + switch_mutex_unlock(session->video_codec_read_mutex); + + //video_raw_read_frame.data is dynamically allocated if necessary, so wipe this also + switch_safe_free(session->video_raw_read_frame.data); + if (flush_dtmf) { while ((has = switch_channel_has_dtmf(channel))) { switch_channel_flush_dtmf(channel); @@ -2419,6 +2430,8 @@ SWITCH_DECLARE(switch_core_session_t *) switch_core_session_request_uuid(switch_ switch_mutex_init(&session->resample_mutex, SWITCH_MUTEX_NESTED, session->pool); switch_mutex_init(&session->codec_read_mutex, SWITCH_MUTEX_NESTED, session->pool); switch_mutex_init(&session->codec_write_mutex, SWITCH_MUTEX_NESTED, session->pool); + switch_mutex_init(&session->video_codec_read_mutex, SWITCH_MUTEX_NESTED, session->pool); + switch_mutex_init(&session->video_codec_write_mutex, SWITCH_MUTEX_NESTED, session->pool); switch_mutex_init(&session->frame_read_mutex, SWITCH_MUTEX_NESTED, session->pool); switch_thread_rwlock_create(&session->bug_rwlock, session->pool); switch_thread_cond_create(&session->cond, session->pool);