FS-7500: Work in progress. Added codec config params that can be set from session and made vpx codec re-init on size change. Also add periodic key frame timer

src/include/switch_core_media.h

@@ -157,6 +157,9 @@ typedef struct switch_core_media_params_s {
 
 	switch_bool_t external_video_source;
 
+	uint32_t video_key_freq;
+	uint32_t video_key_first;
+
 } switch_core_media_params_t;
 
 static inline const char *switch_media_type2str(switch_media_type_t type)
@@ -305,6 +308,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
 																switch_codec_control_type_t *rtype,
 																void **ret_data);
 
+
+#define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \
+																				  SCC_VIDEO_REFRESH, SCCT_NONE, NULL, NULL, NULL) \
+
+
 SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype);
 
 SWITCH_END_EXTERN_C

src/include/switch_module_interfaces.h

@@ -591,12 +591,21 @@ struct switch_directory_handle {
 	void *private_info;
 };
 
-/* nobody has more setting than speex so we will let them set the standard */
+struct switch_audio_codec_settings {
-/*! \brief Various codec settings (currently only relevant to speex) */
-struct switch_codec_settings {
 	int unused;
 };
 
+struct switch_video_codec_settings {
+	uint32_t bandwidth;
+	int32_t width;
+	int32_t height;
+};
+
+union switch_codec_settings {
+	struct switch_audio_codec_settings audio;
+	struct switch_video_codec_settings video;
+};
+
 /*! an abstract handle of a fmtp parsed by codec */
 struct switch_codec_fmtp {
 	/*! actual samples transferred per second for those who are not moron g722 RFC writers */

src/include/switch_rtp.h

@@ -517,6 +517,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_del_dtls(switch_rtp_t *rtp_session, d
 
 SWITCH_DECLARE(int) switch_rtp_has_dtls(void);
 SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session);
+SWITCH_DECLARE(void) switch_rtp_video_loss(switch_rtp_t *rtp_session);
 
 /*!
   \}

src/include/switch_types.h

@@ -1496,7 +1496,8 @@ typedef enum {
 	SFF_RTCP = (1 << 10),
 	SFF_MARKER = (1 << 11),
 	SFF_WAIT_KEY_FRAME = (1 << 12),
-	SFF_RAW_RTP_PARSE_FRAME = (1 << 13)
+	SFF_RAW_RTP_PARSE_FRAME = (1 << 13),
+	SFF_PICTURE_RESET = (1 << 14)
 } switch_frame_flag_enum_t;
 typedef uint32_t switch_frame_flag_t;
 
@@ -2101,7 +2102,7 @@ typedef struct switch_codec switch_codec_t;
 typedef struct switch_core_thread_session switch_core_thread_session_t;
 typedef struct switch_codec_implementation switch_codec_implementation_t;
 typedef struct switch_buffer switch_buffer_t;
-typedef struct switch_codec_settings switch_codec_settings_t;
+typedef union  switch_codec_settings switch_codec_settings_t;
 typedef struct switch_codec_fmtp switch_codec_fmtp_t;
 typedef struct switch_odbc_handle switch_odbc_handle_t;
 typedef struct switch_pgsql_handle switch_pgsql_handle_t;

src/mod/codecs/mod_vpx/mod_vpx.c

@@ -44,18 +44,21 @@
 SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load);
 SWITCH_MODULE_DEFINITION(mod_vpx, mod_vpx_load, NULL, NULL);
 
+
+#define encoder_interface (vpx_codec_vp8_cx())
+#define decoder_interface (vpx_codec_vp8_dx())
+
 struct vpx_context {
 	switch_codec_t *codec;
 	unsigned int flags;
-
+	switch_codec_settings_t codec_settings;
+	unsigned int bandwidth;
 	vpx_codec_enc_cfg_t	config;
 
 	vpx_codec_ctx_t	encoder;
+	uint8_t encoder_init;
 	vpx_image_t *pic;
 	switch_bool_t force_key_frame;
-	int width;
-	int height;
-	int bitrate;
 	int fps;
 	int format;
 	int intra_period;
@@ -65,9 +68,8 @@ struct vpx_context {
 	const vpx_codec_cx_pkt_t *pkt;
 	int pkt_pos;
 	vpx_codec_iter_t iter;
-	switch_time_t last_ts;
-
 	vpx_codec_ctx_t	decoder;
+	uint8_t decoder_init;
 	switch_buffer_t *vpx_packet_buffer;
 	int got_key_frame;
 	switch_size_t last_received_timestamp;
@@ -76,48 +78,17 @@ struct vpx_context {
 };
 typedef struct vpx_context vpx_context_t;
 
-static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_t flags, const switch_codec_settings_t *codec_settings)
+
+static switch_status_t init_codec(switch_codec_t *codec)
 {
-	vpx_context_t *context = NULL;
+	vpx_context_t *context = (vpx_context_t *)codec->private_info;
-	int encoding, decoding;
+	vpx_codec_enc_cfg_t *config = &context->config;
-	vpx_codec_ctx_t	*encoder = NULL;
-	vpx_codec_ctx_t	*decoder = NULL;
-	vpx_codec_enc_cfg_t *config;
-	const vpx_codec_iface_t* encoder_interface = vpx_codec_vp8_cx();
-	const vpx_codec_iface_t* decoder_interface = vpx_codec_vp8_dx();
-
-
-	encoding = (flags & SWITCH_CODEC_FLAG_ENCODE);
-	decoding = (flags & SWITCH_CODEC_FLAG_DECODE);
-
-	if (!(encoding || decoding) || ((context = switch_core_alloc(codec->memory_pool, sizeof(*context))) == 0)) {
-		return SWITCH_STATUS_FALSE;
-	}
-	memset(context, 0, sizeof(*context));
-	context->flags = flags;
-	codec->private_info = context;
-
-	if (codec->fmtp_in) {
-		codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in);
-	}
-
-	config = &context->config;
-
-	if (vpx_codec_enc_config_default(encoder_interface, config, 0) != VPX_CODEC_OK) {
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder config Error\n");
-		return SWITCH_STATUS_FALSE;
-	}
-
-	// very big defaults till we know why scaling segs it
-	context->width = 3840;
-	context->height = 2160;
-	context->bitrate = 3840000;
 
 	// settings
 	config->g_profile = 1;
-	config->g_w = context->width;
+	config->g_w = context->codec_settings.video.width;
-	config->g_h = context->height;
+	config->g_h = context->codec_settings.video.height;
-	config->rc_target_bitrate = context->bitrate;
+	config->rc_target_bitrate = context->bandwidth;
 	config->g_timebase.num = 1;
 	config->g_timebase.den = 1000;
 	config->g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS;
@@ -127,11 +98,11 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_
 	config->rc_dropframe_thresh = 0;
 	config->rc_end_usage = VPX_CBR;
 	config->g_pass = VPX_RC_ONE_PASS;
-	// config->kf_mode = VPX_KF_DISABLED;
+	config->kf_mode = VPX_KF_DISABLED;
-	config->kf_mode = VPX_KF_AUTO;
+	//config->kf_mode = VPX_KF_AUTO;
 	//config->kf_min_dist = FPS;// Intra Period 3 seconds;
-	// config->kf_max_dist = FPS * 3;
+	//config->kf_max_dist = FPS;
-	config->rc_resize_allowed = 0;
+	config->rc_resize_allowed = 1;
 	config->rc_min_quantizer = 2;
 	config->rc_max_quantizer = 56;
 	//Rate control adaptation undershoot control.
@@ -157,59 +128,74 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_
 	//	indicates that the client will buffer (at least) 5000ms worth
 	//	of encoded data. Use the target bitrate (rc_target_bitrate) to
 	//	convert to bits/bytes, if necessary.
-	config->rc_buf_sz = 1000;
+	config->rc_buf_sz = 5000;
 	//Decoder Buffer Initial Size.
 	//	This value indicates the amount of data that will be buffered
 	//	by the decoding application prior to beginning playback.
 	//	This value is expressed in units of time (milliseconds).
 	//	Use the target bitrate (rc_target_bitrate) to convert to
 	//	bits/bytes, if necessary.
-	config->rc_buf_initial_sz = 500;
+	config->rc_buf_initial_sz = 1000;
 	//Decoder Buffer Optimal Size.
 	//	This value indicates the amount of data that the encoder should
 	//	try to maintain in the decoder's buffer. This value is expressed
 	//	in units of time (milliseconds).
 	//	Use the target bitrate (rc_target_bitrate) to convert to
 	//	bits/bytes, if necessary.
-	config->rc_buf_optimal_sz = 600;
+	config->rc_buf_optimal_sz = 1000;
+
+	if (context->flags & SWITCH_CODEC_FLAG_ENCODE) {
+
+		if (context->encoder_init) {
+			vpx_codec_destroy(&context->encoder);
+			context->encoder_init = 0;
+		}
 
-	if (encoding) {
 		if (vpx_codec_enc_init(&context->encoder, encoder_interface, config, 0 & VPX_CODEC_USE_OUTPUT_PARTITION) != VPX_CODEC_OK) {
-			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", encoder->err, encoder->err_detail);
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", context->encoder.err, context->encoder.err_detail);
 			return SWITCH_STATUS_FALSE;
 		}
 
+		context->encoder_init = 1;
+
 		// The static threshold imposes a change threshold on blocks below which they will be skipped by the encoder.
-		vpx_codec_control(encoder, VP8E_SET_STATIC_THRESHOLD, 100);
+		vpx_codec_control(&context->encoder, VP8E_SET_STATIC_THRESHOLD, 100);
 		//Set cpu usage, a bit lower than normal (-6) but higher than android (-12)
-		vpx_codec_control(encoder, VP8E_SET_CPUUSED, -8);
+		vpx_codec_control(&context->encoder, VP8E_SET_CPUUSED, -6);
 		// Only one partition
-		// vpx_codec_control(encoder, VP8E_SET_TOKEN_PARTITIONS, VP8_ONE_TOKENPARTITION);
+		// vpx_codec_control(&context->encoder, VP8E_SET_TOKEN_PARTITIONS, VP8_ONE_TOKENPARTITION);
 		// Enable noise reduction
-		vpx_codec_control(encoder, VP8E_SET_NOISE_SENSITIVITY, 0);
+		vpx_codec_control(&context->encoder, VP8E_SET_NOISE_SENSITIVITY, 1);
 		//Set max data rate for Intra frames.
 		//	This value controls additional clamping on the maximum size of a keyframe.
 		//	It is expressed as a percentage of the average per-frame bitrate, with the
 		//	special (and default) value 0 meaning unlimited, or no additional clamping
 		//	beyond the codec's built-in algorithm.
 		//	For example, to allocate no more than 4.5 frames worth of bitrate to a keyframe, set this to 450.
-		vpx_codec_control(encoder, VP8E_SET_MAX_INTRA_BITRATE_PCT, 0);
+		vpx_codec_control(&context->encoder, VP8E_SET_MAX_INTRA_BITRATE_PCT, 0);
 	}
 
-	if (decoding) {
+	if (context->flags & SWITCH_CODEC_FLAG_DECODE) {
 		vp8_postproc_cfg_t ppcfg;
 
+		if (context->decoder_init) {
+			vpx_codec_destroy(&context->decoder);
+			context->decoder_init = 0;
+		}
+
 		if (vpx_codec_dec_init(&context->decoder, decoder_interface, NULL, VPX_CODEC_USE_POSTPROC) != VPX_CODEC_OK) {
-			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", encoder->err, encoder->err_detail);
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", context->encoder.err, context->encoder.err_detail);
 			return SWITCH_STATUS_FALSE;
 		}
 
+		context->decoder_init = 1;
+
 		// the types of post processing to be done, should be combination of "vp8_postproc_level"
 		ppcfg.post_proc_flag = VP8_DEMACROBLOCK | VP8_DEBLOCK;
 		// the strength of deblocking, valid range [0, 16]
 		ppcfg.deblocking_level = 3;
 		// Set deblocking settings
-		vpx_codec_control(decoder, VP8_SET_POSTPROC, &ppcfg);
+		vpx_codec_control(&context->decoder, VP8_SET_POSTPROC, &ppcfg);
 
 		switch_buffer_create_dynamic(&context->vpx_packet_buffer, 512, 512, 1024000);
 	}
@@ -217,6 +203,39 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_
 	return SWITCH_STATUS_SUCCESS;
 }
 
+static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_t flags, const switch_codec_settings_t *codec_settings)
+{
+	vpx_context_t *context = NULL;
+	int encoding, decoding;
+
+
+	encoding = (flags & SWITCH_CODEC_FLAG_ENCODE);
+	decoding = (flags & SWITCH_CODEC_FLAG_DECODE);
+
+	if (!(encoding || decoding) || ((context = switch_core_alloc(codec->memory_pool, sizeof(*context))) == 0)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	memset(context, 0, sizeof(*context));
+	context->flags = flags;
+	codec->private_info = context;
+
+	if (codec_settings) {
+		context->codec_settings = *codec_settings;
+	}
+
+	if (codec->fmtp_in) {
+		codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in);
+	}
+
+	if (vpx_codec_enc_config_default(encoder_interface, &context->config, 0) != VPX_CODEC_OK) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder config Error\n");
+		return SWITCH_STATUS_FALSE;
+	}
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
 /*	http://tools.ietf.org/html/draft-ietf-payload-vp8-10
 
 	The first octets after the RTP header are the VP8 payload descriptor, with the following structure.
@@ -332,6 +351,8 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t *
 
 	//d_w and d_h are messed up
 
+	//printf("WTF %d %d\n", img->d_w, img->d_h);
+
 	width = img->w;
 	height = img->h;
 
@@ -339,28 +360,31 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t *
 	//switch_assert(height > 0 && (height % 4 == 0));
 
 	if (context->config.g_w != width || context->config.g_h != height) {
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX reset encoder picture from %dx%d to %dx%d\n", context->config.g_w, context->config.g_h, width, height);
+		context->codec_settings.video.width = width;
-		context->config.g_w = width;
+		context->codec_settings.video.height = height;
-		context->config.g_h = height;
+		if (context->codec_settings.video.bandwidth) {
-		if (vpx_codec_enc_config_set(&context->encoder, &context->config) != VPX_CODEC_OK) {
+			context->bandwidth = context->codec_settings.video.bandwidth;
-			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX reset config error!");
+		} else {
-		}
+			context->bandwidth = width * height * 8;
 		}
 
-	if (context->last_ts == 0) context->last_ts = switch_micro_time_now();
+		if (context->bandwidth > 1250000) {
+			context->bandwidth = 1250000;
+		}
 
-	if ((switch_micro_time_now() - context->last_ts) > 2 * 1000000) {
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(codec->session), SWITCH_LOG_NOTICE, 
-		// the config params doesn't seems work for generate regular key frames,
+						  "VPX reset encoder picture from %dx%d to %dx%d %u BW\n", 
-		// so we do some trick here to force a key frame every 2 sec
+						  context->config.g_w, context->config.g_h, width, height, context->bandwidth);
-		// vpx_flags = VPX_EFLAG_FORCE_KF;
+
-		context->last_ts = switch_micro_time_now();
+		init_codec(codec);
+		*flag |= SFF_PICTURE_RESET;
+		context->need_key_frame = 1;
 	}
 
 	if (context->need_key_frame > 0) {
 		// force generate a key frame
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX KEYFRAME REQ\n");
 		vpx_flags |= VPX_EFLAG_FORCE_KF;
-		context->last_ts = switch_micro_time_now();
 		context->need_key_frame--;
 	}
 
@@ -539,7 +563,7 @@ static switch_status_t switch_vpx_destroy(switch_codec_t *codec)
 
 	if (context) {
 		if ((codec->flags & SWITCH_CODEC_FLAG_ENCODE)) {
-			vpx_codec_destroy(&context->encoder); // TODO fix crash
+			vpx_codec_destroy(&context->encoder);
 		}
 
 		if ((codec->flags & SWITCH_CODEC_FLAG_DECODE)) {

src/mod/formats/mod_vlc/mod_vlc.c

@@ -102,6 +102,7 @@ struct vlc_video_context {
 	switch_mutex_t *video_mutex;
 
 	switch_core_session_t *session;
+	switch_channel_t *channel;
 	switch_frame_t *aud_frame;
 	switch_frame_t *vid_frame;
 	uint8_t video_packet[1500 + 12];
@@ -230,20 +231,11 @@ static void *vlc_video_lock_callback(void *data, void **p_pixels)
 	return NULL; /* picture identifier, not needed here */
 }
 
-/* dummy callback so it should be good when no video on channel */
-static void vlc_video_unlock_dummy_callback(void *data, void *id, void *const *p_pixels)
-{
-	vlc_video_context_t *context = (vlc_video_context_t *)data;
-	assert(id == NULL); /* picture identifier, not needed here */
-	switch_mutex_unlock(context->video_mutex);
-}
-
 static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixels)
 {
 	vlc_video_context_t *context = (vlc_video_context_t *) data;
-	switch_frame_t *frame = context->vid_frame;
 
-	switch_assert(id == NULL); /* picture identifier, not needed here */
+	if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return;
 
 	if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0);
 
@@ -251,14 +243,11 @@ static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixel
 
 	yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height);
 
-	switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL);
-
 	switch_mutex_unlock(context->video_mutex);
 }
 
-static void do_buffer_frame(vlc_video_context_t *context)
+static void do_buffer_frame(vlc_video_context_t *context, switch_frame_t *frame)
 {
-	switch_frame_t *frame = context->vid_frame;
 	uint32_t size = sizeof(*frame) + frame->packetlen;
 
 	switch_mutex_lock(context->video_mutex);
@@ -277,32 +266,35 @@ static void do_buffer_frame(vlc_video_context_t *context)
 static void vlc_video_channel_unlock_callback(void *data, void *id, void *const *p_pixels)
 {
 	vlc_video_context_t *context = (vlc_video_context_t *)data;
-	uint32_t flag = 0;
+
-	switch_frame_t *frame = context->vid_frame;
 
 	switch_assert(id == NULL); /* picture identifier, not needed here */
 
+	if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return;
+
 	if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0);
 	switch_assert(context->img);
 
 	yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height);
 
+	switch_mutex_unlock(context->video_mutex);
+}
+
+static void vlc_video_display_callback(void *data, void *id)
+{
+	vlc_video_context_t *context = (vlc_video_context_t *) data;
+	int32_t flag = 0;
+
+	/* VLC wants to display the video */
+
+	if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return;
 
 	if (context->video_refresh_req > 0) {
 		flag |= SFF_WAIT_KEY_FRAME;
 		context->video_refresh_req--;
 	}
 
-	switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, &flag);
+	switch_core_session_write_video_image(context->session, context->vid_frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL);
-
-	switch_mutex_unlock(context->video_mutex);
-}
-
-static void vlc_video_display_callback(void *data, void *id)
-{
-	/* VLC wants to display the video */
-	(void) data;
-	assert(id == NULL);
 }
 
 unsigned video_format_setup_callback(void **opaque, char *chroma, unsigned *width, unsigned *height, unsigned *pitches, unsigned *lines)
@@ -723,7 +715,9 @@ SWITCH_STANDARD_APP(play_video_function)
 
 	audio_datalen = read_impl.decoded_bytes_per_packet; //codec.implementation->actual_samples_per_second / 1000 * (read_impl.microseconds_per_packet / 1000);
 
+
 	context->session = session;
+	context->channel = channel;
 	context->pool = pool;
 	context->aud_frame = &audio_frame;
 	context->vid_frame = &video_frame;
@@ -770,14 +764,8 @@ SWITCH_STANDARD_APP(play_video_function)
 	libvlc_audio_set_format(context->mp, "S16N", read_impl.actual_samples_per_second, read_impl.number_of_channels);
 	libvlc_audio_set_callbacks(context->mp, vlc_play_audio_callback, NULL,NULL,NULL,NULL, (void *) context);
 
-	if (switch_channel_test_flag(channel, CF_VIDEO)) {
-		// libvlc_video_set_format(context->mp, "YUYV", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH * 2);
 	libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
 	libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_callback, vlc_video_display_callback, context);
-	} else {
-		libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
-		libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_dummy_callback, vlc_video_display_callback, context);
-	}
 
 	// start play
 	if (-1 == libvlc_media_player_play(context->mp)) {
@@ -928,6 +916,15 @@ switch_io_routines_t vlc_io_routines = {
 	/*state_run*/ NULL
 };
 
+static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data)
+{
+	vlc_video_context_t *context = (vlc_video_context_t *) user_data;
+
+	do_buffer_frame(context, frame);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
 static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char *path)
 {
 	switch_channel_t *channel = switch_core_session_get_channel(session);
@@ -952,11 +949,12 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char
 	memset(context, 0, sizeof(vlc_file_context_t));
 	tech_pvt->context = context;
 
+
 	switch_buffer_create_dynamic(&(context->audio_buffer), VLC_BUFFER_SIZE, VLC_BUFFER_SIZE * 8, 0);
 
-	if (switch_channel_test_flag(channel, CF_VIDEO)) {
 	switch_buffer_create_dynamic(&(context->video_buffer), VLC_BUFFER_SIZE * 2, VLC_BUFFER_SIZE * 16, 0);
-	}
+
+	switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context);
 
 	if (switch_core_timer_init(&tech_pvt->timer, "soft", 20,
 							   8000 / (1000 / 20), pool) != SWITCH_STATUS_SUCCESS) {
@@ -969,6 +967,9 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char
 	context->pool = pool;
 	context->aud_frame = &tech_pvt->read_frame;
 	context->vid_frame = &tech_pvt->read_video_frame;
+	context->vid_frame->packet = context->video_packet;
+	context->vid_frame->data = context->video_packet + 12;
+
 	context->playing = 0;
 	// context->err = 0;
 
@@ -1010,14 +1011,10 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char
 	libvlc_audio_set_format(context->mp, "S16N", 8000, 1);
 	libvlc_audio_set_callbacks(context->mp, vlc_play_audio_callback, NULL,NULL,NULL,NULL, (void *) context);
 
-	if (switch_channel_test_flag(channel, CF_VIDEO)) {
+	
-		// libvlc_video_set_format(context->mp, "YUYV", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH * 2);
 	libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
 	libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_channel_unlock_callback, vlc_video_display_callback, context);
-	} else {
+
-		libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
-		libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_dummy_callback, vlc_video_display_callback, context);
-	}
 
 	return SWITCH_STATUS_SUCCESS;
 
@@ -1026,25 +1023,13 @@ fail:
 	return status;
 }
 
-static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data)
-{
-	vlc_video_context_t *context = (vlc_video_context_t *) user_data;
-
-	do_buffer_frame(context);
-
-	return SWITCH_STATUS_SUCCESS;
-}
-
-
 static switch_status_t channel_on_init(switch_core_session_t *session)
 {
 	switch_channel_t *channel = switch_core_session_get_channel(session);
-	vlc_private_t *tech_pvt = switch_core_session_get_private(session);
+	//vlc_private_t *tech_pvt = switch_core_session_get_private(session);
 
 	switch_channel_set_state(channel, CS_CONSUME_MEDIA);
 
-	switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context);
-
 	return SWITCH_STATUS_SUCCESS;
 }
 
@@ -1096,6 +1081,10 @@ static switch_status_t channel_on_destroy(switch_core_session_t *session)
 		switch_buffer_destroy(&tech_pvt->context->audio_buffer);
 	}
 
+	if (tech_pvt->context->video_buffer) {
+		switch_buffer_destroy(&tech_pvt->context->video_buffer);
+	}
+
 	if (tech_pvt->timer.interval) {
 		switch_core_timer_destroy(&tech_pvt->timer);
 	}

src/switch_core_codec.c

@@ -646,6 +646,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_init_with_bitrate(switch_codec
 
 	memset(codec, 0, sizeof(*codec));
 
+	if (pool) {
+		codec->session = switch_core_memory_pool_get_data(pool, "__session");
+	}
+
 	if ((codec_interface = switch_loadable_module_get_codec_interface(codec_name)) == 0) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid codec %s!\n", codec_name);
 		return SWITCH_STATUS_GENERR;

src/switch_core_io.c

@@ -36,174 +36,6 @@
 #include <switch.h>
 #include "private/switch_core_pvt.h"
 
-SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data)
-{
-	session->image_write_callback = callback;
-	session->image_write_callback_user_data = user_data;
-}
-
-SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, 
-																	  switch_image_t *img, switch_size_t size, uint32_t *flag)
-{
-	uint32_t encoded_data_len = size, lflag = 0, *flagp = flag;
-	switch_codec_t *codec = switch_core_session_get_video_write_codec(session);
-	switch_timer_t *timer;
-
-	switch_assert(session);
-
-	if (!flag) {
-		flagp = &lflag;
-	}
-
-	timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO);
-	switch_assert(timer);
-
-	switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp);
-
-	while(encoded_data_len) {
-
-		frame->datalen = encoded_data_len;
-		frame->packetlen = frame->datalen + 12;
-		frame->m = (*flagp & SFF_MARKER) ? 1 : 0;
-		frame->timestamp = timer->samplecount;
-
-		switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
-		
-		switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
-
-		if (session->image_write_callback) {
-			session->image_write_callback(session, frame, img, session->image_write_callback_user_data);
-		}
-
-		encoded_data_len = size;
-		switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, flagp);	
-	}
-
-	return SWITCH_STATUS_SUCCESS;
-}
-
-
-SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags,
-																	  int stream_id)
-{
-	switch_io_event_hook_video_write_frame_t *ptr;
-	switch_status_t status = SWITCH_STATUS_FALSE;
-
-	if (switch_channel_down(session->channel)) {
-		return SWITCH_STATUS_FALSE;
-	}
-
-	if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) {
-		return SWITCH_STATUS_SUCCESS;
-	}
-
-	if (session->endpoint_interface->io_routines->write_video_frame) {
-		if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) {
-			for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) {
-				if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) {
-					break;
-				}
-			}
-		}
-	}
-	return status;
-}
-
-SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags,
-																	 int stream_id)
-{
-	switch_status_t status = SWITCH_STATUS_FALSE;
-	switch_io_event_hook_video_read_frame_t *ptr;
-
-	switch_assert(session != NULL);
-
-	if (switch_channel_down(session->channel)) {
-		return SWITCH_STATUS_FALSE;
-	}
-
-	if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) {
-		*frame = &runtime.dummy_cng_frame;
-		switch_yield(20000);
-		return SWITCH_STATUS_SUCCESS;
-	}
-
-	if (session->endpoint_interface->io_routines->read_video_frame) {
-		if ((status = session->endpoint_interface->io_routines->read_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) {
-			for (ptr = session->event_hooks.video_read_frame; ptr; ptr = ptr->next) {
-				if ((status = ptr->video_read_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) {
-					break;
-				}
-			}
-		}
-	}
-
-	if (status == SWITCH_STATUS_INUSE) {
-		*frame = &runtime.dummy_cng_frame;
-		switch_yield(20000);
-		return SWITCH_STATUS_SUCCESS;
-	}
-
-	if (status != SWITCH_STATUS_SUCCESS) {
-		goto done;
-	}
-
-	if (!(*frame)) {
-		goto done;
-	}
-
-	if (switch_test_flag(*frame, SFF_CNG)) {
-		status = SWITCH_STATUS_SUCCESS;
-		goto done;
-	}
-
-	if (session->bugs) {
-		switch_media_bug_t *bp;
-		switch_bool_t ok = SWITCH_TRUE;
-		int prune = 0;
-		switch_thread_rwlock_rdlock(session->bug_rwlock);
-		for (bp = session->bugs; bp; bp = bp->next) {
-			if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) {
-				continue;
-			}
-
-			if (!switch_channel_test_flag(session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bp, SMBF_ANSWER_REQ)) {
-				continue;
-			}
-
-			if (switch_test_flag(bp, SMBF_PRUNE)) {
-				prune++;
-				continue;
-			}
-
-			if (bp->ready && switch_test_flag(bp, SMBF_READ_PING)) {
-				switch_mutex_lock(bp->read_mutex);
-				bp->ping_frame = *frame;
-				if (bp->callback) {
-					if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_READ_VIDEO_PING) == SWITCH_FALSE
-						|| (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) {
-						ok = SWITCH_FALSE;
-					}
-				}
-				bp->ping_frame = NULL;;
-				switch_mutex_unlock(bp->read_mutex);
-			}
-
-			if (ok == SWITCH_FALSE) {
-				switch_set_flag(bp, SMBF_PRUNE);
-				prune++;
-			}
-		}
-		switch_thread_rwlock_unlock(session->bug_rwlock);
-		if (prune) {
-			switch_core_media_bug_prune(session);
-		}
-	}
-
-  done:
-
-	return status;
-}
-
 SWITCH_DECLARE(void) switch_core_gen_encoded_silence(unsigned char *data, const switch_codec_implementation_t *read_impl, switch_size_t len)
 {
 	unsigned char g729_filler[] = {

src/switch_core_media.c

@@ -154,6 +154,9 @@ typedef struct switch_rtp_engine_s {
 	uint8_t pli;
 	uint8_t nack;
 	uint8_t no_crypto;
+
+	switch_codec_settings_t codec_settings;
+	
 } switch_rtp_engine_t;
 
 
@@ -190,6 +193,9 @@ struct switch_media_handle_s {
 
 	switch_rtp_crypto_mode_t crypto_mode;
 	switch_rtp_crypto_key_type_t crypto_suite_order[CRYPTO_INVALID+1];
+	switch_time_t video_last_key_time;
+	switch_time_t video_init;
+	switch_timer_t video_timer;
 };
 
 
@@ -1436,6 +1442,10 @@ SWITCH_DECLARE(void) switch_media_handle_destroy(switch_core_session_t *session)
 	v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO];	
 
 	
+	if (smh->video_timer.timer_interface) {
+		switch_core_timer_destroy(&smh->video_timer);
+	}
+
 	if (switch_core_codec_ready(&a_engine->read_codec)) {
 		switch_core_codec_destroy(&a_engine->read_codec);
 	}
@@ -1499,6 +1509,15 @@ SWITCH_DECLARE(switch_status_t) switch_media_handle_create(switch_media_handle_t
 
 		session->media_handle->mparams = params;
 		
+		if (!session->media_handle->mparams->video_key_freq) {
+			session->media_handle->mparams->video_key_freq = 15000000;
+		}
+
+		if (!session->media_handle->mparams->video_key_first) {
+			session->media_handle->mparams->video_key_first = 1000000;
+		}
+
+
 		for (i = 0; i <= CRYPTO_INVALID; i++) {
 			session->media_handle->crypto_suite_order[i] = CRYPTO_INVALID;
 		}
@@ -2262,6 +2281,43 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_get_offered_pt(switch_core_ses
 	return SWITCH_STATUS_FALSE;
 }
 
+//#define get_int_value(_var, _set) { const char *__v = switch_channel_get_variable(session->channel, _var); if (__v) { _set = atol(__v);} }
+//?
+static void switch_core_session_parse_codec_settings(switch_core_session_t *session, switch_media_type_t type) 
+{
+	switch_media_handle_t *smh;
+	switch_rtp_engine_t *engine;
+
+	switch_assert(session);
+
+	if (!(smh = session->media_handle)) {
+		return;
+	}
+
+	if ((engine = &smh->engines[type]));
+	
+	switch(type) {
+	case SWITCH_MEDIA_TYPE_AUDIO:
+		break;
+	case SWITCH_MEDIA_TYPE_VIDEO: 
+		{
+			const char *bwv = switch_channel_get_variable(session->channel, "video_codec_bandwidth");
+			uint32_t bw = 0;
+
+			if (bwv && (bw = (uint32_t) atol(bwv))) {
+				if (switch_stristr("kb", bwv)) {
+					bw *= 125;
+				} else if (switch_stristr("mb", bwv)) {
+					bw *= 125000;
+				}
+				engine->codec_settings.video.bandwidth = bw;
+			}
+		}
+		break;
+	default:
+		break;
+	}
+}
 
 
 //?
@@ -2300,7 +2356,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se
 		}
 	}
 
-
+	switch_core_session_parse_codec_settings(session, SWITCH_MEDIA_TYPE_VIDEO);
 
 	if (switch_core_codec_init(&v_engine->read_codec,
 							   v_engine->cur_payload_map->rm_encoding,
@@ -2309,7 +2365,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se
 							   0,
 							   1,
 							   SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE,
-							   NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
+							   &v_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
 		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n");
 		return SWITCH_STATUS_FALSE;
 	} else {
@@ -2320,7 +2376,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se
 								   0,
 								   1,
 								   SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE,
-								   NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
+								   &v_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n");
 			return SWITCH_STATUS_FALSE;
 		} else {
@@ -2361,7 +2417,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se
 			switch_channel_set_variable(session->channel, "rtp_use_video_codec_fmtp", v_engine->cur_payload_map->rm_fmtp);
 			switch_channel_set_variable_printf(session->channel, "rtp_use_video_codec_rate", "%d", v_engine->cur_payload_map->rm_rate);
 			switch_channel_set_variable_printf(session->channel, "rtp_use_video_codec_ptime", "%d", 0);
-
 		}
 	}
 	return SWITCH_STATUS_SUCCESS;
@@ -2437,6 +2492,9 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_codec(switch_core_session_
 		}
 	}
 
+
+	switch_core_session_parse_codec_settings(session, SWITCH_MEDIA_TYPE_AUDIO);
+
 	if (switch_core_codec_init_with_bitrate(&a_engine->read_codec,
 											a_engine->cur_payload_map->iananame,
 											a_engine->cur_payload_map->rm_fmtp,
@@ -2445,7 +2503,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_codec(switch_core_session_
 											a_engine->cur_payload_map->channels,
 											a_engine->cur_payload_map->bitrate,
 											SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE | codec_flags,
-											NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
+											&a_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
 		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n");
 		switch_channel_hangup(session->channel, SWITCH_CAUSE_INCOMPATIBLE_DESTINATION);
 		switch_goto_status(SWITCH_STATUS_FALSE, end);
@@ -2462,7 +2520,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_codec(switch_core_session_
 											a_engine->cur_payload_map->channels,
 											a_engine->cur_payload_map->bitrate,
 											SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE | codec_flags,
-											NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
+											&a_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
 		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n");
 		switch_channel_hangup(session->channel, SWITCH_CAUSE_INCOMPATIBLE_DESTINATION);
 		switch_goto_status(SWITCH_STATUS_FALSE, end);
@@ -7211,7 +7269,7 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess
 					nack++;
 				}
 				
-				nack = v_engine->nack = pli = v_engine->pli = 0;
+				nack = v_engine->nack = 0;//pli = v_engine->pli = 0;
 				
 				if (vp8) {
 					
@@ -9502,6 +9560,223 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
 	return SWITCH_STATUS_FALSE;
 }
 
+
+SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data)
+{
+	session->image_write_callback = callback;
+	session->image_write_callback_user_data = user_data;
+}
+
+SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, 
+																	  switch_image_t *img, switch_size_t size, uint32_t *flag)
+{
+	uint32_t encoded_data_len = size, lflag = 0, *flagp = flag;
+	switch_codec_t *codec = switch_core_session_get_video_write_codec(session);
+	switch_timer_t *timer;
+	switch_media_handle_t *smh;
+	//switch_rtp_engine_t *v_engine;
+
+	switch_assert(session);
+
+	if (!(smh = session->media_handle)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	//v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO];	
+
+
+	if (!flag) {
+		flagp = &lflag;
+	}
+
+	if (!(timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO))) {
+
+		if (!smh->video_timer.timer_interface) {
+			switch_core_timer_init(&smh->video_timer, "soft", 1, 90, switch_core_session_get_pool(session));
+		}
+
+		timer = &smh->video_timer;
+	}
+
+
+	do {
+		encoded_data_len = size;
+		switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp);
+
+		if (*flagp & SFF_PICTURE_RESET) {
+			smh->video_init = 0;
+			smh->video_last_key_time = 0;
+			*flagp &= ~SFF_PICTURE_RESET;
+		}
+
+		frame->datalen = encoded_data_len;
+		frame->packetlen = frame->datalen + 12;
+		frame->m = (*flagp & SFF_MARKER) ? 1 : 0;
+		frame->timestamp = timer->samplecount;
+
+		switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
+		
+		switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
+
+		if (session->image_write_callback) {
+			session->image_write_callback(session, frame, img, session->image_write_callback_user_data);
+		}
+
+		img = NULL;
+
+	} while(encoded_data_len);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags,
+																	  int stream_id)
+{
+	switch_io_event_hook_video_write_frame_t *ptr;
+	switch_status_t status = SWITCH_STATUS_FALSE;
+	switch_media_handle_t *smh;
+
+	switch_time_t now = switch_micro_time_now();
+
+	switch_assert(session);
+
+	if (!(smh = session->media_handle)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	if (switch_channel_down(session->channel)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) {
+		return SWITCH_STATUS_SUCCESS;
+	}
+
+	if (!smh->video_init && smh->mparams->video_key_first && (now - smh->video_last_key_time) > smh->mparams->video_key_first) {
+		switch_core_media_gen_key_frame(smh->session);
+
+		if (smh->video_last_key_time) {
+			smh->video_init = 1;
+		}
+
+		smh->video_last_key_time = now;
+	}	
+
+	if (smh->mparams->video_key_freq && (now - smh->video_last_key_time) > smh->mparams->video_key_freq) {
+		switch_core_media_gen_key_frame(smh->session);
+		smh->video_last_key_time = now;
+	}	
+
+	if (session->endpoint_interface->io_routines->write_video_frame) {
+		if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) {
+			for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) {
+				if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) {
+					break;
+				}
+			}
+		}
+	}
+	return status;
+}
+
+SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags,
+																	 int stream_id)
+{
+	switch_status_t status = SWITCH_STATUS_FALSE;
+	switch_io_event_hook_video_read_frame_t *ptr;
+
+	switch_assert(session != NULL);
+
+	if (switch_channel_down(session->channel)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) {
+		*frame = &runtime.dummy_cng_frame;
+		switch_yield(20000);
+		return SWITCH_STATUS_SUCCESS;
+	}
+
+	if (session->endpoint_interface->io_routines->read_video_frame) {
+		if ((status = session->endpoint_interface->io_routines->read_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) {
+			for (ptr = session->event_hooks.video_read_frame; ptr; ptr = ptr->next) {
+				if ((status = ptr->video_read_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) {
+					break;
+				}
+			}
+		}
+	}
+
+	if (status == SWITCH_STATUS_INUSE) {
+		*frame = &runtime.dummy_cng_frame;
+		switch_yield(20000);
+		return SWITCH_STATUS_SUCCESS;
+	}
+
+	if (status != SWITCH_STATUS_SUCCESS) {
+		goto done;
+	}
+
+	if (!(*frame)) {
+		goto done;
+	}
+
+	if (switch_test_flag(*frame, SFF_CNG)) {
+		status = SWITCH_STATUS_SUCCESS;
+		goto done;
+	}
+
+	if (session->bugs) {
+		switch_media_bug_t *bp;
+		switch_bool_t ok = SWITCH_TRUE;
+		int prune = 0;
+		switch_thread_rwlock_rdlock(session->bug_rwlock);
+		for (bp = session->bugs; bp; bp = bp->next) {
+			if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) {
+				continue;
+			}
+
+			if (!switch_channel_test_flag(session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bp, SMBF_ANSWER_REQ)) {
+				continue;
+			}
+
+			if (switch_test_flag(bp, SMBF_PRUNE)) {
+				prune++;
+				continue;
+			}
+
+			if (bp->ready && switch_test_flag(bp, SMBF_READ_PING)) {
+				switch_mutex_lock(bp->read_mutex);
+				bp->ping_frame = *frame;
+				if (bp->callback) {
+					if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_READ_VIDEO_PING) == SWITCH_FALSE
+						|| (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) {
+						ok = SWITCH_FALSE;
+					}
+				}
+				bp->ping_frame = NULL;;
+				switch_mutex_unlock(bp->read_mutex);
+			}
+
+			if (ok == SWITCH_FALSE) {
+				switch_set_flag(bp, SMBF_PRUNE);
+				prune++;
+			}
+		}
+		switch_thread_rwlock_unlock(session->bug_rwlock);
+		if (prune) {
+			switch_core_media_bug_prune(session);
+		}
+	}
+
+  done:
+
+	return status;
+}
+
+
+
 /* For Emacs:
  * Local Variables:
  * mode:c

src/switch_rtp.c

@@ -56,6 +56,7 @@
 #include <switch_ssl.h>
 
 #define FIR_COUNTDOWN 50
+#define PLI_COUNTDOWN 50
 #define JITTER_LEAD_FRAMES 10
 #define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++
 #define READ_DEC(rtp_session)  switch_mutex_unlock(rtp_session->read_mutex); rtp_session->reading--
@@ -299,6 +300,7 @@ struct switch_rtp {
 	rtcp_ext_msg_t rtcp_ext_send_msg;
 	uint8_t fir_seq;
 	uint16_t fir_countdown;
+	uint16_t pli_countdown;
 	ts_normalize_t ts_norm;
 	switch_sockaddr_t *remote_addr, *rtcp_remote_addr;
 	rtp_msg_t recv_msg;
@@ -360,6 +362,7 @@ struct switch_rtp {
 	uint32_t samples_per_second;
 	uint32_t conf_samples_per_interval;
 	uint16_t rtcp_send_rate;
+	switch_time_t rtcp_last_sent;
 	uint32_t rsamples_per_interval;
 	uint32_t ms_per_packet;
 	uint32_t one_second;
@@ -949,6 +952,9 @@ static void handle_ice(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice, void *d
 					ice->rready = 1;
 				}
 				
+				if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
+					switch_core_media_gen_key_frame(rtp_session->session);
+				}
 				switch_rtp_set_flag(rtp_session, SWITCH_RTP_FLAG_FLUSH);
 			}
 		}
@@ -1097,6 +1103,9 @@ static void handle_ice(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice, void *d
 
 			if (!ice->ready) {
 				ice->ready = 1;
+				if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
+					switch_core_media_gen_key_frame(rtp_session->session);
+				}
 				switch_rtp_set_flag(rtp_session, SWITCH_RTP_FLAG_FLUSH);
 			}
 
@@ -2017,21 +2026,23 @@ static int check_rtcp_and_ice(switch_rtp_t *rtp_session)
 	switch_time_t now = switch_micro_time_now();
 
 	if (rtp_session->fir_countdown) {
-		//if (rtp_session->fir_countdown == FIR_COUNTDOWN) {
-		//	do_flush(rtp_session, SWITCH_TRUE);
-		//}
 
 		if (rtp_session->fir_countdown == FIR_COUNTDOWN || (rtp_session->fir_countdown == FIR_COUNTDOWN / 2) || rtp_session->fir_countdown == 1) {
-			if (rtp_session->flags[SWITCH_RTP_FLAG_PLI]) {
-				send_pli(rtp_session);
-			} else {
 			send_fir(rtp_session);
 		}
-		}
 
 		rtp_session->fir_countdown--;
 	}
 
+	if (rtp_session->pli_countdown) {
+
+		if (rtp_session->pli_countdown == PLI_COUNTDOWN || (rtp_session->pli_countdown == PLI_COUNTDOWN / 2) || rtp_session->pli_countdown == 1) {
+			send_pli(rtp_session);
+		}
+
+		rtp_session->pli_countdown--;
+	}
+
 	if (rtp_session->flags[SWITCH_RTP_FLAG_AUTO_CNG] && rtp_session->send_msg.header.ts && rtp_session->cng_pt &&
 		rtp_session->timer.samplecount >= (rtp_session->last_write_samplecount + (rtp_session->samples_per_interval * 60))) {
 		uint8_t data[10] = { 0 };
@@ -2059,17 +2070,16 @@ static int check_rtcp_and_ice(switch_rtp_t *rtp_session)
 		rtcp_ok = 0;
 	}
 
-	if (rtp_session->rtcp_sock_output && rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] &&
-		!rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU] &&
-		(rtp_session->timer.samplecount - rtp_session->stats.rtcp.last_rpt_ts >= rtp_session->samples_per_second * rtp_session->rtcp_send_rate) ) {
 	
+	if (rtp_session->rtcp_sock_output && rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] &&
+		!rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU] && (now - rtp_session->rtcp_last_sent) > rtp_session->rtcp_send_rate * 1000000) {
 		switch_rtcp_numbers_t * stats = &rtp_session->stats.rtcp;
 		struct switch_rtcp_receiver_report *rr;
 		struct switch_rtcp_sender_report *sr;
 		struct switch_rtcp_report_block *rtcp_report_block;
-
 		switch_size_t rtcp_bytes = sizeof(struct switch_rtcp_hdr_s)+sizeof(uint32_t); /* add size of the packet header and the ssrc */
 
+		rtp_session->rtcp_last_sent = now;
 		rtp_session->rtcp_send_msg.header.version = 2;
 		rtp_session->rtcp_send_msg.header.p = 0;
 		rtp_session->rtcp_send_msg.header.count = 1;
@@ -3548,6 +3558,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session
 	rtp_session->recv_msg.header.cc = 0;
 
 	rtp_session->payload = payload;
+	rtp_session->rtcp_last_sent = switch_micro_time_now();
 
 	switch_rtp_set_interval(rtp_session, ms_per_packet, samples_per_interval);
 	rtp_session->conf_samples_per_interval = samples_per_interval;
@@ -3790,7 +3801,8 @@ static void jb_callback(stfu_instance_t *i, void *udata)
 
 SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session)
 {
-	if (rtp_session->timer.timer_interface) {
+
+	if (rtp_session && rtp_session->timer.timer_interface) {
 		if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
 			switch_core_timer_sync(&rtp_session->timer);
 		}
@@ -4066,13 +4078,23 @@ SWITCH_DECLARE(void) switch_rtp_flush(switch_rtp_t *rtp_session)
 SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session)
 {
 	if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && 
-		(rtp_session->ice.ice_user || rtp_session->flags[SWITCH_RTP_FLAG_FIR] || rtp_session->flags[SWITCH_RTP_FLAG_PLI])) {
+		(rtp_session->ice.ice_user || rtp_session->flags[SWITCH_RTP_FLAG_FIR])) {
 		if (!rtp_session->fir_countdown) {
 			rtp_session->fir_countdown = FIR_COUNTDOWN;
 		}
 	}
 }
 
+SWITCH_DECLARE(void) switch_rtp_video_loss(switch_rtp_t *rtp_session)
+{
+	if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && 
+		(rtp_session->ice.ice_user || rtp_session->flags[SWITCH_RTP_FLAG_PLI])) {
+		if (!rtp_session->pli_countdown) {
+			rtp_session->pli_countdown = PLI_COUNTDOWN;
+		}
+	}
+}
+
 SWITCH_DECLARE(void) switch_rtp_break(switch_rtp_t *rtp_session)
 {
 	if (!switch_rtp_ready(rtp_session)) {
@@ -5159,12 +5181,29 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t
 static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_size_t *bytes)
 {
 	switch_status_t status = SWITCH_STATUS_FALSE;
-	switch_core_session_t *session = switch_core_memory_pool_get_data(rtp_session->pool, "__session");
-	switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG10,"Received an RTCP packet of length %" SWITCH_SIZE_T_FMT " bytes\n", *bytes);
-	if (rtp_session->rtcp_recv_msg.header.version == 2) {
-		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG10,"RTCP packet type is %d\n", rtp_session->rtcp_recv_msg.header.type);
 
-		if (rtp_session->rtcp_recv_msg.header.type == 200 || rtp_session->rtcp_recv_msg.header.type == 201) {
+	if (rtp_session->rtcp_recv_msg_p->header.version == 2) {
+
+
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG,
+						  "RTCP packet bytes %" SWITCH_SIZE_T_FMT " type %d\n", *bytes, rtp_session->rtcp_recv_msg_p->header.type);
+
+
+
+		if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && *bytes > 94) {
+			//(rtp_session->rtcp_recv_msg_p->header.type == 205 || //RTPFB
+			//rtp_session->rtcp_recv_msg_p->header.type == 206)) {//PSFB
+			
+			rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) rtp_session->rtcp_recv_msg_p;			
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "PICKED UP XRTCP type: %d fmt: %d\n", 
+							  rtp_session->rtcp_recv_msg_p->header.type, extp->header.fmt);
+			
+			if ((extp->header.fmt == 4) || (extp->header.fmt == 1)) { /* FIR || PLI */
+				switch_core_media_gen_key_frame(rtp_session->session);
+			}
+		} else
+
+		if (rtp_session->rtcp_recv_msg_p->header.type == 200 || rtp_session->rtcp_recv_msg_p->header.type == 201) {
 			struct switch_rtcp_report_block *report_block;
 			switch_time_t now;
 			switch_time_exp_t now_hr;
@@ -5177,8 +5216,9 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz
 			ntp_usec = (uint32_t)(now - (sec*1000000)); /* micro seconds */
 			lsr_now = (uint32_t)(ntp_usec*0.065536) | (ntp_sec&0x0000ffff)<<16; // 0.065536 is used for convertion from useconds 
 
-			if (rtp_session->rtcp_recv_msg.header.type == 200) { /* Sender report */
+			if (rtp_session->rtcp_recv_msg_p->header.type == 200) { /* Sender report */
-				struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg.body;
+				struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg_p->body;
+				
 				report_block = &sr->report_block;
 				rtp_session->stats.rtcp.packet_count += ntohl(sr->sender_info.pc);
 				rtp_session->stats.rtcp.octet_count += ntohl(sr->sender_info.oc);
@@ -5187,7 +5227,7 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz
 				lsr = (ntohl(sr->sender_info.ntp_lsw)&0xffff0000)>>16 | (ntohl(sr->sender_info.ntp_msw)&0x0000ffff)<<16; /* The middle 32 bits out of 64 in the NTP timestamp */
 				rtp_session->stats.rtcp.last_recv_lsr_peer = htonl(lsr);  /* Save it include it in the next SR */
 				rtp_session->stats.rtcp.last_recv_lsr_local = lsr_now;    /* Save it to calculate DLSR when generating next SR */
-				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG10,"Received a SR with %d report blocks, " \
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG10,"Received a SR with %d report blocks, " \
 				                          "length in words = %d, " \
 				                          "SSRC = 0x%X, " \
 				                          "NTP MSW = %u, " \
@@ -5195,8 +5235,8 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz
 				                          "RTP timestamp = %u, " \
 				                          "Sender Packet Count = %u, " \
 				                          "Sender Octet Count = %u\n",
-				                          rtp_session->rtcp_recv_msg.header.count,
+				                          rtp_session->rtcp_recv_msg_p->header.count,
-				                          ntohs((uint16_t)rtp_session->rtcp_recv_msg.header.length),
+				                          ntohs((uint16_t)rtp_session->rtcp_recv_msg_p->header.length),
 				                          ntohl(sr->ssrc),
 				                          ntohl(sr->sender_info.ntp_msw),
 				                          ntohl(sr->sender_info.ntp_lsw),
@@ -5204,7 +5244,7 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz
 				                          ntohl(sr->sender_info.pc),
 				                          ntohl(sr->sender_info.oc));
 			} else { /* Receiver report */
-				struct switch_rtcp_receiver_report* rr = (struct switch_rtcp_receiver_report*)rtp_session->rtcp_recv_msg.body;
+				struct switch_rtcp_receiver_report* rr = (struct switch_rtcp_receiver_report*)rtp_session->rtcp_recv_msg_p->body;
 				report_block = &rr->report_block;
 				packet_ssrc = rr->ssrc;
 			}
@@ -5213,8 +5253,8 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz
 			if (report_block->lsr && !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) {
 				switch_time_exp_gmt(&now_hr,now);
 				/* Calculating RTT = A - DLSR - LSR */
-				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_NOTICE,
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG,
-				   "Receiving an RTCP packet[%04d-%02d-%02d %02d:%02d:%02d.%d] SSRC[%u]"
+								  "Receiving an RTCP packet\n[%04d-%02d-%02d %02d:%02d:%02d.%d] SSRC[%u]\n"
 								  "RTT[%f] A[%u] - DLSR[%u] - LSR[%u]\n",
 								  1900 + now_hr.tm_year, now_hr.tm_mday, now_hr.tm_mon, now_hr.tm_hour, now_hr.tm_min, now_hr.tm_sec, now_hr.tm_usec,
 								  ntohl(packet_ssrc), (double)(lsr_now - ntohl(report_block->dlsr) - ntohl(report_block->lsr))/65536,
@@ -5673,26 +5713,6 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_
 				if (rtcp_status == SWITCH_STATUS_SUCCESS) {
 					switch_rtp_reset_media_timer(rtp_session);
 
-					if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && (rtp_session->rtcp_recv_msg_p->header.type == 205 || //RTPFB
-																	  rtp_session->rtcp_recv_msg_p->header.type == 206)) {//PSFB
-						rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) rtp_session->rtcp_recv_msg_p;
-
-						switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "PICKED UP XRTCP type: %d fmt: %d\n", 
-										  rtp_session->rtcp_recv_msg_p->header.type, extp->header.fmt);
-
-						if ((extp->header.fmt == 4) || (extp->header.fmt == 1)) { /* FIR || PLI */
-
-							switch_core_media_codec_control(rtp_session->session,
-															SWITCH_MEDIA_TYPE_VIDEO,
-															SWITCH_IO_WRITE,
-															SCC_VIDEO_REFRESH,
-															SCCT_NONE,
-															NULL,
-															NULL,
-															NULL);
-						}
-					}
-
 					if (rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) {
 						switch_channel_t *channel = switch_core_session_get_channel(rtp_session->session);
 						const char *uuid = switch_channel_get_partner_uuid(channel);
@@ -6235,14 +6255,14 @@ SWITCH_DECLARE(switch_status_t) switch_rtcp_zerocopy_read_frame(switch_rtp_t *rt
 
 	/* A fresh frame has been found! */
 	if (rtp_session->rtcp_fresh_frame) {
-		struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg.body;
+		struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg_p->body;
 		int i = 0;
 
 		/* turn the flag off! */
 		rtp_session->rtcp_fresh_frame = 0;
 
                 frame->ssrc = ntohl(sr->ssrc);
-                frame->packet_type = (uint16_t)rtp_session->rtcp_recv_msg.header.type;
+                frame->packet_type = (uint16_t)rtp_session->rtcp_recv_msg_p->header.type;
                 frame->ntp_msw = ntohl(sr->sender_info.ntp_msw);
                 frame->ntp_lsw = ntohl(sr->sender_info.ntp_lsw);
                 frame->timestamp = ntohl(sr->sender_info.ts);
@@ -6787,10 +6807,9 @@ static int rtp_common_write(switch_rtp_t *rtp_session,
 
 		if (rtp_session->flags[SWITCH_RTP_FLAG_USE_TIMER]) {
 			rtp_session->last_write_samplecount = rtp_session->timer.samplecount;
-		} else {
-			rtp_session->last_write_timestamp = switch_micro_time_now();
 		}
 
+		rtp_session->last_write_timestamp = switch_micro_time_now();
 	}
 
 	ret = (int) bytes;