kenmirrors
/
freeswitch
mirror of https://github.com/signalwire/freeswitch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

FS-7500: refactor to use switch_core_video 

added switch_core_codec_encode_video and switch_core_codec_decode video and add separate video implementations
the switch_core_video code depends on libvpx, wraped into the switch_ namespace like switch_apr, need to figure out how to find the correct libvpx lib in autotools
This commit is contained in:
Seven Du 2014-08-04 03:15:26 +08:00 committed by Michael Jerris
parent bcbdf8a347
commit ea2f00b3d9
9 changed files with 180 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile.am
View File

@ -188,6 +188,7 @@ library_include_HEADERS = \
src/include/switch_scheduler.h \
src/include/switch_core.h \
src/include/switch_core_media.h \
src/include/switch_core_video.h \
src/include/switch_core_db.h \
src/include/switch_mprintf.h \
src/include/switch_config.h \
@ -254,6 +255,7 @@ libfreeswitch_la_SOURCES = \
src/switch_core.c \
src/switch_version.c \
src/switch_core_media.c \
src/switch_core_video.c \
src/switch_sdp.c \
src/switch_scheduler.c \
src/switch_core_db.c \

1
src/include/switch.h
View File

@ -143,6 +143,7 @@
#include "switch_json.h"
#include "switch_limit.h"
#include "switch_core_media.h"
#include "switch_core_video.h"
#include <libteletone.h>

27
src/include/switch_core.h
View File

@ -1621,6 +1621,33 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode(switch_codec_t *codec,
uint32_t encoded_rate,
void *decoded_data, uint32_t *decoded_data_len, uint32_t *decoded_rate, unsigned int *flag);
/*!
\brief Encode video data using a codec handle
\param codec the codec handle to use
\param img the img in I420 format
\param encoded_data the buffer to write the encoded data to
\param encoded_data_len the size of the encoded_data buffer
\param flag flags to exchange
\return SWITCH_STATUS_SUCCESS if the data was encoded
\note encoded_data_len will be rewritten to the in-use size of encoded_data
*/
SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec,
switch_image_t *img,
void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag);
/*!
\brief Decode video data using a codec handle
\param codec the codec handle to use
\param frame the frame to be decoded
\param img the new image in I420 format, allocated by the codec
\param flag flags to exchange
\return SWITCH_STATUS_SUCCESS if the data was decoded, and a non-NULL img
*/
SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec,
switch_frame_t *frame,
switch_image_t **img, unsigned int *flag);
/*!
\brief Destroy an initalized codec handle
\param codec the codec handle to destroy

39
src/include/switch_loadable_module.h
View File

@ -532,6 +532,45 @@ static inline void switch_core_codec_add_implementation(switch_memory_pool_t *po
///\}
static inline void switch_core_codec_add_video_implementation(switch_memory_pool_t *pool, switch_codec_interface_t *codec_interface,
/*! the IANA code number */
switch_payload_t ianacode,
/*! the IANA code name */
const char *iananame,
/*! default fmtp to send (can be overridden by the init function) */
char *fmtp,
switch_core_codec_init_func_t init,
/*! function to encode raw data into encoded data */
switch_core_codec_video_encode_func_t encode,
/*! function to decode encoded data into raw data */
switch_core_codec_video_decode_func_t decode,
/*! deinitalize a codec handle using this implementation */
switch_core_codec_destroy_func_t destroy)
{
switch_codec_implementation_t *impl = (switch_codec_implementation_t *) switch_core_alloc(pool, sizeof(*impl));
memset(impl, 0, sizeof(*impl));
impl->codec_type = SWITCH_CODEC_TYPE_VIDEO;
impl->ianacode = ianacode;
impl->iananame = switch_core_strdup(pool, iananame);
impl->fmtp = switch_core_strdup(pool, fmtp);
impl->samples_per_second = 90000;
impl->actual_samples_per_second = 90000;
impl->bits_per_second = 0;
impl->microseconds_per_packet = 0;
impl->samples_per_packet = 0;
impl->number_of_channels = 1;
impl->codec_frames_per_packet = 1;
impl->init = init;
impl->encode_video = encode;
impl->decode_video = decode;
impl->destroy = destroy;
impl->codec_id = codec_interface->codec_id;
impl->next = codec_interface->implementations;
impl->impl_id = switch_core_codec_next_id();
codec_interface->implementations = impl;
}
#define SWITCH_DECLARE_STATIC_MODULE(init, load, run, shut) void init(void) { \
switch_loadable_module_build_dynamic(__FILE__, load, run, shut, SWITCH_FALSE); \
}

8
src/include/switch_module_interfaces.h
View File

@ -639,10 +639,6 @@ struct switch_codec {
struct switch_codec *next;
switch_core_session_t *session;
switch_frame_t *cur_frame;
/*! raw picture for encode */
switch_picture_t enc_picture;
/*! decoded picture */
switch_picture_t dec_picture;
};
/*! \brief A table of settings and callbacks that define a paticular implementation of a codec */
@ -679,6 +675,10 @@ struct switch_codec_implementation {
switch_core_codec_encode_func_t encode;
/*! function to decode encoded data into raw data */
switch_core_codec_decode_func_t decode;
/*! function to encode video raw data into encoded data */
switch_core_codec_video_encode_func_t encode_video;
/*! function to decode video encoded data into raw data */
switch_core_codec_video_decode_func_t decode_video;
/*! deinitalize a codec handle using this implementation */
switch_core_codec_destroy_func_t destroy;
uint32_t codec_id;

11
src/include/switch_types.h
View File

@ -541,7 +541,6 @@ SWITCH_DECLARE_DATA extern switch_filenames SWITCH_GLOBAL_filenames;
#define SWITCH_MAX_SAMPLE_LEN 48
#define SWITCH_BYTES_PER_SAMPLE 2 /* slin is 2 bytes per sample */
#define SWITCH_RECOMMENDED_BUFFER_SIZE 8192
#define SWITCH_RECOMMENDED_VIDEO_BUFFER_SIZE 4096 * 1024 /* Fixme: Just Make sure it's big enough for now */
#define SWITCH_MAX_CODECS 50
#define SWITCH_MAX_STATE_HANDLERS 30
#define SWITCH_CORE_QUEUE_LEN 100000
@ -2095,7 +2094,7 @@ typedef struct switch_caller_extension switch_caller_extension_t;
typedef struct switch_caller_application switch_caller_application_t;
typedef struct switch_state_handler_table switch_state_handler_table_t;
typedef struct switch_timer switch_timer_t;
typedef struct switch_picture switch_picture_t;
typedef struct switch_image switch_image_t;
typedef struct switch_codec switch_codec_t;
typedef struct switch_core_thread_session switch_core_thread_session_t;
typedef struct switch_codec_implementation switch_codec_implementation_t;
@ -2168,6 +2167,14 @@ typedef switch_status_t (*switch_core_codec_decode_func_t) (switch_codec_t *code
uint32_t encoded_rate,
void *decoded_data, uint32_t *decoded_data_len, uint32_t *decoded_rate, unsigned int *flag);
typedef switch_status_t (*switch_core_codec_video_encode_func_t) (switch_codec_t *codec,
switch_image_t *img,
void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag);
typedef switch_status_t (*switch_core_codec_video_decode_func_t) (switch_codec_t *codec,
switch_frame_t *frame,
switch_image_t **img, unsigned int *flag);
typedef switch_status_t (*switch_core_codec_init_func_t) (switch_codec_t *, switch_codec_flag_t, const switch_codec_settings_t *codec_settings);
typedef switch_status_t (*switch_core_codec_fmtp_parse_func_t) (const char *fmtp, switch_codec_fmtp_t *codec_fmtp);
typedef switch_status_t (*switch_core_codec_destroy_func_t) (switch_codec_t *);

76
src/mod/applications/mod_fsv/mod_fsv.c
View File

@ -574,12 +574,18 @@ SWITCH_STANDARD_APP(play_yuv_function)
switch_dtmf_t dtmf = { 0 };
switch_frame_t *read_frame;
uint32_t width = 0, height = 0, size;
switch_byte_t *yuv;
switch_image_t *img = NULL;
switch_byte_t *yuv = NULL;
switch_time_t last_video_ts = 0;
int argc;
char *argv[3] = { 0 };
char *mydata = switch_core_session_strdup(session, data);
if (!switch_channel_test_flag(channel, CF_VIDEO)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Channel %s has no video\n", switch_channel_get_name(channel));
goto done;
}
argc = switch_separate_string(mydata, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
if (argc == 0) {
@ -593,13 +599,16 @@ SWITCH_STANDARD_APP(play_yuv_function)
width = width ? width : 352;
height = height ? height : 288;
size = width * height * 3 / 2;
yuv = malloc(size);
if (!yuv) {
img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, width, height, 0);
if (!img) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Memory Error\n");
goto end;
}
yuv = img->planes[SWITCH_PLANE_PACKED];
// switch_channel_set_flag(channel, CF_VIDEO_PASSIVE);
switch_channel_clear_flag(channel, CF_VIDEO_ECHO);
@ -625,7 +634,6 @@ SWITCH_STANDARD_APP(play_yuv_function)
switch_channel_answer(channel);
codec = switch_core_session_get_video_write_codec(session);
switch_assert(codec);
vid_frame.codec = codec;
vid_frame.packet = vid_buffer;
@ -664,18 +672,13 @@ SWITCH_STANDARD_APP(play_yuv_function)
if (read_frame) switch_core_session_write_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0);
{ /* video part */
uint32_t decoded_data_len;
uint32_t flag = 0;
uint32_t encoded_data_len = 1500;
uint32_t encoded_rate = 0;
switch_frame_t *frame = &vid_frame;
switch_time_t now = switch_micro_time_now() / 1000;
char ts_str[33];
int delta;
long delta;
codec->enc_picture.width = width;
codec->enc_picture.height = height;
decoded_data_len = width * height * 3 / 2;
delta = now - last_video_ts;
if (delta > 0) {
@ -684,11 +687,11 @@ SWITCH_STANDARD_APP(play_yuv_function)
}
sprintf(ts_str, "%u", (uint32_t)frame->timestamp);
text(yuv, width, 20, 20, ts_str);
switch_core_codec_encode(codec, NULL, yuv, decoded_data_len, 0, vid_frame.data, &encoded_data_len, &encoded_rate, &flag);
text(img->planes[SWITCH_PLANE_PACKED], width, 20, 20, ts_str);
switch_core_codec_encode_video(codec, img, vid_frame.data, &encoded_data_len, &flag);
while(encoded_data_len) {
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", codec->implementation->iananame, encoded_data_len, flag, last_video_ts);
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%lld\n", codec->implementation->iananame, encoded_data_len, flag, last_video_ts);
frame->datalen = encoded_data_len;
frame->packetlen = frame->datalen + 12;
@ -709,7 +712,7 @@ SWITCH_STANDARD_APP(play_yuv_function)
switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
encoded_data_len = 1500;
switch_core_codec_encode(codec, NULL, NULL, 0, 0, frame->data, &encoded_data_len, &encoded_rate, &flag);
switch_core_codec_encode_video(codec, NULL, vid_frame.data, &encoded_data_len, &flag);
}
}
}
@ -723,7 +726,7 @@ SWITCH_STANDARD_APP(play_yuv_function)
close(fd);
}
switch_safe_free(yuv);
switch_img_free(img);
done:
// switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
@ -737,34 +740,23 @@ SWITCH_STANDARD_APP(decode_video_function)
switch_codec_t *codec = NULL;
switch_dtmf_t dtmf = { 0 };
switch_frame_t *frame;
uint32_t size;
uint32_t width = 0, height = 0;
switch_byte_t *yuv;
switch_time_t last = switch_micro_time_now();
uint32_t max_pictures = 0;
uint32_t decoded_pictures = 0;
if (!zstr(data)) max_pictures = atoi(data);
switch_channel_set_flag(channel, CF_VIDEO_PASSIVE);
switch_channel_clear_flag(channel, CF_VIDEO_ECHO);
switch_channel_answer(channel);
if (!switch_channel_test_flag(channel, CF_VIDEO)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Channel %s has no video\n", switch_channel_get_name(channel));
goto done;
}
switch_channel_set_flag(channel, CF_VIDEO_PASSIVE);
switch_channel_clear_flag(channel, CF_VIDEO_ECHO);
switch_channel_answer(channel);
switch_core_session_refresh_video(session);
size = 1920 * 1080 * 3 / 2;
yuv = malloc(size);
if (!yuv) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Memory Error\n");
goto done;
}
switch_channel_set_variable(channel, SWITCH_PLAYBACK_TERMINATOR_USED, "");
@ -801,7 +793,7 @@ SWITCH_STANDARD_APP(decode_video_function)
}
}
if (frame) {
if (frame && frame->datalen > 0) {
switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
} else {
continue;
@ -816,19 +808,17 @@ SWITCH_STANDARD_APP(decode_video_function)
*((uint8_t *)frame->data + 8), *((uint8_t *)frame->data + 9),
*((uint8_t *)frame->data + 10), frame->m, switch_test_flag(frame, SFF_CNG) ? " CNG" : "");
if (switch_test_flag(frame, SFF_CNG)) {
if (switch_test_flag(frame, SFF_CNG) || frame->datalen < 3) {
continue;
}
if ( 1 ) { /* video part */
uint32_t decoded_rate = 0;
uint32_t decoded_data_len = size;
uint32_t flag = 0;
switch_image_t *img = NULL;
codec->cur_frame = frame;
switch_core_codec_decode(codec, NULL, frame->data, frame->datalen, 0, yuv, &decoded_data_len, &decoded_rate, &flag);
switch_core_codec_decode_video(codec, frame, &img, &flag);
if (switch_test_flag(codec->cur_frame, SFF_WAIT_KEY_FRAME)) {
if ((switch_test_flag(frame, SFF_WAIT_KEY_FRAME))) {
switch_time_t now = switch_micro_time_now();
if (now - last > 3000000) {
switch_core_session_refresh_video(session);
@ -837,19 +827,19 @@ SWITCH_STANDARD_APP(decode_video_function)
continue;
}
if (decoded_data_len > 0) {
if (codec->dec_picture.width > 0 && !width) {
width = codec->dec_picture.width;
if (img) {
if (img->d_w > 0 && !width) {
width = img->d_w;
switch_channel_set_variable_printf(channel, "video_width", "%d", width);
}
if (codec->dec_picture.height > 0 && !height) {
height = codec->dec_picture.height;
if (img->d_h > 0 && !height) {
height = img->d_h;
switch_channel_set_variable_printf(channel, "video_height", "%d", height);
}
decoded_pictures++;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "picture#%d %dx%d len: %d \n", decoded_pictures, codec->dec_picture.width, codec->dec_picture.height, decoded_data_len);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "picture#%d %dx%d\n", decoded_pictures, img->d_w, img->d_h);
if (max_pictures && (decoded_pictures >= max_pictures)) {
break;
@ -861,8 +851,6 @@ SWITCH_STANDARD_APP(decode_video_function)
switch_core_thread_session_end(session);
switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "OK");
switch_safe_free(yuv);
done:
switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE);
switch_channel_set_flag(channel, CF_VIDEO_ECHO);

59
src/switch_core_codec.c
View File

@ -784,6 +784,65 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode(switch_codec_t *codec,
return status;
}
SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec,
switch_image_t *img,
void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag)
{
switch_status_t status = SWITCH_STATUS_FALSE;
switch_assert(codec != NULL);
if (!codec->implementation || !switch_core_codec_ready(codec)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec is not initialized!\n");
return SWITCH_STATUS_NOT_INITALIZED;
}
if (!switch_test_flag(codec, SWITCH_CODEC_FLAG_ENCODE)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec encoder is not initialized!\n");
return SWITCH_STATUS_NOT_INITALIZED;
}
if (codec->mutex) switch_mutex_lock(codec->mutex);
if (codec->implementation->encode_video) {
status = codec->implementation->encode_video(codec, img, encoded_data, encoded_data_len, flag);
}
if (codec->mutex) switch_mutex_unlock(codec->mutex);
return status;
}
SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec,
switch_frame_t *frame,
switch_image_t **img, unsigned int *flag)
{
switch_status_t status = SWITCH_STATUS_FALSE;
switch_assert(codec != NULL);
switch_assert(frame != NULL);
if (!codec->implementation || !switch_core_codec_ready(codec)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decode Codec is not initialized!\n");
return SWITCH_STATUS_NOT_INITALIZED;
}
if (!switch_test_flag(codec, SWITCH_CODEC_FLAG_DECODE)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec decoder is not initialized!\n");
return SWITCH_STATUS_NOT_INITALIZED;
}
if (codec->mutex) switch_mutex_lock(codec->mutex);
if (codec->implementation->decode_video) {
status = codec->implementation->decode_video(codec, frame, img, flag);
}
if (codec->mutex) switch_mutex_unlock(codec->mutex);
return status;
}
SWITCH_DECLARE(switch_status_t) switch_core_codec_destroy(switch_codec_t *codec)
{
switch_mutex_t *mutex = codec->mutex;

20
src/switch_core_media.c
View File

@ -4387,14 +4387,11 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi
switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0);
return status;
} else {
uint8_t raw_buff[BUF_SIZE];
uint8_t rtp_buff[1500] = { 0 };
uint32_t decoded_rate = 0;
uint32_t decoded_data_len = BUF_SIZE;
uint32_t flag = 0;
uint32_t encoded_data_len = 1500;
uint32_t encoded_rate = 0;
switch_frame_t write_frame = { 0 };
switch_image_t *img = NULL;
#if 0
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d/%s != %d/%s need transcoding!!!\n",
@ -4405,23 +4402,20 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi
// if (!strcmp(codec->implementation->iananame, "VP8")) return status;
// if (!strcmp(codec->implementation->iananame, "H264")) return status;
codec->cur_frame = read_frame;
switch_core_codec_decode(codec, NULL, read_frame->data, read_frame->datalen, 0, raw_buff, &decoded_data_len, &decoded_rate, &flag);
switch_core_codec_decode_video(codec, read_frame, &img, &flag);
if (decoded_data_len < 3) return SWITCH_STATUS_SUCCESS;
if (!img) return SWITCH_STATUS_SUCCESS;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s decoded_data_len: %d size: %dx%d\n", codec->implementation->iananame, decoded_data_len, codec->dec_picture.width, codec->dec_picture.height);
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s decoded_data_len: %d size: %dx%d\n", codec->implementation->iananame, decoded_data_len, codec->dec_picture.width, codec->dec_picture.height);
write_frame.packet = rtp_buff;
write_frame.data = rtp_buff + 12;
other_codec->enc_picture.width = codec->dec_picture.width;
other_codec->enc_picture.height = codec->dec_picture.height;
encoded_data_len = 1500;
switch_core_codec_encode(other_codec, NULL, raw_buff, decoded_data_len, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag);
switch_core_codec_encode_video(other_codec, img, rtp_buff+12, &encoded_data_len, &flag);
while(encoded_data_len) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", other_codec->implementation->iananame, encoded_data_len, flag, *ts);
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", other_codec->implementation->iananame, encoded_data_len, flag, *ts);
write_frame.datalen = encoded_data_len;
write_frame.packetlen = write_frame.datalen + 12;
@ -4448,7 +4442,7 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi
switch_core_session_write_video_frame(session_b, &write_frame, SWITCH_IO_FLAG_NONE, 0);
encoded_data_len = 1500;
switch_core_codec_encode(other_codec, NULL, NULL, 0, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag);
switch_core_codec_encode_video(other_codec, NULL, rtp_buff+12, &encoded_data_len, &flag);
}
}