From 2c368307d8a522d8caa7c96f9368db563d3fa2ec Mon Sep 17 00:00:00 2001
From: Anthony Minessale <anthm@freeswitch.org>
Date: Wed, 15 Mar 2017 15:28:08 -0500
Subject: [PATCH] FS-10138: [freeswitch-core,mod_conference] Add alpha video to
conference #resolve
---
src/include/switch_module_interfaces.h | 1 +
src/mod/applications/mod_av/avformat.c | 14 +-
.../mod_conference/conference_file.c | 35 +++++
.../mod_conference/conference_member.c | 45 ++++++
.../mod_conference/conference_video.c | 146 +++++++++++++-----
.../mod_conference/mod_conference.h | 7 +-
6 files changed, 202 insertions(+), 46 deletions(-)
diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h
index faebf15094..98f5e32bf3 100644
--- a/src/include/switch_module_interfaces.h
+++ b/src/include/switch_module_interfaces.h
@@ -339,6 +339,7 @@ typedef struct switch_mm_s {
uint8_t try_hardware_encoder;
int scale_w;
int scale_h;
+ switch_img_fmt_t fmt;
} switch_mm_t;
/*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */
diff --git a/src/mod/applications/mod_av/avformat.c b/src/mod/applications/mod_av/avformat.c
index 88272baa77..2203eb3d0b 100644
--- a/src/mod/applications/mod_av/avformat.c
+++ b/src/mod/applications/mod_av/avformat.c
@@ -1330,7 +1330,6 @@ struct av_file_context {
switch_bool_t read_paused;
int errs;
switch_file_handle_t *handle;
- switch_bool_t alpha_mode;
};
typedef struct av_file_context av_file_context_t;
@@ -1389,7 +1388,12 @@ static switch_status_t open_input_file(av_file_context_t *context, switch_file_h
context->has_video = 1;
handle->duration = av_rescale_q(context->video_st.st->duration, context->video_st.st->time_base, AV_TIME_BASE_Q);
}
- handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate));
+ if (context->video_st.st->avg_frame_rate.num) {
+ handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate));
+ } else {
+ handle->mm.source_fps = 25;
+ }
+
context->read_fps = (int)handle->mm.source_fps;
}
}
@@ -1585,7 +1589,7 @@ again:
if (got_data && error >= 0) {
switch_img_fmt_t fmt = SWITCH_IMG_FMT_I420;
- if (context->alpha_mode && (
+ if ((
vframe->format == AV_PIX_FMT_YUVA420P ||
vframe->format == AV_PIX_FMT_RGBA ||
vframe->format == AV_PIX_FMT_ARGB ||
@@ -1632,6 +1636,8 @@ again:
continue;
}
}
+
+ context->handle->mm.fmt = fmt;
img = switch_img_alloc(NULL, fmt, vframe->width, vframe->height, 1);
@@ -1773,8 +1779,6 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
if (handle->params) {
if ((tmp = switch_event_get_header(handle->params, "av_video_offset"))) {
context->offset = atoi(tmp);
- } else if ((tmp = switch_event_get_header(handle->params, "alpha_mode"))) {
- context->alpha_mode = switch_true(tmp);
}
}
diff --git a/src/mod/applications/mod_conference/conference_file.c b/src/mod/applications/mod_conference/conference_file.c
index 4eab9d9c49..9892d35368 100644
--- a/src/mod/applications/mod_conference/conference_file.c
+++ b/src/mod/applications/mod_conference/conference_file.c
@@ -270,12 +270,18 @@ switch_status_t conference_file_play(conference_obj_t *conference, char *file, u
goto done;
}
+ fnode->layer_lock = -1;
+
if (fnode->fh.params) {
const char *vol = switch_event_get_header(fnode->fh.params, "vol");
const char *position = switch_event_get_header(fnode->fh.params, "position");
const char *canvasstr = switch_event_get_header(fnode->fh.params, "canvas");
const char *loopsstr = switch_event_get_header(fnode->fh.params, "loops");
+ const char *overlay_layer = switch_event_get_header(fnode->fh.params, "overlay_layer");
+ const char *overlay_member = switch_event_get_header(fnode->fh.params, "overlay_member");
+ const char *overlay_role = switch_event_get_header(fnode->fh.params, "overlay_role");
int canvas_id = -1;
+ int layer_id = -1;
if (loopsstr) {
fnode->loops = atoi(loopsstr);
@@ -285,6 +291,35 @@ switch_status_t conference_file_play(conference_obj_t *conference, char *file, u
}
}
+ if (overlay_role) {
+ conference_member_t *member;
+
+ if ((member = conference_member_get_by_role(conference, overlay_role))) {
+ layer_id = member->video_layer_id;
+ switch_thread_rwlock_unlock(member->rwlock);
+ }
+ } else if (overlay_member) {
+ int id = atoi(overlay_member);
+
+ if (id > 0) {
+ conference_member_t *member;
+
+ if ((member = conference_member_get(conference, id))) {
+ layer_id = member->video_layer_id;
+ switch_thread_rwlock_unlock(member->rwlock);
+ }
+ }
+ }
+
+ if (layer_id < 0 && overlay_layer) {
+ layer_id = atoi(overlay_layer);
+ }
+
+ if (layer_id > -1) {
+ fnode->layer_lock = layer_id;
+ fnode->layer_id = layer_id;
+ }
+
if (canvasstr) {
canvas_id = atoi(canvasstr) - 1;
}
diff --git a/src/mod/applications/mod_conference/conference_member.c b/src/mod/applications/mod_conference/conference_member.c
index 62fa008277..03ff78f42d 100644
--- a/src/mod/applications/mod_conference/conference_member.c
+++ b/src/mod/applications/mod_conference/conference_member.c
@@ -435,6 +435,51 @@ conference_member_t *conference_member_get_by_var(conference_obj_t *conference,
return member;
}
+
+/* traverse the conference member list for the specified member with role and return it's pointer */
+conference_member_t *conference_member_get_by_role(conference_obj_t *conference, const char *role_id)
+{
+ conference_member_t *member = NULL;
+
+ switch_assert(conference != NULL);
+ if (zstr(role_id)) {
+ return NULL;
+ }
+
+ switch_mutex_lock(conference->member_mutex);
+ for (member = conference->members; member; member = member->next) {
+
+ if (conference_utils_member_test_flag(member, MFLAG_NOCHANNEL)) {
+ continue;
+ }
+
+ if (!zstr(member->video_role_id) && !strcmp(role_id, member->video_role_id)) {
+ break;
+ }
+ }
+
+ if (member) {
+ if (!conference_utils_member_test_flag(member, MFLAG_INTREE) ||
+ conference_utils_member_test_flag(member, MFLAG_KICKED) ||
+ (member->session && !switch_channel_up(switch_core_session_get_channel(member->session)))) {
+
+ /* member is kicked or hanging up so forget it */
+ member = NULL;
+ }
+ }
+
+ if (member) {
+ if (switch_thread_rwlock_tryrdlock(member->rwlock) != SWITCH_STATUS_SUCCESS) {
+ /* if you cant readlock it's way to late to do anything */
+ member = NULL;
+ }
+ }
+
+ switch_mutex_unlock(conference->member_mutex);
+
+ return member;
+}
+
void conference_member_check_channels(switch_frame_t *frame, conference_member_t *member, switch_bool_t in)
{
if (member->conference->channels != member->read_impl.number_of_channels || conference_utils_member_test_flag(member, MFLAG_POSITIONAL)) {
diff --git a/src/mod/applications/mod_conference/conference_video.c b/src/mod/applications/mod_conference/conference_video.c
index ea5fc919fb..74a938be69 100644
--- a/src/mod/applications/mod_conference/conference_video.c
+++ b/src/mod/applications/mod_conference/conference_video.c
@@ -415,7 +415,6 @@ void conference_video_reset_layer(mcu_layer_t *layer)
{
switch_img_free(&layer->banner_img);
switch_img_free(&layer->logo_img);
- switch_img_free(&layer->logo_text_img);
layer->bugged = 0;
layer->mute_patched = 0;
@@ -435,6 +434,9 @@ void conference_video_reset_layer(mcu_layer_t *layer)
conference_video_clear_layer(layer);
switch_img_free(&layer->cur_img);
+ switch_mutex_lock(layer->overlay_mutex);
+ switch_img_free(&layer->overlay_img);
+ switch_mutex_unlock(layer->overlay_mutex);
}
static void set_pan(int crop_point, int *target_point, int accel_speed, int accel_min, int speed)
@@ -848,13 +850,8 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg,
//printf("SCALE %d,%d %dx%d\n", x_pos, y_pos, img_w, img_h);
switch_img_scale(img, &layer->img, img_w, img_h);
-
- if (layer->img) {
- //switch_img_copy(img, &layer->img);
- switch_img_patch(IMG, layer->img, x_pos + layer->geometry.border, y_pos + layer->geometry.border);
- }
-
-
+
+
if (layer->logo_img) {
int ew = layer->screen_w - (layer->geometry.border * 2), eh = layer->screen_h - (layer->banner_img ? layer->banner_img->d_h : 0) - (layer->geometry.border * 2);
int ex = 0, ey = 0;
@@ -863,18 +860,26 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg,
switch_img_find_position(layer->logo_pos, ew, eh, layer->logo_img->d_w, layer->logo_img->d_h, &ex, &ey);
- switch_img_patch(IMG, layer->logo_img, layer->x_pos + ex + layer->geometry.border, layer->y_pos + ey + layer->geometry.border);
- if (layer->logo_text_img) {
- int tx = 0, ty = 0;
+ switch_img_patch(layer->img, layer->logo_img, ex, ey);
+ //switch_img_patch(IMG, layer->logo_img, layer->x_pos + ex + layer->geometry.border, layer->y_pos + ey + layer->geometry.border);
+ }
- switch_img_fit(&layer->logo_text_img, (ew / 2) + 1, (eh / 2) + 1, SWITCH_FIT_SIZE);
- switch_img_find_position(POS_LEFT_BOT,
- layer->logo_img->d_w, layer->logo_img->d_h, layer->logo_text_img->d_w, layer->logo_text_img->d_h, &tx, &ty);
- switch_img_patch(IMG, layer->logo_text_img, layer->x_pos + ex + tx + layer->geometry.border, layer->y_pos + ey + ty + layer->geometry.border);
+ if (layer->img) {
+ //switch_img_copy(img, &layer->img);
+
+ switch_mutex_lock(layer->overlay_mutex);
+ if (layer->overlay_img) {
+ switch_img_fit(&layer->overlay_img, layer->img->d_w, layer->img->d_h, SWITCH_FIT_SCALE);
+ switch_img_patch(layer->img, layer->overlay_img, 0, 0);
}
+ switch_mutex_unlock(layer->overlay_mutex);
+
+ switch_img_patch(IMG, layer->img, x_pos + layer->geometry.border, y_pos + layer->geometry.border);
}
+
+
layer->last_img_addr = img_addr;
} else {
@@ -883,6 +888,7 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg,
}
switch_mutex_unlock(layer->canvas->mutex);
+
}
void conference_video_set_canvas_bgcolor(mcu_canvas_t *canvas, char *color)
@@ -1060,7 +1066,6 @@ void conference_video_layer_set_logo(conference_member_t *member, mcu_layer_t *l
switch_img_free(&layer->logo_img);
- switch_img_free(&layer->logo_text_img);
if (member->video_logo) {
switch_img_copy(member->video_logo, &layer->logo_img);
@@ -1260,7 +1265,6 @@ void conference_video_layer_set_banner(conference_member_t *member, mcu_layer_t
switch_img_free(&layer->banner_img);
//switch_img_free(&layer->logo_img);
- //switch_img_free(&layer->logo_text_img);
layer->banner_img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, layer->screen_w, font_size * 2, 1);
conference_video_reset_image(layer->banner_img, &bgcolor);
switch_img_txt_handle_render(layer->txthandle, layer->banner_img, font_size / 2, font_size / 2, text, NULL, fg, bg, 0, 0);
@@ -1397,6 +1401,13 @@ void conference_video_init_canvas_layers(conference_obj_t *conference, mcu_canva
switch_mutex_lock(canvas->mutex);
canvas->layout_floor_id = -1;
+ for (i = 0; i < MCU_MAX_LAYERS; i++) {
+ mcu_layer_t *layer = &canvas->layers[i];
+ if (!layer->overlay_mutex) {
+ switch_mutex_init(&layer->overlay_mutex, SWITCH_MUTEX_NESTED, canvas->pool);
+ }
+ }
+
if (canvas->vlayout && canvas->vlayout->transition_out) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Play transition out [%s]\n", canvas->vlayout->transition_out);
conference_file_play(conference, canvas->vlayout->transition_out, 0, NULL, 0);
@@ -1473,6 +1484,7 @@ void conference_video_init_canvas_layers(conference_obj_t *conference, mcu_canva
for (i = 0; i < MCU_MAX_LAYERS; i++) {
mcu_layer_t *layer = &canvas->layers[i];
+
if (layer->member) {
//conference_video_detach_video_layer(layer->member);
conference_video_clear_managed_kps(layer->member);
@@ -1871,7 +1883,9 @@ void conference_video_canvas_del_fnode_layer(conference_obj_t *conference, confe
fnode->layer_id = -1;
fnode->canvas_id = -1;
xlayer->fnode = NULL;
- conference_video_reset_layer(xlayer);
+ if (fnode->layer_lock < 0) {
+ conference_video_reset_layer(xlayer);
+ }
}
switch_mutex_unlock(canvas->mutex);
}
@@ -1883,6 +1897,14 @@ void conference_video_canvas_set_fnode_layer(mcu_canvas_t *canvas, conference_fi
switch_mutex_lock(canvas->mutex);
+ if (fnode->layer_lock > -1) {
+ layer = &canvas->layers[fnode->layer_lock];
+ layer->fnode = fnode;
+ fnode->layer_id = fnode->layer_lock;
+ fnode->canvas_id = canvas->canvas_id;
+ goto end;
+ }
+
if (idx == -1) {
int i;
@@ -2312,8 +2334,22 @@ void conference_video_patch_fnode(mcu_canvas_t *canvas, conference_file_node_t *
switch_status_t status = switch_core_file_read_video(&fnode->fh, &file_frame, SVR_FLUSH);
if (status == SWITCH_STATUS_SUCCESS) {
- switch_img_free(&layer->cur_img);
- layer->cur_img = file_frame.img;
+ if (fnode->layer_lock > -1 && layer->member_id > 0) {
+ switch_mutex_lock(layer->overlay_mutex);
+ switch_img_free(&layer->overlay_img);
+ layer->overlay_img = file_frame.img;
+ switch_mutex_unlock(layer->overlay_mutex);
+ } else {
+ switch_img_free(&layer->cur_img);
+ if (file_frame.img && file_frame.img->fmt != SWITCH_IMG_FMT_I420) {
+ switch_image_t *tmp = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, file_frame.img->d_w, file_frame.img->d_h, 1);
+ switch_img_copy(file_frame.img, &tmp);
+ switch_img_free(&file_frame.img);
+ file_frame.img = tmp;
+ }
+ layer->cur_img = file_frame.img;
+ }
+
layer->tagged = 1;
} else if (status == SWITCH_STATUS_IGNORE) {
if (canvas && fnode->layer_id > -1 ) {
@@ -2351,7 +2387,11 @@ void conference_video_fnode_check(conference_file_node_t *fnode, int canvas_id)
if (full_screen) {
canvas->play_file = 1;
- canvas->conference->playing_video_file = 1;
+ if (fnode->fh.mm.fmt == SWITCH_IMG_FMT_ARGB) {
+ canvas->conference->overlay_video_file = 1;
+ } else {
+ canvas->conference->playing_video_file = 1;
+ }
} else {
conference_video_canvas_set_fnode_layer(canvas, fnode, -1);
@@ -3605,27 +3645,55 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr
write_img = canvas->img;
timestamp = canvas->timer.samplecount;
- if (conference->playing_video_file) {
- if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) {
- switch_img_free(&file_img);
-
- if (canvas->play_file) {
- canvas->send_keyframe = 1;
- canvas->play_file = 0;
- }
+ switch_mutex_lock(conference->file_mutex);
+ if (conference->fnode && switch_test_flag(&conference->fnode->fh, SWITCH_FILE_OPEN)) {
+ if (conference->overlay_video_file) {
+ if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) {
- switch_img_free(&file_img);
- switch_img_fit(&write_frame.img, canvas->img->d_w, canvas->img->d_h, SWITCH_FIT_SIZE);
- file_img = write_img = write_frame.img;
+ if (canvas->play_file) {
+ canvas->send_keyframe = 1;
+ canvas->play_file = 0;
+ }
+
+ switch_img_free(&file_img);
+ switch_img_fit(&write_frame.img, canvas->img->d_w, canvas->img->d_h, SWITCH_FIT_SIZE);
+ file_img = write_frame.img;
+
+ if (file_img->fmt == SWITCH_IMG_FMT_ARGB) {
+ switch_img_patch(write_img, file_img, 0, 0);
+ switch_img_free(&file_img);
+ switch_img_copy(write_img, &file_img);
+ } else {
+ write_img = file_img;
+ }
+
+ //switch_core_timer_sync(&canvas->timer);
+ timestamp = canvas->timer.samplecount;
+ } else if (file_img) {
+ write_img = file_img;
+ }
+ } else if (conference->playing_video_file) {
+ if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) {
+
+ if (canvas->play_file) {
+ canvas->send_keyframe = 1;
+ canvas->play_file = 0;
+ }
+
+ switch_img_free(&file_img);
+ switch_img_fit(&write_frame.img, canvas->img->d_w, canvas->img->d_h, SWITCH_FIT_SIZE);
+ file_img = write_img = write_frame.img;
- //switch_core_timer_sync(&canvas->timer);
- timestamp = canvas->timer.samplecount;
+ //switch_core_timer_sync(&canvas->timer);
+ timestamp = canvas->timer.samplecount;
+ } else if (file_img) {
+ write_img = file_img;
+ }
} else if (file_img) {
- write_img = file_img;
+ switch_img_free(&file_img);
}
- } else if (file_img) {
- switch_img_free(&file_img);
}
+ switch_mutex_unlock(conference->file_mutex);
write_frame.img = write_img;
@@ -3734,11 +3802,11 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr
switch_mutex_lock(canvas->mutex);
switch_img_free(&layer->cur_img);
+ switch_img_free(&layer->overlay_img);
switch_img_free(&layer->img);
layer->banner_patched = 0;
switch_img_free(&layer->banner_img);
switch_img_free(&layer->logo_img);
- switch_img_free(&layer->logo_text_img);
switch_img_free(&layer->mute_img);
switch_mutex_unlock(canvas->mutex);
@@ -4081,11 +4149,11 @@ void *SWITCH_THREAD_FUNC conference_video_super_muxing_thread_run(switch_thread_
switch_mutex_lock(canvas->mutex);
switch_img_free(&layer->cur_img);
+ switch_img_free(&layer->overlay_img);
switch_img_free(&layer->img);
layer->banner_patched = 0;
switch_img_free(&layer->banner_img);
switch_img_free(&layer->logo_img);
- switch_img_free(&layer->logo_text_img);
switch_img_free(&layer->mute_img);
switch_mutex_unlock(canvas->mutex);
diff --git a/src/mod/applications/mod_conference/mod_conference.h b/src/mod/applications/mod_conference/mod_conference.h
index 5f6af27566..1b3fcc2623 100644
--- a/src/mod/applications/mod_conference/mod_conference.h
+++ b/src/mod/applications/mod_conference/mod_conference.h
@@ -389,6 +389,7 @@ typedef struct conference_file_node {
char *res_id;
int loops;
int new_fnode;
+ int layer_lock;
} conference_file_node_t;
typedef enum {
@@ -476,9 +477,9 @@ typedef struct mcu_layer_s {
switch_size_t last_img_addr;
switch_image_t *img;
switch_image_t *cur_img;
+ switch_image_t *overlay_img;
switch_image_t *banner_img;
switch_image_t *logo_img;
- switch_image_t *logo_text_img;
switch_image_t *mute_img;
switch_img_txt_handle_t *txthandle;
conference_file_node_t *fnode;
@@ -494,6 +495,7 @@ typedef struct mcu_layer_s {
switch_frame_geometry_t pan_geometry;
switch_frame_geometry_t manual_geometry;
mcu_layer_cam_opts_t cam_opts;
+ switch_mutex_t *overlay_mutex;
} mcu_layer_t;
typedef struct video_layout_s {
@@ -717,6 +719,7 @@ typedef struct conference_obj {
switch_hash_t *layout_group_hash;
struct conference_fps video_fps;
int playing_video_file;
+ int overlay_video_file;
int recording_members;
uint32_t video_floor_packets;
video_layout_t *new_personal_vlayout;
@@ -1073,7 +1076,7 @@ void conference_fnode_check_status(conference_file_node_t *fnode, switch_stream_
conference_relationship_t *conference_member_add_relationship(conference_member_t *member, uint32_t id);
conference_member_t *conference_member_get(conference_obj_t *conference, uint32_t id);
conference_member_t *conference_member_get_by_var(conference_obj_t *conference, const char *var, const char *val);
-
+conference_member_t *conference_member_get_by_role(conference_obj_t *conference, const char *role_id);
switch_status_t conference_member_del_relationship(conference_member_t *member, uint32_t id);
switch_status_t conference_member_add(conference_obj_t *conference, conference_member_t *member);
switch_status_t conference_member_del(conference_obj_t *conference, conference_member_t *member);