kenmirrors
/
freeswitch
mirror of https://github.com/signalwire/freeswitch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

FS-10050: [core] chromakey

This commit is contained in:
Anthony Minessale 2017-03-02 18:04:25 -06:00
parent 05632bb057
commit b733e8d974
4 changed files with 346 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/include/switch_core_video.h
View File

@ -44,6 +44,16 @@
SWITCH_BEGIN_EXTERN_C
#define CHROMAKEY_MAX_MASK 25
typedef enum {
SWITCH_SHADE_NONE = 0,
SWITCH_SHADE_RED,
SWITCH_SHADE_GREEN,
SWITCH_SHADE_BLUE,
SWITCH_SHADE_AUTO
} switch_shade_t;
typedef enum {
POS_LEFT_TOP = 0,
POS_LEFT_MID,
@ -420,8 +430,16 @@ SWITCH_DECLARE(switch_status_t) switch_I420_copy2(uint8_t *src_planes[], int src
/*!\brief chromakey an img, img must be RGBA and return modified img */
SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold);
SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_image_t *cache_img, switch_rgb_color_t *mask, int *thresholds, int count);
SWITCH_DECLARE(switch_status_t) switch_chromakey_clear_colors(switch_chromakey_t *ck);
SWITCH_DECLARE(switch_status_t) switch_chromakey_autocolor(switch_chromakey_t *ck, switch_shade_t autocolor, uint32_t threshold);
SWITCH_DECLARE(switch_status_t) switch_chromakey_add_color(switch_chromakey_t *ck, switch_rgb_color_t *color, uint32_t threshold);
SWITCH_DECLARE(switch_status_t) switch_chromakey_destroy(switch_chromakey_t **ckP);
SWITCH_DECLARE(switch_status_t) switch_chromakey_create(switch_chromakey_t **ckP);
SWITCH_DECLARE(void) switch_chromakey_set_default_threshold(switch_chromakey_t *ck, uint32_t threshold);
SWITCH_DECLARE(void) switch_chromakey_process(switch_chromakey_t *ck, switch_image_t *img);
SWITCH_DECLARE(switch_image_t *) switch_chromakey_cache_image(switch_chromakey_t *ck);
SWITCH_DECLARE(switch_shade_t) switch_chromakey_str2shade(switch_chromakey_t *ck, const char *shade_name);
SWITCH_END_EXTERN_C
#endif

3
src/include/switch_types.h
View File

@ -2666,6 +2666,9 @@ struct switch_rtp_text_factory_s;
typedef struct switch_rtp_text_factory_s switch_rtp_text_factory_t;
typedef struct switch_agc_s switch_agc_t;
struct switch_chromakey_s;
typedef struct switch_chromakey_s switch_chromakey_t;
SWITCH_END_EXTERN_C
#endif
/* For Emacs:

47
src/mod/applications/mod_video_filter/mod_video_filter.c
View File

@ -38,13 +38,11 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_video_filter_load);
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_video_filter_shutdown);
SWITCH_MODULE_DEFINITION(mod_video_filter, mod_video_filter_load, mod_video_filter_shutdown, NULL);
#define MAX_MASK 25
typedef struct chromakey_context_s {
int threshold;
switch_image_t *bgimg;
switch_image_t *bgimg_scaled;
switch_image_t *last_img;
switch_image_t *imgfg;
switch_image_t *imgbg;
void *data;
@ -53,13 +51,11 @@ typedef struct chromakey_context_s {
switch_size_t patch_datalen;
switch_file_handle_t vfh;
switch_rgb_color_t bgcolor;
switch_rgb_color_t mask[MAX_MASK];
int thresholds[MAX_MASK];
int mask_len;
switch_core_session_t *session;
switch_mutex_t *command_mutex;
int patch;
int mod;
switch_chromakey_t *ck;
} chromakey_context_t;
static void init_context(chromakey_context_t *context)
@ -67,6 +63,7 @@ static void init_context(chromakey_context_t *context)
switch_color_set_rgb(&context->bgcolor, "#000000");
context->threshold = 300;
switch_mutex_init(&context->command_mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(context->session));
switch_chromakey_create(&context->ck);
}
static void uninit_context(chromakey_context_t *context)
@ -80,9 +77,9 @@ static void uninit_context(chromakey_context_t *context)
memset(&context->vfh, 0, sizeof(context->vfh));
}
switch_img_free(&context->last_img);
switch_safe_free(context->data);
switch_safe_free(context->patch_data);
switch_chromakey_destroy(&context->ck);
}
static void parse_params(chromakey_context_t *context, int start, int argc, char **argv, const char **function, switch_media_bug_flag_t *flags)
@ -96,29 +93,30 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char
if (n > 0 && argv[i]) { // color
int j = 0;
char *list[MAX_MASK];
char *list[CHROMAKEY_MAX_MASK];
int list_argc;
list_argc = switch_split(argv[i], ':', list);
context->mask_len = 0;
memset(context->thresholds, 0, sizeof(context->thresholds[0]) * MAX_MASK);
switch_chromakey_clear_colors(context->ck);
for (j = 0; j < list_argc; j++) {
char *p;
int thresh = 0;
switch_rgb_color_t color = { 0 };
if ((p = strchr(list[j], '+'))) {
*p++ = '\0';
thresh = atoi(p);
if (thresh < 0) thresh = 0;
}
switch_color_set_rgb(&context->mask[j], list[j]);
if (thresh) {
context->thresholds[j] = thresh*thresh;
if (*list[j] == '#') {
switch_color_set_rgb(&color, list[j]);
switch_chromakey_add_color(context->ck, &color, thresh);
} else {
switch_chromakey_autocolor(context->ck, switch_chromakey_str2shade(context->ck, list[j]), thresh);
}
context->mask_len++;
}
}
@ -126,14 +124,9 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char
if (n > 1 && argv[i]) { // thresh
int thresh = atoi(argv[i]);
int j;
if (thresh > 0) {
context->threshold = thresh * thresh;
for (j = 0; j < context->mask_len; j++) {
if (!context->thresholds[j]) context->thresholds[j] = context->threshold;
}
switch_chromakey_set_default_threshold(context->ck, thresh);
}
}
@ -237,7 +230,12 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi
context->mod = 0;
if (switch_mutex_trylock(context->command_mutex) != SWITCH_STATUS_SUCCESS) {
switch_img_patch(frame->img, context->last_img, 0, 0);
switch_image_t *last_img = switch_chromakey_cache_image(context->ck);
if (last_img) {
switch_img_patch(frame->img, last_img, 0, 0);
}
return SWITCH_STATUS_SUCCESS;
}
@ -256,10 +254,7 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi
img = switch_img_wrap(NULL, SWITCH_IMG_FMT_ARGB, frame->img->d_w, frame->img->d_h, 1, context->data);
switch_assert(img);
switch_img_chromakey_multi(img, context->last_img, context->mask, context->thresholds, context->mask_len);
switch_img_free(&context->last_img);
switch_img_copy(img, &context->last_img);
switch_chromakey_process(context->ck, img);
if (context->bgimg) {
switch_image_t *tmp = NULL;

321
src/switch_core_video.c
View File

@ -87,7 +87,7 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color
* \param[in] count number of colors in list
* \param[in] threshold hint of target threshold to stop processing list
*/
static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds);
static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, uint32_t *thresholds);
/*!\brief Draw a pixel on an image
*
@ -334,7 +334,7 @@ SWITCH_DECLARE(void) switch_img_patch_rgb(switch_image_t *IMG, switch_image_t *i
tmp_a = ((RGB->a * (255 - alpha)) >> 8) + ((rgb->a * alpha) >> 8);
RGB->a = RGB->a > tmp_a ? RGB->a : tmp_a;
}
RGB->r = ((RGB->r * (255 - alpha)) >> 8) + ((rgb->r * alpha) >> 8);
RGB->g = ((RGB->g * (255 - alpha)) >> 8) + ((rgb->g * alpha) >> 8);
RGB->b = ((RGB->b * (255 - alpha)) >> 8) + ((rgb->b * alpha) >> 8);
@ -827,7 +827,7 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color
}
static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds)
static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, uint32_t *thresholds)
{
int x = 0, hits = 0;
@ -844,38 +844,212 @@ static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb
}
struct switch_chromakey_s {
switch_image_t *cache_img;
switch_rgb_color_t mask[CHROMAKEY_MAX_MASK];
uint32_t thresholds[CHROMAKEY_MAX_MASK];
int mask_len;
switch_shade_t autocolor;
uint32_t dft_thresh;
uint32_t dft_thresh_squared;
uint32_t rr;
uint32_t gg;
uint32_t bb;
uint32_t color_count;
switch_rgb_color_t auto_color;
int no_cache;
};
SWITCH_DECLARE(switch_shade_t) switch_chromakey_str2shade(switch_chromakey_t *ck, const char *shade_name)
{
switch_shade_t shade = SWITCH_SHADE_NONE;
if (!strcasecmp(shade_name, "red")) {
shade = SWITCH_SHADE_RED;
} else if (!strcasecmp(shade_name, "green")) {
shade = SWITCH_SHADE_GREEN;
} else if (!strcasecmp(shade_name, "blue")) {
shade = SWITCH_SHADE_BLUE;
} else if (!strcasecmp(shade_name, "auto")) {
shade = SWITCH_SHADE_AUTO;
}
return shade;
}
SWITCH_DECLARE(void) switch_chromakey_set_default_threshold(switch_chromakey_t *ck, uint32_t threshold)
{
int i;
ck->dft_thresh = threshold;
ck->dft_thresh_squared = threshold * threshold;
for (i = 0; i < ck->mask_len; i++) {
if (!ck->thresholds[i]) ck->thresholds[i] = ck->dft_thresh_squared;
}
}
SWITCH_DECLARE(switch_status_t) switch_chromakey_clear_colors(switch_chromakey_t *ck)
{
switch_assert(ck);
ck->autocolor = SWITCH_SHADE_NONE;
ck->mask_len = 0;
memset(ck->mask, 0, sizeof(ck->mask[0]) * CHROMAKEY_MAX_MASK);
memset(ck->thresholds, 0, sizeof(ck->thresholds[0]) * CHROMAKEY_MAX_MASK);
ck->no_cache = 1;
return SWITCH_STATUS_SUCCESS;
}
SWITCH_DECLARE(switch_status_t) switch_chromakey_autocolor(switch_chromakey_t *ck, switch_shade_t autocolor, uint32_t threshold)
{
switch_assert(ck);
switch_chromakey_clear_colors(ck);
ck->autocolor = autocolor;
ck->dft_thresh = threshold;
ck->dft_thresh_squared = threshold * threshold;
switch_img_free(&ck->cache_img);
ck->no_cache = 90;
memset(&ck->auto_color, 0, sizeof(ck->auto_color));
return SWITCH_STATUS_SUCCESS;
}
SWITCH_DECLARE(switch_status_t) switch_chromakey_add_color(switch_chromakey_t *ck, switch_rgb_color_t *color, uint32_t threshold)
{
switch_assert(ck);
if (ck->mask_len == CHROMAKEY_MAX_MASK) {
return SWITCH_STATUS_FALSE;
}
ck->mask[ck->mask_len] = *color;
ck->thresholds[ck->mask_len] = threshold * threshold;
ck->mask_len++;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Adding color %d:%d:%d #%.2x%.2x%.2x\n",
ck->auto_color.r, ck->auto_color.g, ck->auto_color.b, ck->auto_color.r, ck->auto_color.g, ck->auto_color.b);
return SWITCH_STATUS_SUCCESS;
}
SWITCH_DECLARE(switch_status_t) switch_chromakey_destroy(switch_chromakey_t **ckP)
{
switch_chromakey_t *ck;
switch_assert(ckP);
ck = *ckP;
*ckP = NULL;
if (ck) {
switch_img_free(&ck->cache_img);
free(ck);
}
return SWITCH_STATUS_SUCCESS;
}
SWITCH_DECLARE(switch_status_t) switch_chromakey_create(switch_chromakey_t **ckP)
{
switch_chromakey_t *ck;
switch_assert(ckP);
switch_zmalloc(ck, sizeof(*ck));
*ckP = ck;
return SWITCH_STATUS_SUCCESS;
}
SWITCH_DECLARE(switch_image_t *) switch_chromakey_cache_image(switch_chromakey_t *ck)
{
switch_assert(ck);
return ck->cache_img;
}
static inline int get_max(switch_rgb_color_t *c1)
{
if (c1->r > c1->g && c1->r > c1->b) {
return 1;
}
if (c1->g > c1->r && c1->g > c1->b) {
return 2;
}
if (c1->b > c1->r && c1->b > c1->g) {
return 3;
}
return 0;
}
static inline int switch_color_dom_cmp(switch_rgb_color_t *c1, switch_rgb_color_t *c2)
{
int c1_max = get_max(c1);
int c2_max = get_max(c2);
if (c1_max && c1_max == c2_max) return 1;
return 0;
}
static inline int switch_color_distance_literal(switch_rgb_color_t *c1, switch_rgb_color_t *c2, int distance)
{
int r = abs(c1->r - c2->r);
int g = abs(c1->g - c2->g);
int b = abs(c1->b - c2->b);
if (r < distance && g < distance && b < distance) return 1;
return 0;
}
static inline int switch_color_distance_cheap(switch_rgb_color_t *c1, switch_rgb_color_t *c2)
{
int r = c1->r - c2->r;
int g = c1->g - c2->g;
int b = c1->b - c2->b;
int r = abs(c1->r - c2->r);
int g = abs(c1->g - c2->g);
int b = abs(c1->b - c2->b);
if (!r && !g && !b) return 0;
if (r < 5 && g < 5 && b < 5) return 0;
return (3*abs(r)) + (4*abs(g)) + (3*abs(b));
return (3*r) + (4*g) + (3*b);
}
SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_image_t *cache_img, switch_rgb_color_t *mask, int *thresholds, int count)
SWITCH_DECLARE(void) switch_chromakey_process(switch_chromakey_t *ck, switch_image_t *img)
{
uint8_t *pixel, *last_pixel = NULL, *cache_pixel = NULL, *end_pixel = NULL;
int last_hits = 0;
switch_image_t *cache_img;
int same = 0;
int same_same = 0;
#ifdef DEBUG_CHROMA
int other_img_cached = 0, color_cached = 0, checked = 0, hit_total = 0, total_pixel = 0, delta_hits = 0;
#endif
switch_assert(ck);
switch_assert(img);
if (img->fmt != SWITCH_IMG_FMT_ARGB) return;
pixel = img->planes[SWITCH_PLANE_PACKED];
cache_img = ck->cache_img;
ck->cache_img = NULL;
if (cache_img && (cache_img->d_w != img->d_w || cache_img->d_h != img->d_h)) {
cache_img = NULL;
switch_img_free(&cache_img);
}
if (cache_img) {
@ -884,16 +1058,23 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag
end_pixel = (img->planes[SWITCH_PLANE_PACKED] + img->d_w * img->d_h * 4);
if (ck->autocolor) {
ck->color_count = 0;
ck->rr = ck->gg = ck->bb = 0;
}
for (; pixel < end_pixel; pixel += 4) {
switch_rgb_color_t *color = (switch_rgb_color_t *)pixel;
switch_rgb_color_t *last_color = (switch_rgb_color_t *)last_pixel;
int hits = 0;
#ifdef DEBUG_CHROMA
total_pixel++;
#endif
if (cache_img && cache_pixel) {
if (!ck->no_cache && cache_img && cache_pixel) {
switch_rgb_color_t *cache_color = (switch_rgb_color_t *)cache_pixel;
if (switch_color_distance_cheap(color, cache_color) < 5) {
@ -902,28 +1083,92 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag
#endif
*pixel = *cache_pixel;
goto end;
}
}
}
if (last_color && switch_color_distance_cheap(color, last_color) < 5) {
if (last_color) {
if (switch_color_distance_cheap(color, last_color) < 5) {
hits = last_hits;
hits = last_hits;
#ifdef DEBUG_CHROMA
color_cached++;
color_cached++;
#endif
same++;
} else {
same = 0;
}
}
if (!hits) {
hits = switch_color_distance_multi(color, mask, count, thresholds);
if (ck->autocolor) {
int dom, a, b;
switch(ck->autocolor) {
case SWITCH_SHADE_RED:
dom = color->r;
a = color->g;
b = color->b;
break;
case SWITCH_SHADE_GREEN:
dom = color->g;
a = color->r;
b = color->b;
break;
case SWITCH_SHADE_BLUE:
dom = color->b;
a = color->r;
b = color->g;
break;
default:
dom = 0;
a = 0;
b = 0;
break;
}
if (ck->autocolor != SWITCH_SHADE_AUTO) {
//printf("WTF %d\n", ck->dft_thresh);
int tol = ck->dft_thresh;
int a_tol = tol/6;
int b_tol = tol/6;
if (dom > a && dom > b && dom > tol) {
if (dom - a > a_tol && dom - b > b_tol) {
hits = 1;
}
}
}
}
if (!hits && ck->mask_len) {
hits = switch_color_distance_multi(color, ck->mask, ck->mask_len, ck->thresholds);
}
#ifdef DEBUG_CHROMA
checked++;
#endif
}
end:
if (same > 100 && last_color && switch_color_dom_cmp(color, last_color)) {
same_same++;
} else {
same_same = 0;
}
if (!hits && ck->autocolor == SWITCH_SHADE_AUTO && (same > 300 || same_same > 50) && ck->no_cache) {
ck->color_count++;
ck->rr += color->r;
ck->gg += color->g;
ck->bb += color->b;
}
if (cache_pixel) {
cache_pixel += 4;
}
@ -938,10 +1183,48 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag
last_pixel = pixel;
last_hits = hits;
}
if (ck->color_count > 1000) {
switch_rgb_color_t *last_color = NULL;
int skip = 0;
ck->auto_color.r = ck->rr / ck->color_count;
ck->auto_color.g = ck->gg / ck->color_count;
ck->auto_color.b = ck->bb / ck->color_count;
if (ck->mask_len) {
int i = 0;
for (i = 0; i < ck->mask_len; i++) {
last_color = &ck->mask[i];
if (switch_color_distance_literal(&ck->auto_color, last_color, 10) || !switch_color_dom_cmp(&ck->auto_color, last_color)) {
skip = 1;
break;
}
}
}
if (!ck->mask_len || !skip) {
switch_chromakey_add_color(ck, &ck->auto_color, ck->dft_thresh);
}
}
if (ck->no_cache > 0 && ck->mask_len) {
ck->no_cache--;
}
#ifdef DEBUG_CHROMA
printf("total %d: other img cache %d color cache %d Checked %d Hit Total %d Delta hits: %d\n", total_pixel, other_img_cached, color_cached, checked, hit_total, delta_hits);
#endif
if (!ck->no_cache) {
switch_img_copy(img, &ck->cache_img);
}
switch_img_free(&cache_img);
return;
}