diff --git a/src/include/switch_core_video.h b/src/include/switch_core_video.h index 676e909bbf..4f26203485 100644 --- a/src/include/switch_core_video.h +++ b/src/include/switch_core_video.h @@ -44,6 +44,16 @@ SWITCH_BEGIN_EXTERN_C +#define CHROMAKEY_MAX_MASK 25 + +typedef enum { + SWITCH_SHADE_NONE = 0, + SWITCH_SHADE_RED, + SWITCH_SHADE_GREEN, + SWITCH_SHADE_BLUE, + SWITCH_SHADE_AUTO +} switch_shade_t; + typedef enum { POS_LEFT_TOP = 0, POS_LEFT_MID, @@ -420,8 +430,16 @@ SWITCH_DECLARE(switch_status_t) switch_I420_copy2(uint8_t *src_planes[], int src /*!\brief chromakey an img, img must be RGBA and return modified img */ SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold); -SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_image_t *cache_img, switch_rgb_color_t *mask, int *thresholds, int count); - +SWITCH_DECLARE(switch_status_t) switch_chromakey_clear_colors(switch_chromakey_t *ck); + +SWITCH_DECLARE(switch_status_t) switch_chromakey_autocolor(switch_chromakey_t *ck, switch_shade_t autocolor, uint32_t threshold); +SWITCH_DECLARE(switch_status_t) switch_chromakey_add_color(switch_chromakey_t *ck, switch_rgb_color_t *color, uint32_t threshold); +SWITCH_DECLARE(switch_status_t) switch_chromakey_destroy(switch_chromakey_t **ckP); +SWITCH_DECLARE(switch_status_t) switch_chromakey_create(switch_chromakey_t **ckP); +SWITCH_DECLARE(void) switch_chromakey_set_default_threshold(switch_chromakey_t *ck, uint32_t threshold); +SWITCH_DECLARE(void) switch_chromakey_process(switch_chromakey_t *ck, switch_image_t *img); +SWITCH_DECLARE(switch_image_t *) switch_chromakey_cache_image(switch_chromakey_t *ck); +SWITCH_DECLARE(switch_shade_t) switch_chromakey_str2shade(switch_chromakey_t *ck, const char *shade_name); SWITCH_END_EXTERN_C #endif diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 949877d0bb..9f86622a63 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -2666,6 +2666,9 @@ struct switch_rtp_text_factory_s; typedef struct switch_rtp_text_factory_s switch_rtp_text_factory_t; typedef struct switch_agc_s switch_agc_t; +struct switch_chromakey_s; +typedef struct switch_chromakey_s switch_chromakey_t; + SWITCH_END_EXTERN_C #endif /* For Emacs: diff --git a/src/mod/applications/mod_video_filter/mod_video_filter.c b/src/mod/applications/mod_video_filter/mod_video_filter.c index e855bd1f79..9b860347f9 100644 --- a/src/mod/applications/mod_video_filter/mod_video_filter.c +++ b/src/mod/applications/mod_video_filter/mod_video_filter.c @@ -38,13 +38,11 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_video_filter_load); SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_video_filter_shutdown); SWITCH_MODULE_DEFINITION(mod_video_filter, mod_video_filter_load, mod_video_filter_shutdown, NULL); -#define MAX_MASK 25 typedef struct chromakey_context_s { int threshold; switch_image_t *bgimg; switch_image_t *bgimg_scaled; - switch_image_t *last_img; switch_image_t *imgfg; switch_image_t *imgbg; void *data; @@ -53,13 +51,11 @@ typedef struct chromakey_context_s { switch_size_t patch_datalen; switch_file_handle_t vfh; switch_rgb_color_t bgcolor; - switch_rgb_color_t mask[MAX_MASK]; - int thresholds[MAX_MASK]; - int mask_len; switch_core_session_t *session; switch_mutex_t *command_mutex; int patch; int mod; + switch_chromakey_t *ck; } chromakey_context_t; static void init_context(chromakey_context_t *context) @@ -67,6 +63,7 @@ static void init_context(chromakey_context_t *context) switch_color_set_rgb(&context->bgcolor, "#000000"); context->threshold = 300; switch_mutex_init(&context->command_mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(context->session)); + switch_chromakey_create(&context->ck); } static void uninit_context(chromakey_context_t *context) @@ -80,9 +77,9 @@ static void uninit_context(chromakey_context_t *context) memset(&context->vfh, 0, sizeof(context->vfh)); } - switch_img_free(&context->last_img); switch_safe_free(context->data); switch_safe_free(context->patch_data); + switch_chromakey_destroy(&context->ck); } static void parse_params(chromakey_context_t *context, int start, int argc, char **argv, const char **function, switch_media_bug_flag_t *flags) @@ -96,29 +93,30 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char if (n > 0 && argv[i]) { // color int j = 0; - char *list[MAX_MASK]; + char *list[CHROMAKEY_MAX_MASK]; int list_argc; list_argc = switch_split(argv[i], ':', list); - context->mask_len = 0; - memset(context->thresholds, 0, sizeof(context->thresholds[0]) * MAX_MASK); + switch_chromakey_clear_colors(context->ck); for (j = 0; j < list_argc; j++) { char *p; int thresh = 0; - + switch_rgb_color_t color = { 0 }; + if ((p = strchr(list[j], '+'))) { *p++ = '\0'; thresh = atoi(p); if (thresh < 0) thresh = 0; } - - switch_color_set_rgb(&context->mask[j], list[j]); - if (thresh) { - context->thresholds[j] = thresh*thresh; + + if (*list[j] == '#') { + switch_color_set_rgb(&color, list[j]); + switch_chromakey_add_color(context->ck, &color, thresh); + } else { + switch_chromakey_autocolor(context->ck, switch_chromakey_str2shade(context->ck, list[j]), thresh); } - context->mask_len++; } } @@ -126,14 +124,9 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char if (n > 1 && argv[i]) { // thresh int thresh = atoi(argv[i]); - int j; if (thresh > 0) { - context->threshold = thresh * thresh; - - for (j = 0; j < context->mask_len; j++) { - if (!context->thresholds[j]) context->thresholds[j] = context->threshold; - } + switch_chromakey_set_default_threshold(context->ck, thresh); } } @@ -237,7 +230,12 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi context->mod = 0; if (switch_mutex_trylock(context->command_mutex) != SWITCH_STATUS_SUCCESS) { - switch_img_patch(frame->img, context->last_img, 0, 0); + switch_image_t *last_img = switch_chromakey_cache_image(context->ck); + + if (last_img) { + switch_img_patch(frame->img, last_img, 0, 0); + } + return SWITCH_STATUS_SUCCESS; } @@ -256,10 +254,7 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi img = switch_img_wrap(NULL, SWITCH_IMG_FMT_ARGB, frame->img->d_w, frame->img->d_h, 1, context->data); switch_assert(img); - switch_img_chromakey_multi(img, context->last_img, context->mask, context->thresholds, context->mask_len); - - switch_img_free(&context->last_img); - switch_img_copy(img, &context->last_img); + switch_chromakey_process(context->ck, img); if (context->bgimg) { switch_image_t *tmp = NULL; diff --git a/src/switch_core_video.c b/src/switch_core_video.c index 7127e518d8..d1c1d75874 100644 --- a/src/switch_core_video.c +++ b/src/switch_core_video.c @@ -87,7 +87,7 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color * \param[in] count number of colors in list * \param[in] threshold hint of target threshold to stop processing list */ -static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds); +static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, uint32_t *thresholds); /*!\brief Draw a pixel on an image * @@ -334,7 +334,7 @@ SWITCH_DECLARE(void) switch_img_patch_rgb(switch_image_t *IMG, switch_image_t *i tmp_a = ((RGB->a * (255 - alpha)) >> 8) + ((rgb->a * alpha) >> 8); RGB->a = RGB->a > tmp_a ? RGB->a : tmp_a; } - + RGB->r = ((RGB->r * (255 - alpha)) >> 8) + ((rgb->r * alpha) >> 8); RGB->g = ((RGB->g * (255 - alpha)) >> 8) + ((rgb->g * alpha) >> 8); RGB->b = ((RGB->b * (255 - alpha)) >> 8) + ((rgb->b * alpha) >> 8); @@ -827,7 +827,7 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color } -static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds) +static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, uint32_t *thresholds) { int x = 0, hits = 0; @@ -844,38 +844,212 @@ static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb } +struct switch_chromakey_s { + switch_image_t *cache_img; + switch_rgb_color_t mask[CHROMAKEY_MAX_MASK]; + uint32_t thresholds[CHROMAKEY_MAX_MASK]; + int mask_len; + switch_shade_t autocolor; + uint32_t dft_thresh; + uint32_t dft_thresh_squared; + uint32_t rr; + uint32_t gg; + uint32_t bb; + uint32_t color_count; + + switch_rgb_color_t auto_color; + int no_cache; +}; +SWITCH_DECLARE(switch_shade_t) switch_chromakey_str2shade(switch_chromakey_t *ck, const char *shade_name) +{ + switch_shade_t shade = SWITCH_SHADE_NONE; + + if (!strcasecmp(shade_name, "red")) { + shade = SWITCH_SHADE_RED; + } else if (!strcasecmp(shade_name, "green")) { + shade = SWITCH_SHADE_GREEN; + } else if (!strcasecmp(shade_name, "blue")) { + shade = SWITCH_SHADE_BLUE; + } else if (!strcasecmp(shade_name, "auto")) { + shade = SWITCH_SHADE_AUTO; + } + + return shade; +} + +SWITCH_DECLARE(void) switch_chromakey_set_default_threshold(switch_chromakey_t *ck, uint32_t threshold) +{ + int i; + + ck->dft_thresh = threshold; + ck->dft_thresh_squared = threshold * threshold; + + for (i = 0; i < ck->mask_len; i++) { + if (!ck->thresholds[i]) ck->thresholds[i] = ck->dft_thresh_squared; + } +} + +SWITCH_DECLARE(switch_status_t) switch_chromakey_clear_colors(switch_chromakey_t *ck) +{ + switch_assert(ck); + + ck->autocolor = SWITCH_SHADE_NONE; + ck->mask_len = 0; + memset(ck->mask, 0, sizeof(ck->mask[0]) * CHROMAKEY_MAX_MASK); + memset(ck->thresholds, 0, sizeof(ck->thresholds[0]) * CHROMAKEY_MAX_MASK); + ck->no_cache = 1; + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_DECLARE(switch_status_t) switch_chromakey_autocolor(switch_chromakey_t *ck, switch_shade_t autocolor, uint32_t threshold) +{ + switch_assert(ck); + + switch_chromakey_clear_colors(ck); + ck->autocolor = autocolor; + ck->dft_thresh = threshold; + ck->dft_thresh_squared = threshold * threshold; + switch_img_free(&ck->cache_img); + ck->no_cache = 90; + memset(&ck->auto_color, 0, sizeof(ck->auto_color)); + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_DECLARE(switch_status_t) switch_chromakey_add_color(switch_chromakey_t *ck, switch_rgb_color_t *color, uint32_t threshold) +{ + switch_assert(ck); + + if (ck->mask_len == CHROMAKEY_MAX_MASK) { + return SWITCH_STATUS_FALSE; + } + + ck->mask[ck->mask_len] = *color; + ck->thresholds[ck->mask_len] = threshold * threshold; + ck->mask_len++; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Adding color %d:%d:%d #%.2x%.2x%.2x\n", + ck->auto_color.r, ck->auto_color.g, ck->auto_color.b, ck->auto_color.r, ck->auto_color.g, ck->auto_color.b); + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_DECLARE(switch_status_t) switch_chromakey_destroy(switch_chromakey_t **ckP) +{ + switch_chromakey_t *ck; + + switch_assert(ckP); + + ck = *ckP; + *ckP = NULL; + + if (ck) { + switch_img_free(&ck->cache_img); + free(ck); + } + + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_DECLARE(switch_status_t) switch_chromakey_create(switch_chromakey_t **ckP) +{ + switch_chromakey_t *ck; + + switch_assert(ckP); + + switch_zmalloc(ck, sizeof(*ck)); + + *ckP = ck; + + return SWITCH_STATUS_SUCCESS; + +} + +SWITCH_DECLARE(switch_image_t *) switch_chromakey_cache_image(switch_chromakey_t *ck) +{ + switch_assert(ck); + + return ck->cache_img; +} + +static inline int get_max(switch_rgb_color_t *c1) +{ + if (c1->r > c1->g && c1->r > c1->b) { + return 1; + } + + if (c1->g > c1->r && c1->g > c1->b) { + return 2; + } + + if (c1->b > c1->r && c1->b > c1->g) { + return 3; + } + + return 0; +} + +static inline int switch_color_dom_cmp(switch_rgb_color_t *c1, switch_rgb_color_t *c2) +{ + + int c1_max = get_max(c1); + int c2_max = get_max(c2); + + if (c1_max && c1_max == c2_max) return 1; + + return 0; + +} + +static inline int switch_color_distance_literal(switch_rgb_color_t *c1, switch_rgb_color_t *c2, int distance) +{ + int r = abs(c1->r - c2->r); + int g = abs(c1->g - c2->g); + int b = abs(c1->b - c2->b); + + if (r < distance && g < distance && b < distance) return 1; + + return 0; +} static inline int switch_color_distance_cheap(switch_rgb_color_t *c1, switch_rgb_color_t *c2) { - int r = c1->r - c2->r; - int g = c1->g - c2->g; - int b = c1->b - c2->b; + int r = abs(c1->r - c2->r); + int g = abs(c1->g - c2->g); + int b = abs(c1->b - c2->b); - if (!r && !g && !b) return 0; + if (r < 5 && g < 5 && b < 5) return 0; - return (3*abs(r)) + (4*abs(g)) + (3*abs(b)); + return (3*r) + (4*g) + (3*b); } -SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_image_t *cache_img, switch_rgb_color_t *mask, int *thresholds, int count) +SWITCH_DECLARE(void) switch_chromakey_process(switch_chromakey_t *ck, switch_image_t *img) { uint8_t *pixel, *last_pixel = NULL, *cache_pixel = NULL, *end_pixel = NULL; int last_hits = 0; + switch_image_t *cache_img; + int same = 0; + int same_same = 0; #ifdef DEBUG_CHROMA int other_img_cached = 0, color_cached = 0, checked = 0, hit_total = 0, total_pixel = 0, delta_hits = 0; #endif - + switch_assert(ck); switch_assert(img); if (img->fmt != SWITCH_IMG_FMT_ARGB) return; pixel = img->planes[SWITCH_PLANE_PACKED]; + cache_img = ck->cache_img; + ck->cache_img = NULL; + if (cache_img && (cache_img->d_w != img->d_w || cache_img->d_h != img->d_h)) { - cache_img = NULL; + switch_img_free(&cache_img); } if (cache_img) { @@ -884,16 +1058,23 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag end_pixel = (img->planes[SWITCH_PLANE_PACKED] + img->d_w * img->d_h * 4); + if (ck->autocolor) { + ck->color_count = 0; + ck->rr = ck->gg = ck->bb = 0; + } + for (; pixel < end_pixel; pixel += 4) { switch_rgb_color_t *color = (switch_rgb_color_t *)pixel; switch_rgb_color_t *last_color = (switch_rgb_color_t *)last_pixel; int hits = 0; + + #ifdef DEBUG_CHROMA total_pixel++; #endif - - if (cache_img && cache_pixel) { + + if (!ck->no_cache && cache_img && cache_pixel) { switch_rgb_color_t *cache_color = (switch_rgb_color_t *)cache_pixel; if (switch_color_distance_cheap(color, cache_color) < 5) { @@ -902,28 +1083,92 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag #endif *pixel = *cache_pixel; goto end; - } + } } - if (last_color && switch_color_distance_cheap(color, last_color) < 5) { + if (last_color) { + if (switch_color_distance_cheap(color, last_color) < 5) { - hits = last_hits; + hits = last_hits; #ifdef DEBUG_CHROMA - color_cached++; + color_cached++; #endif + + same++; + } else { + same = 0; + } } if (!hits) { - hits = switch_color_distance_multi(color, mask, count, thresholds); + + if (ck->autocolor) { + int dom, a, b; + + switch(ck->autocolor) { + case SWITCH_SHADE_RED: + dom = color->r; + a = color->g; + b = color->b; + break; + case SWITCH_SHADE_GREEN: + dom = color->g; + a = color->r; + b = color->b; + break; + case SWITCH_SHADE_BLUE: + dom = color->b; + a = color->r; + b = color->g; + break; + default: + dom = 0; + a = 0; + b = 0; + break; + } + + if (ck->autocolor != SWITCH_SHADE_AUTO) { + //printf("WTF %d\n", ck->dft_thresh); + + int tol = ck->dft_thresh; + int a_tol = tol/6; + int b_tol = tol/6; + + if (dom > a && dom > b && dom > tol) { + if (dom - a > a_tol && dom - b > b_tol) { + hits = 1; + } + } + } + } + + if (!hits && ck->mask_len) { + hits = switch_color_distance_multi(color, ck->mask, ck->mask_len, ck->thresholds); + } + #ifdef DEBUG_CHROMA checked++; #endif } - + end: + if (same > 100 && last_color && switch_color_dom_cmp(color, last_color)) { + same_same++; + } else { + same_same = 0; + } + + if (!hits && ck->autocolor == SWITCH_SHADE_AUTO && (same > 300 || same_same > 50) && ck->no_cache) { + ck->color_count++; + ck->rr += color->r; + ck->gg += color->g; + ck->bb += color->b; + } + if (cache_pixel) { cache_pixel += 4; } @@ -938,10 +1183,48 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag last_pixel = pixel; last_hits = hits; } + + if (ck->color_count > 1000) { + switch_rgb_color_t *last_color = NULL; + int skip = 0; + + ck->auto_color.r = ck->rr / ck->color_count; + ck->auto_color.g = ck->gg / ck->color_count; + ck->auto_color.b = ck->bb / ck->color_count; + + if (ck->mask_len) { + int i = 0; + + for (i = 0; i < ck->mask_len; i++) { + last_color = &ck->mask[i]; + if (switch_color_distance_literal(&ck->auto_color, last_color, 10) || !switch_color_dom_cmp(&ck->auto_color, last_color)) { + skip = 1; + break; + } + } + } + + if (!ck->mask_len || !skip) { + switch_chromakey_add_color(ck, &ck->auto_color, ck->dft_thresh); + } + + } + + if (ck->no_cache > 0 && ck->mask_len) { + ck->no_cache--; + } + + #ifdef DEBUG_CHROMA printf("total %d: other img cache %d color cache %d Checked %d Hit Total %d Delta hits: %d\n", total_pixel, other_img_cached, color_cached, checked, hit_total, delta_hits); #endif + if (!ck->no_cache) { + switch_img_copy(img, &ck->cache_img); + } + + switch_img_free(&cache_img); + return; }