From a537d803728f844d50f4243a6b0861db0a02dccc Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Tue, 22 Jul 2014 05:59:54 +0000 Subject: [PATCH] Add new module: mod_prefix mod_prefix is an in-memory data store optimized for fast lookups according to the longest prefix match (LPM) rule. Tables of key-value string pairs in JSON format can be loaded at startup via configuration and at runtime via the API. The implementation uses a bitwise trie (aka binary prefix tree), so arbitrary string keys are supported. --- build/modules.conf.in | 1 + configure.ac | 1 + debian/control-modules | 5 + docs/AUTHORS | 2 +- src/mod/applications/mod_prefix/Makefile.am | 9 + .../mod_prefix/conf/prefix.conf.xml | 5 + src/mod/applications/mod_prefix/mod_prefix.c | 278 ++++++++++++++++++ src/mod/applications/mod_prefix/trie.c | 113 +++++++ src/mod/applications/mod_prefix/trie.h | 40 +++ 9 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 src/mod/applications/mod_prefix/Makefile.am create mode 100644 src/mod/applications/mod_prefix/conf/prefix.conf.xml create mode 100644 src/mod/applications/mod_prefix/mod_prefix.c create mode 100644 src/mod/applications/mod_prefix/trie.c create mode 100644 src/mod/applications/mod_prefix/trie.h diff --git a/build/modules.conf.in b/build/modules.conf.in index 7368cc8f01..44b386f151 100644 --- a/build/modules.conf.in +++ b/build/modules.conf.in @@ -30,6 +30,7 @@ applications/mod_httapi #applications/mod_nibblebill #applications/mod_oreka #applications/mod_osp +#applications/mod_prefix #applications/mod_rad_auth #applications/mod_redis #applications/mod_rss diff --git a/configure.ac b/configure.ac index df70f63565..d255162dbe 100644 --- a/configure.ac +++ b/configure.ac @@ -1459,6 +1459,7 @@ AC_CONFIG_FILES([Makefile src/mod/applications/mod_nibblebill/Makefile src/mod/applications/mod_oreka/Makefile src/mod/applications/mod_osp/Makefile + src/mod/applications/mod_prefix/Makefile src/mod/applications/mod_rad_auth/Makefile src/mod/applications/mod_random/Makefile src/mod/applications/mod_redis/Makefile diff --git a/debian/control-modules b/debian/control-modules index dce770f279..57b62dd6d1 100644 --- a/debian/control-modules +++ b/debian/control-modules @@ -156,6 +156,11 @@ Description: Open Settlement Protocol This module adds support for the Open Settlement Protocol (OSP). Build-Depends: libosptk3-dev +Module: applications/mod_prefix +Description: Longest prefix match search + This module provides a data store with fast lookups by the longest + prefix match (LPM) rule. + Module: applications/mod_rad_auth Description: RADIUS AA This module implements RADIUS Authentication and Authorization. diff --git a/docs/AUTHORS b/docs/AUTHORS index 495889c269..67925ca8c6 100644 --- a/docs/AUTHORS +++ b/docs/AUTHORS @@ -28,7 +28,7 @@ that much better: Justin Unger - Lots of help with patches and SIP testing. Thanks! Paul D. Tinsley - Various patches and support. Ken Rice - - xmlcdr, sofia improvements, load testing, 1 liners here and there. - Travis Cross - git migration, Debian packaging, ZRTP integration, and many other improvements + Travis Cross - git migration, Debian packaging, ZRTP integration, mod_prefix, and many other improvements Neal Horman - conference improvements, switch_ivr menu additions and other tweaks. Johny Kadarisman - mod_python fixups. Michael Murdock - testing, documentation, bug finding and usability enhancements. diff --git a/src/mod/applications/mod_prefix/Makefile.am b/src/mod/applications/mod_prefix/Makefile.am new file mode 100644 index 0000000000..4c9ba6e2e2 --- /dev/null +++ b/src/mod/applications/mod_prefix/Makefile.am @@ -0,0 +1,9 @@ +include $(top_srcdir)/build/modmake.rulesam +MODNAME=mod_prefix + +mod_LTLIBRARIES = mod_prefix.la +mod_prefix_la_SOURCES = mod_prefix.c trie.c +mod_prefix_la_CFLAGS = $(AM_CFLAGS) +mod_prefix_la_CPPFLAGS = -I. $(AM_CPPFLAGS) +mod_prefix_la_LIBADD = $(switch_builddir)/libfreeswitch.la +mod_prefix_la_LDFLAGS = -avoid-version -module -no-undefined -shared diff --git a/src/mod/applications/mod_prefix/conf/prefix.conf.xml b/src/mod/applications/mod_prefix/conf/prefix.conf.xml new file mode 100644 index 0000000000..d5be03393e --- /dev/null +++ b/src/mod/applications/mod_prefix/conf/prefix.conf.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/src/mod/applications/mod_prefix/mod_prefix.c b/src/mod/applications/mod_prefix/mod_prefix.c new file mode 100644 index 0000000000..1a1ebacfde --- /dev/null +++ b/src/mod/applications/mod_prefix/mod_prefix.c @@ -0,0 +1,278 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2014, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Anthony Minessale II + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Travis Cross + * + * mod_prefix.c -- Longest-prefix match in-memory data store + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "trie.h" + +SWITCH_MODULE_LOAD_FUNCTION(mod_prefix_load); +SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_prefix_shutdown); +SWITCH_MODULE_DEFINITION(mod_prefix, mod_prefix_load, mod_prefix_shutdown, NULL); + +static struct { + switch_memory_pool_t *pool; + switch_hash_t *trees; + switch_thread_rwlock_t *trees_rwlock; +} globals; + +struct prefix_tree { + struct bit_trie_node *node; + switch_thread_rwlock_t *rwlock; +}; + +static cJSON* parse_file(const char *file) { + cJSON *root; + char *buf; + struct stat s; + int fd = open(file, O_RDONLY); + if (fd < 0) return NULL; + fstat(fd, &s); + buf = mmap(0, s.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (buf == MAP_FAILED) return NULL; + root = cJSON_Parse(buf); + if (munmap(buf, s.st_size) < 0) abort(); + return root; +} + +static struct bit_trie_node *load_file(const char *file) { + struct bit_trie_node *node; + cJSON *item, *root = parse_file(file); + if (!root) return NULL; + node = bit_trie_create(); + item = root->child; + while(item) { + if (item->string && item->valuestring) { + bit_trie_set(node, (unsigned char*)item->string, strlen(item->string), + strdup(item->valuestring), strlen(item->valuestring)+1); + } + item = item->next; + } + cJSON_Delete(root); + return node; +} + +static int search_prefix_tree(switch_stream_handle_t *stream, + const char *name, const char *key) { + struct prefix_tree *tr; + switch_thread_rwlock_rdlock(globals.trees_rwlock); + if ((tr = switch_core_hash_find(globals.trees, name))) { + struct bit_trie_node *res = NULL; + switch_thread_rwlock_rdlock(tr->rwlock); + bit_trie_get(&res, tr->node, (unsigned char*)key, strlen(key)); + if (res && res->value) { + stream->write_function(stream, "%s", res->value); + } else { + stream->write_function(stream, ""); + } + switch_thread_rwlock_unlock(tr->rwlock); + } + switch_thread_rwlock_unlock(globals.trees_rwlock); + return 0; +} + +static int load_prefix_tree(const char *name, const char *file) { + struct prefix_tree *tr; + struct bit_trie_node *node; + node = load_file(file); + if (node) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Loaded prefix file %s\n", file); + } else { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error loading prefix file %s\n", file); + return 2; + } + switch_thread_rwlock_rdlock(globals.trees_rwlock); + if ((tr = switch_core_hash_find(globals.trees, name))) { + switch_thread_rwlock_wrlock(tr->rwlock); + bit_trie_free(tr->node); + tr->node = node; + switch_thread_rwlock_unlock(tr->rwlock); + switch_thread_rwlock_unlock(globals.trees_rwlock); + } else { + switch_thread_rwlock_unlock(globals.trees_rwlock); + switch_thread_rwlock_wrlock(globals.trees_rwlock); + if (!(tr = malloc(sizeof(struct prefix_tree)))) abort(); + memset(tr, 0, sizeof(struct prefix_tree)); + tr->node = node; + switch_thread_rwlock_create(&tr->rwlock, globals.pool); + switch_core_hash_insert(globals.trees, name, tr); + switch_thread_rwlock_unlock(globals.trees_rwlock); + } + return 0; +} + +static int drop_prefix_tree(const char *name) { + struct prefix_tree *tr; + switch_thread_rwlock_wrlock(globals.trees_rwlock); + if ((tr = switch_core_hash_find(globals.trees, name))) { + switch_core_hash_delete(globals.trees, name); + switch_thread_rwlock_wrlock(tr->rwlock); + bit_trie_free(tr->node); + switch_thread_rwlock_unlock(tr->rwlock); + switch_thread_rwlock_destroy(tr->rwlock); + free(tr); + } + switch_thread_rwlock_unlock(globals.trees_rwlock); + return 0; +} + +static void do_config(switch_bool_t reload); + +#define PREFIX_API_USAGE "get
| load
| drop
| reload" +SWITCH_STANDARD_API(prefix_api_function) +{ + int argc = 0; + char *argv[4] = { 0 }; + char *mydata = NULL; + + if (!zstr(cmd)) { + mydata = strdup(cmd); + switch_assert(mydata); + argc = switch_separate_string(mydata, ' ', argv, (sizeof(argv) / sizeof(argv[0]))); + } + + if (argc < 1 || !argv[0]) { + goto usage; + } + + if (!strcasecmp(argv[0], "get")) { + char *name, *key; + int ret; + if (argc < 3) goto usage; + name = argv[1]; key = argv[2]; + ret = search_prefix_tree(stream, name, key); + if (ret == 1) goto usage; + } else if (!strcasecmp(argv[0], "load")) { + char *name, *file; + if (argc < 3) goto usage; + name = argv[1]; file = argv[2]; + if (!load_prefix_tree(name, file)) { + stream->write_function(stream, "+OK\n"); + } else { + stream->write_function(stream, "-ERR\n"); + } + } else if (!strcasecmp(argv[0], "drop")) { + char *name; + if (argc < 2) goto usage; + name = argv[1]; + if (!drop_prefix_tree(name)) { + stream->write_function(stream, "+OK\n"); + } else { + stream->write_function(stream, "-ERR\n"); + } + } else if (!strcasecmp(argv[0], "reload")) { + do_config(1); + stream->write_function(stream, "+OK\n"); + } else { + goto usage; + } + goto done; + + usage: + stream->write_function(stream, "-ERR Usage: prefix %s\n", PREFIX_API_USAGE); + + done: + switch_safe_free(mydata); + return SWITCH_STATUS_SUCCESS; +} + +static void do_config(switch_bool_t reload) +{ + switch_xml_t xml = NULL, x_lists = NULL, x_list = NULL, cfg = NULL; + if ((xml = switch_xml_open_cfg("prefix.conf", &cfg, NULL))) { + if ((x_lists = switch_xml_child(cfg, "tables"))) { + for (x_list = switch_xml_child(x_lists, "table"); x_list; x_list = x_list->next) { + const char *name = switch_xml_attr(x_list, "name"); + const char *file = switch_xml_attr(x_list, "file"); + load_prefix_tree(name, file); + } + } + switch_xml_free(xml); + } +} + +SWITCH_MODULE_LOAD_FUNCTION(mod_prefix_load) +{ + switch_api_interface_t *api_interface; + memset(&globals, 0, sizeof(globals)); + globals.pool = pool; + switch_thread_rwlock_create(&globals.trees_rwlock, globals.pool); + switch_core_hash_init(&globals.trees); + *module_interface = switch_loadable_module_create_module_interface(pool, modname); + SWITCH_ADD_API(api_interface, "prefix", "prefix handling", prefix_api_function, PREFIX_API_USAGE); + switch_console_set_complete("add prefix get"); + switch_console_set_complete("add prefix load"); + switch_console_set_complete("add prefix drop"); + switch_console_set_complete("add prefix reload"); + do_config(SWITCH_FALSE); + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_prefix_shutdown) +{ + switch_hash_index_t *hi = NULL; + switch_thread_rwlock_wrlock(globals.trees_rwlock); + while ((hi = switch_core_hash_first_iter(globals.trees, hi))) { + void *val = NULL; + struct prefix_tree *tr = NULL; + const void *key; + switch_ssize_t keylen; + switch_core_hash_this(hi, &key, &keylen, &val); + tr = (struct prefix_tree*)val; + switch_core_hash_delete(globals.trees, key); + switch_thread_rwlock_wrlock(tr->rwlock); + bit_trie_free(tr->node); + switch_thread_rwlock_unlock(tr->rwlock); + switch_thread_rwlock_destroy(tr->rwlock); + free(tr); + } + switch_core_hash_destroy(&globals.trees); + switch_thread_rwlock_unlock(globals.trees_rwlock); + switch_thread_rwlock_destroy(globals.trees_rwlock); + return SWITCH_STATUS_SUCCESS; +} + +/* For Emacs: + * Local Variables: + * mode:c + * indent-tabs-mode:t + * tab-width:4 + * c-basic-offset:4 + * show-trailing-whitespace:t + * End: + * For VIM: + * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet: + */ diff --git a/src/mod/applications/mod_prefix/trie.c b/src/mod/applications/mod_prefix/trie.c new file mode 100644 index 0000000000..667c5c4166 --- /dev/null +++ b/src/mod/applications/mod_prefix/trie.c @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2014 Travis Cross + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include +#include +#include +#include "trie.h" + +struct bit_trie_node* bit_trie_create() { + struct bit_trie_node *node = malloc(sizeof(struct bit_trie_node)); + if (!node) abort(); + memset(node, 0, sizeof(struct bit_trie_node)); + return node; +} + +uint32_t bit_trie_get(struct bit_trie_node **node_out, + struct bit_trie_node *node, + unsigned char *key, + uint32_t key_len) { + unsigned char *keyp = key, *keyp0 = key; + struct bit_trie_node *node0 = node; + while(keyp < key + key_len) { + uint8_t offset = 0; + while(offset < 8) { + uint8_t bit = (*keyp >> offset) & 0x01; + struct bit_trie_node *next = node->next[bit]; + if (next) { + node = next; + } else { + *node_out = node0; + return keyp0 - key; + } + offset++; + } + keyp++; + if (node->value) { + node0 = node; + keyp0 = keyp; + } + } + *node_out = node0; + return keyp0 - key; +} + +struct bit_trie_node* bit_trie_set(struct bit_trie_node *node, + unsigned char *key, + uint32_t key_len, + void *value, + uint32_t value_len) { + unsigned char *keyp = key; + while(keyp < key + key_len) { + uint8_t offset = 0; + while(offset < 8) { + uint8_t bit = (*keyp >> offset) & 0x01; + struct bit_trie_node *next = node->next[bit]; + if (next) { + node = next; + } else { + next = bit_trie_create(); + node->next[bit] = next; + node = next; + } + offset++; + } + keyp++; + } + node->value = value; + node->value_len = value_len; + return node; +} + +static uint32_t bit_trie_free_r(struct bit_trie_node *node) { + uint32_t count = 0; + if (!node) return count; + count += bit_trie_free_r(node->next[0]); + count += bit_trie_free_r(node->next[1]); + free(node->value); + free(node); + count++; + return count; +} + +uint32_t bit_trie_free(struct bit_trie_node *node) { + return bit_trie_free_r(node); +} + +uint32_t bit_trie_byte_size(struct bit_trie_node *node) { + uint32_t size = 0; + if (!node) return size; + size += bit_trie_byte_size(node->next[0]); + size += bit_trie_byte_size(node->next[1]); + size += sizeof(struct bit_trie_node) + node->value_len; + return size; +} diff --git a/src/mod/applications/mod_prefix/trie.h b/src/mod/applications/mod_prefix/trie.h new file mode 100644 index 0000000000..69d2323e38 --- /dev/null +++ b/src/mod/applications/mod_prefix/trie.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2014 Travis Cross + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +struct bit_trie_node { + struct bit_trie_node *next[2]; + void *value; + uint32_t value_len; +}; + +struct bit_trie_node* bit_trie_create(); +uint32_t bit_trie_get(struct bit_trie_node **node_out, + struct bit_trie_node *node, + unsigned char *key, + uint32_t key_len); +struct bit_trie_node* bit_trie_set(struct bit_trie_node *node, + unsigned char *key, + uint32_t key_len, + void *value, + uint32_t value_len); +uint32_t bit_trie_free(struct bit_trie_node *node); +uint32_t bit_trie_byte_size(struct bit_trie_node *node);