kenmirrors
/
freeswitch
mirror of https://github.com/signalwire/freeswitch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

FS-9775: A bunch of stuff related to chaining multiple jobs, bug fixes, few other changes

This commit is contained in:
Shane Bryldt 2017-01-03 07:09:02 +00:00 committed by Mike Jerris
parent a698651018
commit 183116452b
12 changed files with 925 additions and 511 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libs/libks/Makefile.am
View File

@ -14,7 +14,8 @@ libks_la_SOURCES += src/ks_ssl.c src/kws.c src/ks_rng.c
libks_la_SOURCES += src/utp/utp_api.cpp src/utp/utp_callbacks.cpp src/utp/utp_hash.cpp src/utp/utp_internal.cpp
libks_la_SOURCES += src/utp/utp_packedsockaddr.cpp src/utp/utp_utils.cpp src/ks_bencode.c
libks_la_SOURCES += src/dht/ks_dht.c src/dht/ks_dht_datagram.c src/dht/ks_dht_endpoint.c src/dht/ks_dht_message.c src/dht/ks_dht_transaction.c
libks_la_SOURCES += src/dht/ks_dht_job.c src/dht/ks_dht_search.c src/dht/ks_dht_storageitem.c src/dht/ks_dht_bucket.c
libks_la_SOURCES += src/dht/ks_dht_job.c src/dht/ks_dht_search.c src/dht/ks_dht_publish.c src/dht/ks_dht_distribute.c src/dht/ks_dht_storageitem.c
libks_la_SOURCES += src/dht/ks_dht_bucket.c
libks_la_SOURCES += crypt/aeskey.c crypt/aestab.c crypt/sha2.c crypt/twofish.c crypt/aes_modes.c crypt/aescrypt.c crypt/twofish_cfb.c
#aes.h aescpp.h brg_endian.h aesopt.h aestab.h brg_types.h sha2.h twofish.h

44
libs/libks/src/dht/ks_dht-int.h
View File

@ -22,11 +22,11 @@ KS_BEGIN_EXTERN_C
KS_DECLARE(ks_status_t) ks_dht_autoroute_check(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_endpoint_t **endpoint);
/**
* Called internally to expire search data.
* Handles completing and purging of finished searches.
* Called internally to expire or reannounce storage item data.
* Handles reannouncing and purging of expiring storage items.
* @param dht pointer to the dht instance
*/
KS_DECLARE(void) ks_dht_pulse_searches(ks_dht_t *dht);
KS_DECLARE(void) ks_dht_pulse_storageitems(ks_dht_t *dht);
/**
* Called internally to process job state machine.
@ -276,19 +276,18 @@ KS_DECLARE(void) ks_dht_datagram_destroy(ks_dht_datagram_t **datagram);
KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
ks_pool_t *pool,
const ks_sockaddr_t *raddr,
int32_t attempts);
int32_t attempts,
void *data);
KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t query_callback, ks_dht_job_callback_t finish_callback);
KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
ks_dht_search_t *search,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback,
ks_dht_nodeid_t *target);
KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
ks_dht_search_t *search,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback,
ks_dht_nodeid_t *target,
uint8_t *salt,
const uint8_t *salt,
ks_size_t salt_length);
KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
ks_dht_job_callback_t query_callback,
@ -296,6 +295,9 @@ KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
ks_dht_token_t *token,
int64_t cas,
ks_dht_storageitem_t *item);
KS_DECLARE(void) ks_dht_job_build_search(ks_dht_job_t *job,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback);
KS_DECLARE(void) ks_dht_job_build_search_findnode(ks_dht_job_t *job,
ks_dht_nodeid_t *target,
uint32_t family,
@ -344,9 +346,35 @@ KS_DECLARE(ks_status_t) ks_dht_message_response(ks_dht_message_t *message,
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target, ks_dht_search_callback_t callback);
KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search,
ks_pool_t *pool,
ks_dhtrt_routetable_t *table,
const ks_dht_nodeid_t *target,
ks_dht_job_callback_t callback,
void *data);
KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search);
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_publish_create(ks_dht_publish_t **publish,
ks_pool_t *pool,
ks_dht_job_callback_t callback,
void *data,
int64_t cas,
ks_dht_storageitem_t *item);
KS_DECLARE(void) ks_dht_publish_destroy(ks_dht_publish_t **publish);
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_distribute_create(ks_dht_distribute_t **distribute,
ks_pool_t *pool,
ks_dht_storageitem_callback_t callback,
void *data,
int64_t cas,
ks_dht_storageitem_t *item);
KS_DECLARE(void) ks_dht_distribute_destroy(ks_dht_distribute_t **distribute);
/**
*

783
libs/libks/src/dht/ks_dht.c
View File

File diff suppressed because it is too large Load Diff

164
libs/libks/src/dht/ks_dht.h
View File

@ -31,7 +31,6 @@ KS_BEGIN_EXTERN_C
#define KS_DHT_TRANSACTION_EXPIRATION 10
#define KS_DHT_TRANSACTIONS_PULSE 1
#define KS_DHT_SEARCH_EXPIRATION 10
#define KS_DHT_SEARCH_RESULTS_MAX_SIZE 8 // @todo replace with KS_DHTRT_BUCKET_SIZE
#define KS_DHT_STORAGEITEM_PKEY_SIZE crypto_sign_PUBLICKEYBYTES
@ -39,6 +38,8 @@ KS_BEGIN_EXTERN_C
#define KS_DHT_STORAGEITEM_SALT_MAX_SIZE 64
#define KS_DHT_STORAGEITEM_SIGNATURE_SIZE crypto_sign_BYTES
#define KS_DHT_STORAGEITEM_EXPIRATION 7200
#define KS_DHT_STORAGEITEM_KEEPALIVE 300
#define KS_DHT_STORAGEITEMS_PULSE 10
#define KS_DHT_TOKEN_SIZE SHA_DIGEST_LENGTH
#define KS_DHT_TOKEN_EXPIRATION 300
@ -58,7 +59,8 @@ typedef struct ks_dht_message_s ks_dht_message_t;
typedef struct ks_dht_endpoint_s ks_dht_endpoint_t;
typedef struct ks_dht_transaction_s ks_dht_transaction_t;
typedef struct ks_dht_search_s ks_dht_search_t;
typedef struct ks_dht_search_pending_s ks_dht_search_pending_t;
typedef struct ks_dht_publish_s ks_dht_publish_t;
typedef struct ks_dht_distribute_s ks_dht_distribute_t;
typedef struct ks_dht_node_s ks_dht_node_t;
typedef struct ks_dhtrt_routetable_s ks_dhtrt_routetable_t;
typedef struct ks_dhtrt_querynodes_s ks_dhtrt_querynodes_t;
@ -67,7 +69,9 @@ typedef struct ks_dht_storageitem_s ks_dht_storageitem_t;
typedef ks_status_t (*ks_dht_job_callback_t)(ks_dht_t *dht, ks_dht_job_t *job);
typedef ks_status_t (*ks_dht_message_callback_t)(ks_dht_t *dht, ks_dht_message_t *message);
typedef ks_status_t (*ks_dht_search_callback_t)(ks_dht_t *dht, ks_dht_search_t *search);
//typedef ks_status_t (*ks_dht_search_callback_t)(ks_dht_t *dht, ks_dht_search_t *search);
typedef ks_status_t (*ks_dht_storageitem_callback_t)(ks_dht_t *dht, ks_dht_storageitem_t *item);
struct ks_dht_datagram_s {
ks_pool_t *pool;
@ -115,11 +119,12 @@ enum ks_dht_job_state_t {
KS_DHT_JOB_STATE_COMPLETING,
};
//enum ks_dht_job_type_t {
// KS_DHT_JOB_TYPE_NONE = 0,
// KS_DHT_JOB_TYPE_PING,
// KS_DHT_JOB_TYPE_FINDNODE,
//};
enum ks_dht_job_result_t {
KS_DHT_JOB_RESULT_SUCCESS = 0,
KS_DHT_JOB_RESULT_EXPIRED,
KS_DHT_JOB_RESULT_ERROR,
KS_DHT_JOB_RESULT_FAILURE,
};
struct ks_dht_job_s {
ks_pool_t *pool;
@ -127,8 +132,7 @@ struct ks_dht_job_s {
ks_dht_job_t *next;
enum ks_dht_job_state_t state;
ks_dht_search_t *search;
enum ks_dht_job_result_t result;
ks_sockaddr_t raddr; // will obtain local endpoint node id when creating message using raddr
int32_t attempts;
@ -137,6 +141,7 @@ struct ks_dht_job_s {
ks_dht_job_callback_t query_callback;
ks_dht_job_callback_t finish_callback;
void *data;
ks_dht_message_t *response;
// job specific query parameters
@ -145,15 +150,22 @@ struct ks_dht_job_s {
int64_t query_cas;
ks_dht_token_t query_token;
ks_dht_storageitem_t *query_storageitem;
uint32_t query_family;
int32_t query_family;
// error response parameters
int64_t error_code;
struct bencode *error_description;
// job specific response parameters
ks_dht_nodeid_t response_id;
ks_dht_node_t *response_id;
ks_dht_node_t *response_nodes[KS_DHT_RESPONSE_NODES_MAX_SIZE];
ks_size_t response_nodes_count;
ks_dht_node_t *response_nodes6[KS_DHT_RESPONSE_NODES_MAX_SIZE];
ks_size_t response_nodes6_count;
ks_dht_token_t response_token;
int64_t response_seq;
ks_bool_t response_hasitem;
ks_dht_storageitem_t *response_storageitem;
};
@ -219,25 +231,48 @@ struct ks_dht_transaction_s {
struct ks_dht_search_s {
ks_pool_t *pool;
ks_dht_search_t *next;
ks_dhtrt_routetable_t *table;
ks_dht_nodeid_t target;
ks_dht_search_callback_t callback;
ks_dht_job_callback_t callback;
void *data;
ks_mutex_t *mutex;
ks_hash_t *searched;
ks_hash_t *searching;
int32_t searching;
ks_dht_node_t *results[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
ks_dht_nodeid_t distances[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
ks_size_t results_length;
};
struct ks_dht_publish_s {
ks_pool_t *pool;
ks_dht_job_callback_t callback;
void *data;
int64_t cas;
ks_dht_storageitem_t *item;
};
struct ks_dht_distribute_s {
ks_pool_t *pool;
ks_dht_storageitem_callback_t callback;
void *data;
ks_mutex_t *mutex;
int32_t publishing;
int64_t cas;
ks_dht_storageitem_t *item;
};
struct ks_dht_storageitem_s {
ks_pool_t *pool;
ks_dht_nodeid_t id;
ks_time_t expiration;
ks_time_t keepalive;
struct bencode *v;
ks_bool_t mutable;
ks_mutex_t *mutex;
volatile int32_t refc;
ks_dht_storageitem_callback_t callback;
ks_bool_t mutable;
ks_dht_storageitem_pkey_t pk;
ks_dht_storageitem_skey_t sk;
struct bencode *salt;
@ -282,15 +317,12 @@ struct ks_dht_s {
ks_dhtrt_routetable_t *rt_ipv4;
ks_dhtrt_routetable_t *rt_ipv6;
ks_mutex_t *searches_mutex;
ks_dht_search_t *searches_first;
ks_dht_search_t *searches_last;
ks_time_t tokens_pulse;
volatile uint32_t token_secret_current;
volatile uint32_t token_secret_previous;
ks_time_t token_secret_expiration;
ks_time_t storageitems_pulse;
ks_hash_t *storageitems_hash;
};
@ -395,7 +427,22 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_signature_generate(ks_dht_storageitem
int64_t sequence,
const uint8_t *value,
ks_size_t value_length);
/**
*
*/
KS_DECLARE(void) ks_dht_storageitem_reference(ks_dht_storageitem_t *item);
/**
*
*/
KS_DECLARE(void) ks_dht_storageitem_dereference(ks_dht_storageitem_t *item);
/**
*
*/
KS_DECLARE(void) ks_dht_storageitem_callback(ks_dht_storageitem_t *item, ks_dht_storageitem_callback_t callback);
/**
*
*/
@ -429,37 +476,38 @@ KS_DECLARE(ks_status_t) ks_dht_storageitems_insert(ks_dht_t *dht, ks_dht_storage
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback);
KS_DECLARE(void) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback, void *data);
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_findnode(ks_dht_t *dht,
ks_dht_search_t *search,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
ks_dht_nodeid_t *target);
KS_DECLARE(void) ks_dht_findnode(ks_dht_t *dht,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
void *data,
ks_dht_nodeid_t *target);
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_get(ks_dht_t *dht,
ks_dht_search_t *search,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
ks_dht_nodeid_t *target,
uint8_t *salt,
ks_size_t salt_length);
KS_DECLARE(void) ks_dht_get(ks_dht_t *dht,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
void *data,
ks_dht_nodeid_t *target,
const uint8_t *salt,
ks_size_t salt_length);
/**
*
*/
KS_DECLARE(ks_status_t) ks_dht_put(ks_dht_t *dht,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
ks_dht_token_t *token,
int64_t cas,
ks_dht_storageitem_t *item);
KS_DECLARE(void) ks_dht_put(ks_dht_t *dht,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
void *data,
ks_dht_token_t *token,
int64_t cas,
ks_dht_storageitem_t *item);
/**
* Create a network search of the closest nodes to a target.
@ -468,27 +516,29 @@ KS_DECLARE(ks_status_t) ks_dht_put(ks_dht_t *dht,
* @param target pointer to the nodeid for the target to be searched
* @param callback an optional callback to add to the search when it is finished
* @param search dereferenced out pointer to the allocated search, may be NULL to ignore search output
* @return The ks_status_t result: KS_STATUS_SUCCESS, KS_STATUS_FAIL
* @see ks_dht_search_create
* @see ks_dht_search_callback_add
* @see ks_hash_insert
* @see ks_dht_search_pending_create
* @see ks_dht_send_findnode
* @see ks_dht_findnode
*/
KS_DECLARE(ks_status_t) ks_dht_search_findnode(ks_dht_t *dht,
int32_t family,
ks_dht_nodeid_t *target,
ks_dht_search_callback_t callback,
ks_dht_search_t **search);
KS_DECLARE(ks_status_t) ks_dht_queue_search_findnode(ks_dht_t* dht,
ks_dhtrt_routetable_t *rt,
ks_dht_nodeid_t *target,
ks_dht_job_callback_t callback);
KS_DECLARE(ks_status_t) ks_dht_exec_search_findnode(ks_dht_t *dht, ks_dht_job_t *job);
KS_DECLARE(void) ks_dht_search(ks_dht_t *dht,
ks_dht_job_callback_t callback,
void *data,
ks_dhtrt_routetable_t *table,
ks_dht_nodeid_t *target);
KS_DECLARE(void) ks_dht_publish(ks_dht_t *dht,
const ks_sockaddr_t *raddr,
ks_dht_job_callback_t callback,
void *data,
int64_t cas,
ks_dht_storageitem_t *item);
KS_DECLARE(void) ks_dht_distribute(ks_dht_t *dht,
ks_dht_storageitem_callback_t callback,
void *data,
ks_dhtrt_routetable_t *table,
int64_t cas,
ks_dht_storageitem_t *item);
/**
* route table methods

4
libs/libks/src/dht/ks_dht_bucket.c
View File

@ -1673,7 +1673,7 @@ void ks_dhtrt_ping(ks_dhtrt_internal_t *internal, ks_dhtrt_bucket_entry_t *entry
#endif
ks_dht_node_t* node = entry->gptr;
ks_log(KS_LOG_DEBUG, "Node addr %s %d\n", node->addr.host, node->addr.port);
ks_dht_ping(internal->dht, &node->addr, NULL);
ks_dht_ping(internal->dht, &node->addr, NULL, NULL);
return;
}
@ -1683,7 +1683,7 @@ void ks_dhtrt_find(ks_dhtrt_routetable_t *table, ks_dhtrt_internal_t *internal,
char buf[100];
ks_log(KS_LOG_DEBUG, "Find queued for target %s\n", ks_dhtrt_printableid(target->id, buf));
ks_dht_queue_search_findnode(internal->dht, table, target, NULL);
ks_dht_search(internal->dht, NULL, NULL, table, target);
return;
}

65
libs/libks/src/dht/ks_dht_distribute.c Normal file
View File

@ -0,0 +1,65 @@
#include "ks_dht.h"
#include "ks_dht-int.h"
#include "sodium.h"
KS_DECLARE(ks_status_t) ks_dht_distribute_create(ks_dht_distribute_t **distribute,
ks_pool_t *pool,
ks_dht_storageitem_callback_t callback,
void *data,
int64_t cas,
ks_dht_storageitem_t *item)
{
ks_dht_distribute_t *d;
ks_status_t ret = KS_STATUS_SUCCESS;
ks_assert(distribute);
ks_assert(pool);
ks_assert(cas >= 0);
ks_assert(item);
*distribute = d = ks_pool_alloc(pool, sizeof(ks_dht_distribute_t));
ks_assert(d);
d->pool = pool;
d->callback = callback;
d->data = data;
ks_mutex_create(&d->mutex, KS_MUTEX_FLAG_DEFAULT, d->pool);
ks_assert(d->mutex);
d->cas = cas;
d->item = item;
ks_dht_storageitem_reference(d->item);
// done:
if (ret != KS_STATUS_SUCCESS) {
if (d) ks_dht_distribute_destroy(distribute);
}
return ret;
}
KS_DECLARE(void) ks_dht_distribute_destroy(ks_dht_distribute_t **distribute)
{
ks_dht_distribute_t *d;
ks_assert(distribute);
ks_assert(*distribute);
d = *distribute;
if (d->mutex) ks_mutex_destroy(&d->mutex);
ks_dht_storageitem_dereference(d->item);
ks_pool_free(d->pool, distribute);
}
/* For Emacs:
* Local Variables:
* mode:c
* indent-tabs-mode:t
* tab-width:4
* c-basic-offset:4
* End:
* For VIM:
* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
*/

39
libs/libks/src/dht/ks_dht_job.c
View File

@ -4,7 +4,8 @@
KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
ks_pool_t *pool,
const ks_sockaddr_t *raddr,
int32_t attempts)
int32_t attempts,
void *data)
{
ks_dht_job_t *j;
ks_status_t ret = KS_STATUS_SUCCESS;
@ -12,7 +13,6 @@ KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
ks_assert(job);
ks_assert(pool);
//ks_assert(dht);
ks_assert(raddr);
ks_assert(attempts > 0 && attempts <= 10);
*job = j = ks_pool_alloc(pool, sizeof(ks_dht_job_t));
@ -20,13 +20,9 @@ KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
j->pool = pool;
j->state = KS_DHT_JOB_STATE_QUERYING;
j->raddr = *raddr;
if (raddr) j->raddr = *raddr;
j->attempts = attempts;
//ks_mutex_lock(dht->jobs_mutex);
//if (dht->jobs_last) dht->jobs_last = dht->jobs_last->next = j;
//else dht->jobs_first = dht->jobs_last = j;
//ks_mutex_unlock(dht->jobs_mutex);
j->data = data;
// done:
if (ret != KS_STATUS_SUCCESS) {
@ -45,7 +41,6 @@ KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t
}
KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
ks_dht_search_t *search,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback,
ks_dht_nodeid_t *target)
@ -54,25 +49,22 @@ KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
ks_assert(query_callback);
ks_assert(target);
job->search = search;
job->query_callback = query_callback;
job->finish_callback = finish_callback;
job->query_target = *target;
}
KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
ks_dht_search_t *search,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback,
ks_dht_nodeid_t *target,
uint8_t *salt,
const uint8_t *salt,
ks_size_t salt_length)
{
ks_assert(job);
ks_assert(query_callback);
ks_assert(target);
job->search = search;
job->query_callback = query_callback;
job->finish_callback = finish_callback;
job->query_target = *target;
@ -96,23 +88,18 @@ KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
job->query_token = *token;
job->query_cas = cas;
job->query_storageitem = item;
ks_dht_storageitem_reference(job->query_storageitem);
}
KS_DECLARE(void) ks_dht_job_build_search_findnode(ks_dht_job_t *job,
ks_dht_nodeid_t *target,
uint32_t family,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback)
KS_DECLARE(void) ks_dht_job_build_search(ks_dht_job_t *job,
ks_dht_job_callback_t query_callback,
ks_dht_job_callback_t finish_callback)
{
ks_assert(job);
ks_assert(target);
ks_assert(family);
ks_assert(query_callback);
job->search = NULL;
job->query_callback = query_callback;
job->finish_callback = finish_callback;
job->query_target = *target;
job->query_family = family;
}
KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)
@ -125,9 +112,15 @@ KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)
j = *job;
if (j->query_salt) ben_free(j->query_salt);
if (j->response_id) ks_dhtrt_release_node(j->response_id);
for (int32_t i = 0; i < j->response_nodes_count; ++i) ks_dhtrt_release_node(j->response_nodes[i]);
for (int32_t i = 0; i < j->response_nodes6_count; ++i) ks_dhtrt_release_node(j->response_nodes6[i]);
if (j->query_storageitem) ks_dht_storageitem_dereference(j->query_storageitem);
if (j->response_storageitem) ks_dht_storageitem_dereference(j->response_storageitem);
if (j->error_description) ben_free(j->error_description);
ks_pool_free(j->pool, job);
}

62
libs/libks/src/dht/ks_dht_publish.c Normal file
View File

@ -0,0 +1,62 @@
#include "ks_dht.h"
#include "ks_dht-int.h"
#include "sodium.h"
KS_DECLARE(ks_status_t) ks_dht_publish_create(ks_dht_publish_t **publish,
ks_pool_t *pool,
ks_dht_job_callback_t callback,
void *data,
int64_t cas,
ks_dht_storageitem_t *item)
{
ks_dht_publish_t *p;
ks_status_t ret = KS_STATUS_SUCCESS;
ks_assert(publish);
ks_assert(pool);
ks_assert(cas >= 0);
ks_assert(item);
*publish = p = ks_pool_alloc(pool, sizeof(ks_dht_publish_t));
ks_assert(p);
p->pool = pool;
p->callback = callback;
p->data = data;
p->cas = cas;
p->item = item;
ks_dht_storageitem_reference(p->item);
// done:
if (ret != KS_STATUS_SUCCESS) {
if (p) ks_dht_publish_destroy(publish);
}
return ret;
}
KS_DECLARE(void) ks_dht_publish_destroy(ks_dht_publish_t **publish)
{
ks_dht_publish_t *p;
ks_assert(publish);
ks_assert(*publish);
p = *publish;
ks_dht_storageitem_dereference(p->item);
ks_pool_free(p->pool, publish);
}
/* For Emacs:
* Local Variables:
* mode:c
* indent-tabs-mode:t
* tab-width:4
* c-basic-offset:4
* End:
* For VIM:
* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
*/

15
libs/libks/src/dht/ks_dht_search.c
View File

@ -2,13 +2,19 @@
#include "ks_dht-int.h"
#include "sodium.h"
KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target, ks_dht_search_callback_t callback)
KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search,
ks_pool_t *pool,
ks_dhtrt_routetable_t *table,
const ks_dht_nodeid_t *target,
ks_dht_job_callback_t callback,
void *data)
{
ks_dht_search_t *s;
ks_status_t ret = KS_STATUS_SUCCESS;
ks_assert(search);
ks_assert(pool);
ks_assert(table);
ks_assert(target);
*search = s = ks_pool_alloc(pool, sizeof(ks_dht_search_t));
@ -19,17 +25,17 @@ KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t
ks_mutex_create(&s->mutex, KS_MUTEX_FLAG_DEFAULT, s->pool);
ks_assert(s->mutex);
s->table = table;
memcpy(s->target.id, target->id, KS_DHT_NODEID_SIZE);
s->callback = callback;
s->data = data;
ks_hash_create(&s->searched, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_NOLOCK | KS_HASH_FLAG_DUP_CHECK, s->pool);
ks_assert(s->searched);
ks_hash_set_keysize(s->searched, KS_DHT_NODEID_SIZE);
ks_hash_create(&s->searching, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_NOLOCK | KS_HASH_FLAG_DUP_CHECK, s->pool);
ks_assert(s->searching);
ks_hash_set_keysize(s->searching, KS_DHT_NODEID_SIZE);
s->searching = 0;
// done:
if (ret != KS_STATUS_SUCCESS) {
@ -47,7 +53,6 @@ KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search)
s = *search;
if (s->searching) ks_hash_destroy(&s->searching);
if (s->searched) ks_hash_destroy(&s->searched);
if (s->mutex) ks_mutex_destroy(&s->mutex);

33
libs/libks/src/dht/ks_dht_storageitem.c
View File

@ -23,9 +23,12 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable_internal(ks_dht_stor
si->id = *target;
si->mutable = KS_FALSE;
si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
si->keepalive = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_KEEPALIVE * KS_USEC_PER_SEC);
si->v = clone_v ? ben_clone(v) : v;
ks_assert(si->v);
si->refc = 1;
// done:
if (ret != KS_STATUS_SUCCESS) {
if (si) ks_dht_storageitem_destroy(item);
@ -81,9 +84,12 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable_internal(ks_dht_storag
si->id = *target;
si->mutable = KS_TRUE;
si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
si->keepalive = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_KEEPALIVE * KS_USEC_PER_SEC);
si->v = clone_v ? ben_clone(v) : v;
ks_assert(si->v);
si->refc = 1;
ks_mutex_create(&si->mutex, KS_MUTEX_FLAG_DEFAULT, si->pool);
ks_assert(si->mutex);
@ -169,6 +175,33 @@ KS_DECLARE(void) ks_dht_storageitem_destroy(ks_dht_storageitem_t **item)
ks_pool_free(si->pool, item);
}
KS_DECLARE(void) ks_dht_storageitem_reference(ks_dht_storageitem_t *item)
{
ks_assert(item);
ks_mutex_lock(item->mutex);
item->refc++;
ks_mutex_unlock(item->mutex);
}
KS_DECLARE(void) ks_dht_storageitem_dereference(ks_dht_storageitem_t *item)
{
ks_assert(item);
ks_mutex_lock(item->mutex);
item->refc--;
ks_mutex_unlock(item->mutex);
ks_assert(item->refc >= 0);
}
KS_DECLARE(void) ks_dht_storageitem_callback(ks_dht_storageitem_t *item, ks_dht_storageitem_callback_t callback)
{
ks_assert(item);
item->callback = callback;
}
/* For Emacs:
* Local Variables:
* mode:c

8
libs/libks/src/ks_thread_pool.c
View File

@ -102,10 +102,10 @@ static int check_queue(ks_thread_pool_t *tp, ks_bool_t adding)
need--;
}
/*
ks_log(KS_LOG_DEBUG, "WORKER check: adding %d need %d running %d dying %d total %d max %d\n",
adding, need, tp->running_thread_count, tp->dying_thread_count, tp->thread_count, tp->max);
*/
return need;
}
@ -129,10 +129,10 @@ static void *worker_thread(ks_thread_t *thread, void *data)
ks_status_t status;
status = ks_q_pop_timeout(tp->q, &pop, 1000);
/*
ks_log(KS_LOG_DEBUG, "WORKER %d idle_sec %d running %d dying %d total %d max %d\n",
my_id, idle_sec, tp->running_thread_count, tp->dying_thread_count, tp->thread_count, tp->max);
*/
check_queue(tp, KS_FALSE);
if (status == KS_STATUS_TIMEOUT) {

216
libs/libks/test/testdht2.c
View File

@ -6,6 +6,18 @@
ks_dht_storageitem_skey_t sk;
ks_dht_storageitem_pkey_t pk;
ks_status_t dht2_updated_callback(ks_dht_t *dht, ks_dht_storageitem_t *item)
{
diag("dht2_updated_callback\n");
return KS_STATUS_SUCCESS;
}
ks_status_t dht2_distribute_callback(ks_dht_t *dht, ks_dht_storageitem_t *item)
{
diag("dht2_distribute_callback\n");
return KS_STATUS_SUCCESS;
}
ks_status_t dht2_put_callback(ks_dht_t *dht, ks_dht_job_t *job)
{
diag("dht2_put_callback\n");
@ -28,13 +40,14 @@ ks_status_t dht2_get_token_callback(ks_dht_t *dht, ks_dht_job_t *job)
mutable->sk = sk;
ks_dht_storageitems_insert(dht, mutable);
ks_dht_put(dht, &job->raddr, dht2_put_callback, &job->response_token, 0, mutable);
ks_dht_put(dht, &job->raddr, dht2_put_callback, NULL, &job->response_token, 0, mutable);
return KS_STATUS_SUCCESS;
}
ks_status_t dht2_search_findnode_callback(ks_dht_t *dht, ks_dht_search_t *search)
ks_status_t dht2_search_callback(ks_dht_t *dht, ks_dht_job_t *job)
{
diag("dht2_search_findnode_callback %d\n", search->results_length);
ks_dht_search_t *search = (ks_dht_search_t *)job->data;
diag("dht2_search_callback %d\n", search->results_length);
return KS_STATUS_SUCCESS;
}
@ -54,11 +67,12 @@ int main() {
ks_sockaddr_t raddr1;
//ks_sockaddr_t raddr2;
//ks_sockaddr_t raddr3;
//ks_dht_nodeid_t target;
ks_dht_nodeid_t target;
//ks_dht_storageitem_t *immutable = NULL;
//ks_dht_storageitem_t *mutable = NULL;
//const char *v = "Hello World!";
//size_t v_len = strlen(v);
ks_dht_storageitem_t *mutable1 = NULL;
ks_dht_storageitem_t *mutable2 = NULL;
const char *v = "Hello World!";
size_t v_len = strlen(v);
//ks_dht_storageitem_skey_t sk; //= { { 0xe0, 0x6d, 0x31, 0x83, 0xd1, 0x41, 0x59, 0x22, 0x84, 0x33, 0xed, 0x59, 0x92, 0x21, 0xb8, 0x0b,
//0xd0, 0xa5, 0xce, 0x83, 0x52, 0xe4, 0xbd, 0xf0, 0x26, 0x2f, 0x76, 0x78, 0x6e, 0xf1, 0xc7, 0x4d,
//0xb7, 0xe7, 0xa9, 0xfe, 0xa2, 0xc0, 0xeb, 0x26, 0x9d, 0x61, 0xe3, 0xb3, 0x8e, 0x45, 0x0a, 0x22,
@ -67,7 +81,7 @@ int main() {
//0x24, 0x32, 0xfc, 0xd9, 0x04, 0xa4, 0x35, 0x11, 0x87, 0x6d, 0xf5, 0xcd, 0xf3, 0xe7, 0xe5, 0x48 } };
//uint8_t sk1[KS_DHT_STORAGEITEM_SKEY_SIZE];
//uint8_t pk1[KS_DHT_STORAGEITEM_PKEY_SIZE];
//ks_dht_storageitem_signature_t sig;
ks_dht_storageitem_signature_t sig;
//char sk_buf[KS_DHT_STORAGEITEM_SKEY_SIZE * 2 + 1];
//char pk_buf[KS_DHT_STORAGEITEM_PKEY_SIZE * 2 + 1];
//const char *test1vector = "3:seqi1e1:v12:Hello World!";
@ -155,7 +169,7 @@ int main() {
diag("Ping test\n");
ks_dht_ping(dht2, &raddr1, NULL); // (QUERYING)
ks_dht_ping(dht2, &raddr1, NULL, NULL); // (QUERYING)
ks_dht_pulse(dht2, 100); // Send queued ping from dht2 to dht1 (RESPONDING)
@ -178,7 +192,7 @@ int main() {
ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
ks_dht_ping(dht3, &raddr1, NULL); // (QUERYING)
ks_dht_ping(dht3, &raddr1, NULL, NULL); // (QUERYING)
ks_dht_pulse(dht3, 100); // Send queued ping from dht3 to dht1 (RESPONDING)
@ -193,80 +207,19 @@ int main() {
ks_dht_pulse(dht3, 100); // Call finish callback and purge the job (COMPLETING)
diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
for (int i = 0; i < 20; ++i) {
for (int i = 0; i < 10; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
ok(ks_dhtrt_find_node(dht1->rt_ipv4, ep2->nodeid) != NULL); // The node should be good by now, and thus be returned as good
//diag("Get test\n");
/*
ks_dht_storageitem_target_immutable((uint8_t *)v, v_len, &target);
ks_dht_storageitem_create_immutable(&immutable, dht1->pool, &target, (uint8_t *)v, v_len);
ks_dht_storageitems_insert(dht1, immutable);
*/
/*
crypto_sign_keypair(pk.key, sk.key);
ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 1, (uint8_t *)v, v_len);
ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
ks_dht_storageitem_create_mutable(&mutable, dht1->pool, &target, (uint8_t *)v, v_len, &pk, NULL, 0, 1, &sig);
mutable->sk = sk;
ks_dht_storageitems_insert(dht1, mutable);
ks_dht_get(dht2, &raddr1, dht2_get_callback, &target, NULL, 0);
ks_dht_pulse(dht2, 100); // send get query
ks_dht_pulse(dht1, 100); // receive get query and send get response
ks_dht_pulse(dht2, 100); // receive get response
ok(ks_dht_storageitems_find(dht2, &target) != NULL); // item should be verified and stored
ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
*/
/*
diag("Put test\n");
crypto_sign_keypair(pk.key, sk.key);
ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
ks_dht_get(dht2, NULL, &raddr1, dht2_get_token_callback, &target, NULL, 0); // create job
ks_dht_pulse(dht2, 100); // send get query
ks_dht_pulse(dht1, 100); // receive get query and send get response
ks_dht_pulse(dht2, 100); // receive get response
ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING), send put query
ks_dht_pulse(dht1, 100); // receive put query and send put response
ks_dht_pulse(dht2, 100); // receive put response
ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
for (int i = 0; i < 10; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
*/
// Test bootstrap find_node from dht3 to dht1 to find dht2 nodeid
/*
diag("Find_Node test\n");
ks_dht_findnode(dht3, NULL, &raddr1, NULL, &ep2->nodeid);
ks_dht_findnode(dht3, NULL, &raddr1, NULL, NULL, &ep2->nodeid);
ks_dht_pulse(dht3, 100); // Send queued findnode from dht3 to dht1
@ -290,14 +243,129 @@ int main() {
*/
diag("Search test\n");
ks_dht_search_findnode(dht3, AF_INET, &ep2->nodeid, dht2_search_findnode_callback, NULL);
ks_dht_search(dht3, dht2_search_callback, NULL, dht3->rt_ipv4, &ep2->nodeid);
diag("Pulsing for route table pings\n"); // Wait for route table pinging to catch up
for (int i = 0; i < 20; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
//diag("Get test\n");
/*
ks_dht_storageitem_target_immutable((uint8_t *)v, v_len, &target);
ks_dht_storageitem_create_immutable(&immutable, dht1->pool, &target, (uint8_t *)v, v_len);
ks_dht_storageitems_insert(dht1, immutable);
*/
/*
crypto_sign_keypair(pk.key, sk.key);
ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 1, (uint8_t *)v, v_len);
ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
ks_dht_storageitem_create_mutable(&mutable, dht1->pool, &target, (uint8_t *)v, v_len, &pk, NULL, 0, 1, &sig);
mutable->sk = sk;
ks_dht_storageitems_insert(dht1, mutable);
ks_dht_get(dht2, &raddr1, dht2_get_callback, NULL, &target, NULL, 0);
ks_dht_pulse(dht2, 100); // send get query
ks_dht_pulse(dht1, 100); // receive get query and send get response
ks_dht_pulse(dht2, 100); // receive get response
ok(ks_dht_storageitems_find(dht2, &target) != NULL); // item should be verified and stored
ks_dht_pulse(dht2, 100); // Call finish callback and purge the job (COMPLETING)
*/
/*
diag("Put test\n");
crypto_sign_keypair(pk.key, sk.key);
ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
ks_dht_get(dht2, &raddr1, dht2_get_token_callback, NULL, &target, NULL, 0); // create job
for (int i = 0; i < 20; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
*/
/*
diag("Publish test\n");
crypto_sign_keypair(pk.key, sk.key);
ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 1, (uint8_t *)v, v_len);
ks_dht_storageitem_create_mutable(&mutable, dht2->pool, &target, (uint8_t *)v, v_len, &pk, NULL, 0, 1, &sig);
mutable->sk = sk;
ks_dht_storageitems_insert(dht2, mutable);
ks_dht_publish(dht2, &raddr1, dht2_put_callback, NULL, 0, mutable); // create job
for (int i = 0; i < 20; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
*/
diag("Distribute test\n");
crypto_sign_keypair(pk.key, sk.key);
ks_dht_storageitem_target_mutable(&pk, NULL, 0, &target);
ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 1, (uint8_t *)v, v_len);
ks_dht_storageitem_create_mutable(&mutable2, dht2->pool, &target, (uint8_t *)v, v_len, &pk, NULL, 0, 1, &sig);
mutable2->sk = sk;
ks_dht_storageitems_insert(dht2, mutable2);
ks_dht_distribute(dht2, dht2_distribute_callback, NULL, dht2->rt_ipv4, 0, mutable2); // create job
for (int i = 0; i < 30; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
ks_dht_storageitem_dereference(mutable2);
ok(mutable2->refc == 0);
mutable1 = ks_dht_storageitems_find(dht1, &target);
ok(mutable1 != NULL);
ks_dht_storageitem_callback(mutable1, dht2_updated_callback);
ks_dht_storageitem_callback(mutable2, dht2_updated_callback);
ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 2, (uint8_t *)v, v_len);
mutable1->seq = 2;
mutable1->sig = sig;
//ks_dht_storageitem_signature_generate(&sig, &sk, NULL, 0, 2, (uint8_t *)v, v_len);
//mutable2->seq = 2;
//mutable2->sig = sig;
ks_dht_distribute(dht2, dht2_distribute_callback, NULL, dht2->rt_ipv4, 0, mutable2);
for (int i = 0; i < 30; ++i) {
ks_dht_pulse(dht1, 100);
ks_dht_pulse(dht2, 100);
ks_dht_pulse(dht3, 100);
}
ks_dht_storageitem_dereference(mutable1);
/* Cleanup and shutdown */
diag("Cleanup\n");