- Add --with-libhiredis, unbound support for a new cached backend

that uses a Redis server as the storage.  This implementation
  depends on the hiredis client library (https://redislabs.com/lp/hiredis/).
  And unbound should be built with both --enable-cachedb and
  --with-libhiredis[=PATH] (where $PATH/include/hiredis/hiredis.h
  should exist).  Patch from Jinmei Tatuya (Infoblox).


git-svn-id: file:///svn/unbound/trunk@4586 be551aaa-1e26-0410-a405-d3ace91eadb9
This commit is contained in:
Wouter Wijngaards 2018-03-15 12:33:51 +00:00
parent f7fe1a1093
commit e784758a21
19 changed files with 3408 additions and 2635 deletions

View File

@ -124,7 +124,7 @@ validator/val_nsec3.c validator/val_nsec.c validator/val_secalgo.c \
validator/val_sigcrypt.c validator/val_utils.c dns64/dns64.c \
edns-subnet/edns-subnet.c edns-subnet/subnetmod.c \
edns-subnet/addrtree.c edns-subnet/subnet-whitelist.c \
cachedb/cachedb.c respip/respip.c $(CHECKLOCK_SRC) \
cachedb/cachedb.c cachedb/redis.c respip/respip.c $(CHECKLOCK_SRC) \
$(DNSTAP_SRC) $(DNSCRYPT_SRC) $(IPSECMOD_SRC)
COMMON_OBJ_WITHOUT_NETCALL=dns.lo infra.lo rrset.lo dname.lo msgencode.lo \
as112.lo msgparse.lo msgreply.lo packed_rrset.lo iterator.lo iter_delegpt.lo \
@ -135,7 +135,7 @@ fptr_wlist.lo locks.lo log.lo mini_event.lo module.lo net_help.lo \
random.lo rbtree.lo regional.lo rtt.lo dnstree.lo lookup3.lo lruhash.lo \
slabhash.lo timehist.lo tube.lo winsock_event.lo autotrust.lo val_anchor.lo \
validator.lo val_kcache.lo val_kentry.lo val_neg.lo val_nsec3.lo val_nsec.lo \
val_secalgo.lo val_sigcrypt.lo val_utils.lo dns64.lo cachedb.lo authzone.lo\
val_secalgo.lo val_sigcrypt.lo val_utils.lo dns64.lo cachedb.lo redis.lo authzone.lo\
$(SUBNET_OBJ) $(PYTHONMOD_OBJ) $(CHECKLOCK_OBJ) $(DNSTAP_OBJ) $(DNSCRYPT_OBJ) \
$(IPSECMOD_OBJ) respip.lo
COMMON_OBJ_WITHOUT_UB_EVENT=$(COMMON_OBJ_WITHOUT_NETCALL) netevent.lo listen_dnsport.lo \
@ -645,7 +645,8 @@ infra.lo infra.o: $(srcdir)/services/cache/infra.c config.h $(srcdir)/sldns/rrde
rrset.lo rrset.o: $(srcdir)/services/cache/rrset.c config.h $(srcdir)/services/cache/rrset.h \
$(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/testcode/checklocks.h \
$(srcdir)/util/storage/slabhash.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/sldns/rrdef.h \
$(srcdir)/util/config_file.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/regional.h $(srcdir)/util/alloc.h
$(srcdir)/util/config_file.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/regional.h $(srcdir)/util/alloc.h \
$(srcdir)/util/net_help.h
as112.lo as112.o: $(srcdir)/util/as112.c $(srcdir)/util/as112.h
dname.lo dname.o: $(srcdir)/util/data/dname.c config.h $(srcdir)/util/data/dname.h \
$(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/testcode/checklocks.h \
@ -960,11 +961,11 @@ validator.lo validator.o: $(srcdir)/validator/validator.c config.h $(srcdir)/val
$(srcdir)/validator/val_anchor.h $(srcdir)/util/rbtree.h $(srcdir)/validator/val_kcache.h \
$(srcdir)/util/storage/slabhash.h $(srcdir)/validator/val_kentry.h $(srcdir)/validator/val_nsec.h \
$(srcdir)/validator/val_nsec3.h $(srcdir)/validator/val_neg.h $(srcdir)/validator/val_sigcrypt.h \
$(srcdir)/validator/autotrust.h $(srcdir)/services/cache/dns.h $(srcdir)/util/data/dname.h \
$(srcdir)/util/net_help.h $(srcdir)/util/regional.h $(srcdir)/util/config_file.h $(srcdir)/util/fptr_wlist.h \
$(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h \
$(srcdir)/dnscrypt/cert.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h \
$(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h
$(srcdir)/validator/autotrust.h $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/rrset.h \
$(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h $(srcdir)/util/config_file.h \
$(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/dnscrypt/dnscrypt.h \
$(srcdir)/dnscrypt/cert.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h \
$(srcdir)/services/modstack.h $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h
val_kcache.lo val_kcache.o: $(srcdir)/validator/val_kcache.c config.h $(srcdir)/validator/val_kcache.h \
$(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \
$(srcdir)/testcode/checklocks.h $(srcdir)/validator/val_kentry.h $(srcdir)/util/config_file.h \
@ -1054,11 +1055,16 @@ subnet-whitelist.lo subnet-whitelist.o: $(srcdir)/edns-subnet/subnet-whitelist.c
cachedb.lo cachedb.o: $(srcdir)/cachedb/cachedb.c config.h $(srcdir)/cachedb/cachedb.h $(srcdir)/util/module.h \
$(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/testcode/checklocks.h \
$(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \
$(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h \
$(srcdir)/util/config_file.h $(srcdir)/util/data/msgencode.h $(srcdir)/services/cache/dns.h \
$(srcdir)/validator/val_neg.h $(srcdir)/util/rbtree.h $(srcdir)/validator/val_secalgo.h \
$(srcdir)/iterator/iter_utils.h $(srcdir)/iterator/iter_resptype.h $(srcdir)/sldns/parseutil.h \
$(srcdir)/sldns/wire2str.h $(srcdir)/sldns/sbuffer.h
$(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/cachedb/redis.h $(srcdir)/util/regional.h \
$(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/util/data/msgencode.h \
$(srcdir)/services/cache/dns.h $(srcdir)/validator/val_neg.h $(srcdir)/util/rbtree.h \
$(srcdir)/validator/val_secalgo.h $(srcdir)/iterator/iter_utils.h $(srcdir)/iterator/iter_resptype.h \
$(srcdir)/sldns/parseutil.h $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/sbuffer.h
redis.lo redis.o: $(srcdir)/cachedb/redis.c config.h $(srcdir)/cachedb/redis.h $(srcdir)/cachedb/cachedb.h \
$(srcdir)/util/module.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \
$(srcdir)/testcode/checklocks.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \
$(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/alloc.h \
$(srcdir)/util/config_file.h $(srcdir)/sldns/sbuffer.h
respip.lo respip.o: $(srcdir)/respip/respip.c config.h $(srcdir)/services/localzone.h $(srcdir)/util/rbtree.h \
$(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/testcode/checklocks.h $(srcdir)/util/storage/dnstree.h \
$(srcdir)/util/module.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/data/msgreply.h \

View File

@ -43,6 +43,7 @@
#include "config.h"
#ifdef USE_CACHEDB
#include "cachedb/cachedb.h"
#include "cachedb/redis.h"
#include "util/regional.h"
#include "util/net_help.h"
#include "util/config_file.h"
@ -56,8 +57,6 @@
#include "sldns/wire2str.h"
#include "sldns/sbuffer.h"
#define CACHEDB_HASHSIZE 256 /* bit hash */
/** the unit test testframe for cachedb, its module state contains
* a cache for a couple queries (in memory). */
struct testframe_moddata {
@ -176,6 +175,10 @@ static struct cachedb_backend testframe_backend = { "testframe",
static struct cachedb_backend*
cachedb_find_backend(const char* str)
{
#ifdef USE_REDIS
if(strcmp(str, redis_backend.name) == 0)
return &redis_backend;
#endif
if(strcmp(str, testframe_backend.name) == 0)
return &testframe_backend;
/* TODO add more backends here */

View File

@ -87,6 +87,8 @@ struct cachedb_backend {
uint8_t*, size_t);
};
#define CACHEDB_HASHSIZE 256 /* bit hash */
/** Init the cachedb module */
int cachedb_init(struct module_env* env, int id);
/** Deinit the cachedb module */

298
cachedb/redis.c Normal file
View File

@ -0,0 +1,298 @@
/*
* cachedb/redis.c - cachedb redis module
*
* Copyright (c) 2018, NLnet Labs. All rights reserved.
*
* This software is open source.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of the NLNET LABS nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/**
* \file
*
* This file contains a module that uses the redis database to cache
* dns responses.
*/
#include "config.h"
#ifdef USE_CACHEDB
#include "cachedb/redis.h"
#include "cachedb/cachedb.h"
#include "util/alloc.h"
#include "util/config_file.h"
#include "sldns/sbuffer.h"
/* header file for htobe64 */
#ifdef HAVE_ENDIAN_H
# include <endian.h>
#endif
#ifdef HAVE_SYS_ENDIAN_H
# include <sys/endian.h>
#endif
#ifdef HAVE_LIBKERN_OSBYTEORDER_H
/* In practice this is specific to MacOS X. We assume it doesn't have
* htobe64/be64toh but has alternatives with a different name. */
# include <libkern/OSByteOrder.h>
# define htobe64(x) OSSwapHostToBigInt64(x)
# define be64toh(x) OSSwapBigToHostInt64(x)
#endif
#ifdef USE_REDIS
#include "hiredis/hiredis.h"
struct redis_moddata {
redisContext** ctxs; /* thread-specific redis contexts */
int numctxs; /* number of ctx entries */
const char* server_host; /* server's IP address or host name */
int server_port; /* server's TCP port */
struct timeval timeout; /* timeout for connection setup and commands */
};
static redisContext*
redis_connect(const struct redis_moddata* moddata)
{
redisContext* ctx;
ctx = redisConnectWithTimeout(moddata->server_host,
moddata->server_port, moddata->timeout);
if(!ctx || ctx->err) {
const char *errstr = "out of memory";
if(ctx)
errstr = ctx->errstr;
log_err("failed to connect to redis server: %s", errstr);
goto fail;
}
if(redisSetTimeout(ctx, moddata->timeout) != REDIS_OK) {
log_err("failed to set redis timeout");
goto fail;
}
return ctx;
fail:
if(ctx)
redisFree(ctx);
return NULL;
}
static int
redis_init(struct module_env* env, struct cachedb_env* cachedb_env)
{
int i;
struct redis_moddata* moddata = NULL;
verbose(VERB_ALGO, "redis_init");
moddata = calloc(1, sizeof(struct redis_moddata));
if(!moddata) {
log_err("out of memory");
return 0;
}
moddata->numctxs = env->cfg->num_threads;
moddata->ctxs = calloc(env->cfg->num_threads, sizeof(redisContext*));
if(!moddata->ctxs) {
log_err("out of memory");
free(moddata);
return 0;
}
/* note: server_host is a shallow reference to configured string.
* we don't have to free it in this module. */
moddata->server_host = env->cfg->redis_server_host;
moddata->server_port = env->cfg->redis_server_port;
moddata->timeout.tv_sec = env->cfg->redis_timeout / 1000;
moddata->timeout.tv_usec = (env->cfg->redis_timeout % 1000) * 1000;
for(i = 0; i < moddata->numctxs; i++)
moddata->ctxs[i] = redis_connect(moddata);
cachedb_env->backend_data = moddata;
return 1;
}
static void
redis_deinit(struct module_env* env, struct cachedb_env* cachedb_env)
{
struct redis_moddata* moddata = (struct redis_moddata*)
cachedb_env->backend_data;
(void)env;
verbose(VERB_ALGO, "redis_deinit");
if(!moddata)
return;
if(moddata->ctxs) {
int i;
for(i = 0; i < moddata->numctxs; i++) {
if(moddata->ctxs[i])
redisFree(moddata->ctxs[i]);
}
free(moddata->ctxs);
}
free(moddata);
}
/*
* Send a redis command and get a reply. Unified so that it can be used for
* both SET and GET. If 'data' is non-NULL the command is supposed to be
* SET and GET otherwise, but the implementation of this function is agnostic
* about the semantics (except for logging): 'command', 'data', and 'data_len'
* are opaquely passed to redisCommand().
* This function first checks whether a connection with a redis server has
* been established; if not it tries to set up a new one.
* It returns redisReply returned from redisCommand() or NULL if some low
* level error happens. The caller is responsible to check the return value,
* if it's non-NULL, it has to free it with freeReplyObject().
*/
static redisReply*
redis_command(struct module_env* env, struct cachedb_env* cachedb_env,
const char* command, const uint8_t* data, size_t data_len)
{
redisContext* ctx;
redisReply* rep;
struct redis_moddata* d = (struct redis_moddata*)
cachedb_env->backend_data;
/* We assume env->alloc->thread_num is a unique ID for each thread
* in [0, num-of-threads). We could treat it as an error condition
* if the assumption didn't hold, but it seems to be a fundamental
* assumption throughout the unbound architecture, so we simply assert
* it. */
log_assert(env->alloc->thread_num < d->numctxs);
ctx = d->ctxs[env->alloc->thread_num];
/* If we've not established a connection to the server or we've closed
* it on a failure, try to re-establish a new one. Failures will be
* logged in redis_connect(). */
if(!ctx) {
ctx = redis_connect(d);
d->ctxs[env->alloc->thread_num] = ctx;
}
if(!ctx)
return NULL;
/* Send the command and get a reply, synchronously. */
rep = (redisReply*)redisCommand(ctx, command, data, data_len);
if(!rep) {
/* Once an error as a NULL-reply is returned the context cannot
* be reused and we'll need to set up a new connection. */
log_err("redis_command: failed to receive a reply, "
"closing connection: %s", ctx->errstr);
redisFree(ctx);
d->ctxs[env->alloc->thread_num] = NULL;
return NULL;
}
/* Check error in reply to unify logging in that case.
* The caller may perform context-dependent checks and logging. */
if(rep->type == REDIS_REPLY_ERROR)
log_err("redis: %s resulted in an error: %s",
data ? "set" : "get", rep->str);
return rep;
}
static int
redis_lookup(struct module_env* env, struct cachedb_env* cachedb_env,
char* key, struct sldns_buffer* result_buffer)
{
redisReply* rep;
char cmdbuf[4+(CACHEDB_HASHSIZE/8)*2+1]; /* "GET " + key */
int n;
int ret = 0;
verbose(VERB_ALGO, "redis_lookup of %s", key);
n = snprintf(cmdbuf, sizeof(cmdbuf), "GET %s", key);
if(n < 0 || n >= (int)sizeof(cmdbuf)) {
log_err("redis_lookup: unexpected failure to build command");
return 0;
}
rep = redis_command(env, cachedb_env, cmdbuf, NULL, 0);
if(!rep)
return 0;
switch (rep->type) {
case REDIS_REPLY_NIL:
verbose(VERB_ALGO, "redis_lookup: no data cached");
break;
case REDIS_REPLY_STRING:
verbose(VERB_ALGO, "redis_lookup found %d bytes",
(int)rep->len);
if((size_t)rep->len > sldns_buffer_capacity(result_buffer)) {
log_err("redis_lookup: replied data too long: %lu",
(size_t)rep->len);
break;
}
sldns_buffer_clear(result_buffer);
sldns_buffer_write(result_buffer, rep->str, rep->len);
sldns_buffer_flip(result_buffer);
ret = 1;
break;
case REDIS_REPLY_ERROR:
break; /* already logged */
default:
log_err("redis_lookup: unexpected type of reply for (%d)",
rep->type);
break;
}
freeReplyObject(rep);
return ret;
}
static void
redis_store(struct module_env* env, struct cachedb_env* cachedb_env,
char* key, uint8_t* data, size_t data_len)
{
redisReply* rep;
char cmdbuf[4+(CACHEDB_HASHSIZE/8)*2+3+1]; /* "SET " + key + " %b" */
int n;
verbose(VERB_ALGO, "redis_store %s (%d bytes)", key, (int)data_len);
/* build command to set to a binary safe string */
n = snprintf(cmdbuf, sizeof(cmdbuf), "SET %s %%b", key);
if(n < 0 || n >= (int)sizeof(cmdbuf)) {
log_err("redis_store: unexpected failure to build command");
return;
}
rep = redis_command(env, cachedb_env, cmdbuf, data, data_len);
if(rep) {
verbose(VERB_ALGO, "redis_store set completed");
if(rep->type != REDIS_REPLY_STATUS &&
rep->type != REDIS_REPLY_ERROR) {
log_err("redis_store: unexpected type of reply (%d)",
rep->type);
}
freeReplyObject(rep);
}
}
struct cachedb_backend redis_backend = { "redis",
redis_init, redis_deinit, redis_lookup, redis_store
};
#endif /* USE_REDIS */
#endif /* USE_CACHEDB */

45
cachedb/redis.h Normal file
View File

@ -0,0 +1,45 @@
/*
* cachedb/redis.h - cachedb redis module
*
* Copyright (c) 2018, NLnet Labs. All rights reserved.
*
* This software is open source.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of the NLNET LABS nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/**
* \file
*
* This file contains a module that uses the redis database to cache
* dns responses.
*/
/** the redis backend definition, contains callable functions
* and name string */
extern struct cachedb_backend redis_backend;

View File

@ -95,6 +95,10 @@
don't. */
#undef HAVE_DECL_REALLOCARRAY
/* Define to 1 if you have the declaration of `redisConnect', and to 0 if you
don't. */
#undef HAVE_DECL_REDISCONNECT
/* Define to 1 if you have the declaration of `sk_SSL_COMP_pop_free', and to 0
if you don't. */
#undef HAVE_DECL_SK_SSL_COMP_POP_FREE
@ -233,6 +237,9 @@
/* Define to 1 if you have the <grp.h> header file. */
#undef HAVE_GRP_H
/* Define to 1 if you have the <hiredis/hiredis.h> header file. */
#undef HAVE_HIREDIS_HIREDIS_H
/* If you have HMAC_Update */
#undef HAVE_HMAC_UPDATE
@ -263,6 +270,9 @@
/* Define to 1 if you have the `kill' function. */
#undef HAVE_KILL
/* Define to 1 if you have the <libkern/OSByteOrder.h> header file. */
#undef HAVE_LIBKERN_OSBYTEORDER_H
/* Define if we have LibreSSL */
#undef HAVE_LIBRESSL
@ -479,6 +489,9 @@
/* Define to 1 if systemd should be used */
#undef HAVE_SYSTEMD
/* Define to 1 if you have the <sys/endian.h> header file. */
#undef HAVE_SYS_ENDIAN_H
/* Define to 1 if you have the <sys/ipc.h> header file. */
#undef HAVE_SYS_IPC_H
@ -718,6 +731,9 @@
/* Define this to enable client TCP Fast Open. */
#undef USE_OSX_MSG_FASTOPEN
/* Define this to use hiredis client. */
#undef USE_REDIS
/* Define this to enable SHA1 support. */
#undef USE_SHA1

68
configure vendored
View File

@ -864,6 +864,7 @@ enable_tfo_client
enable_tfo_server
with_libevent
with_libexpat
with_libhiredis
enable_static_exe
enable_systemd
enable_lock_checks
@ -1610,6 +1611,7 @@ Optional Packages:
an explicit path). Slower, but allows use of large
outgoing port ranges.
--with-libexpat=path specify explicit path for libexpat.
--with-libhiredis=path specify explicit path for libhiredis.
--with-dnstap-socket-path=pathname
set default dnstap socket path
--with-protobuf-c=path Path where protobuf-c is installed, for dnstap
@ -14478,7 +14480,7 @@ CC=$lt_save_CC
# Checks for header files.
for ac_header in stdarg.h stdbool.h netinet/in.h netinet/tcp.h sys/param.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h sys/ipc.h sys/shm.h
for ac_header in stdarg.h stdbool.h netinet/in.h netinet/tcp.h sys/param.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h sys/endian.h libkern/OSByteOrder.h sys/ipc.h sys/shm.h
do :
as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default
@ -18811,6 +18813,70 @@ cat >>confdefs.h <<_ACEOF
_ACEOF
# hiredis (redis C client for cachedb)
# Check whether --with-libhiredis was given.
if test "${with_libhiredis+set}" = set; then :
withval=$with_libhiredis;
else
withval="no"
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for libhiredis" >&5
$as_echo_n "checking for libhiredis... " >&6; }
found_libhiredis="no"
if test x_$withval = x_yes -o x_$withval != x_no; then
if test x_$withval = x_ -o x_$withval = x_yes; then
withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr"
fi
for dir in $withval ; do
if test -f "$dir/include/hiredis/hiredis.h"; then
found_libhiredis="yes"
if test "$dir" != "/usr"; then
CPPFLAGS="$CPPFLAGS -I$dir/include"
LDFLAGS="$LDFLAGS -L$dir/lib"
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: found in $dir" >&5
$as_echo "found in $dir" >&6; }
$as_echo "#define USE_REDIS 1" >>confdefs.h
LIBS="$LIBS -lhiredis"
break;
fi
done
if test x_$found_libhiredis != x_yes; then
as_fn_error $? "Could not find libhiredis, hiredis.h" "$LINENO" 5
fi
for ac_header in hiredis/hiredis.h
do :
ac_fn_c_check_header_compile "$LINENO" "hiredis/hiredis.h" "ac_cv_header_hiredis_hiredis_h" "$ac_includes_default
"
if test "x$ac_cv_header_hiredis_hiredis_h" = xyes; then :
cat >>confdefs.h <<_ACEOF
#define HAVE_HIREDIS_HIREDIS_H 1
_ACEOF
fi
done
ac_fn_c_check_decl "$LINENO" "redisConnect" "ac_cv_have_decl_redisConnect" "$ac_includes_default
#include <hiredis/hiredis.h>
"
if test "x$ac_cv_have_decl_redisConnect" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL_REDISCONNECT $ac_have_decl
_ACEOF
fi
# set static linking if requested
staticexe=""

View File

@ -332,7 +332,7 @@ AC_CHECK_TOOL(STRIP, strip)
ACX_LIBTOOL_C_ONLY
# Checks for header files.
AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h netinet/tcp.h sys/param.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h sys/ipc.h sys/shm.h],,, [AC_INCLUDES_DEFAULT])
AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h netinet/tcp.h sys/param.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h sys/endian.h libkern/OSByteOrder.h sys/ipc.h sys/shm.h],,, [AC_INCLUDES_DEFAULT])
# check for types.
# Using own tests for int64* because autoconf builtin only give 32bit.
@ -1151,6 +1151,39 @@ AC_CHECK_DECLS([XML_StopParser], [], [], [AC_INCLUDES_DEFAULT
#include <expat.h>
])
# hiredis (redis C client for cachedb)
AC_ARG_WITH(libhiredis, AC_HELP_STRING([--with-libhiredis=path],
[specify explicit path for libhiredis.]),
[ ],[ withval="no" ])
AC_MSG_CHECKING(for libhiredis)
found_libhiredis="no"
if test x_$withval = x_yes -o x_$withval != x_no; then
if test x_$withval = x_ -o x_$withval = x_yes; then
withval="/usr/local /opt/local /usr/lib /usr/pkg /usr/sfw /usr"
fi
for dir in $withval ; do
if test -f "$dir/include/hiredis/hiredis.h"; then
found_libhiredis="yes"
dnl assume /usr is in default path.
if test "$dir" != "/usr"; then
CPPFLAGS="$CPPFLAGS -I$dir/include"
LDFLAGS="$LDFLAGS -L$dir/lib"
fi
AC_MSG_RESULT(found in $dir)
AC_DEFINE([USE_REDIS], [1], [Define this to use hiredis client.])
LIBS="$LIBS -lhiredis"
break;
fi
done
if test x_$found_libhiredis != x_yes; then
AC_ERROR([Could not find libhiredis, hiredis.h])
fi
AC_CHECK_HEADERS([hiredis/hiredis.h],,, [AC_INCLUDES_DEFAULT])
AC_CHECK_DECLS([redisConnect], [], [], [AC_INCLUDES_DEFAULT
#include <hiredis/hiredis.h>
])
fi
# set static linking if requested
AC_SUBST(staticexe)
staticexe=""

View File

@ -35,3 +35,6 @@ distribution but may be helpful.
instead of SERVFAIL. Contributed by SIDN.
* fastrpz.patch: fastrpz support from Farsight Security.
* libunbound.so.conf: ltrace.conf file, see ltrace.conf(5), for libunbound.
* unbound-querycachedb.py: utility to show data stored in cachedb backend
for a particular query name and type. It requires dnspython and (for
redis backend) redis Python modules.

View File

@ -0,0 +1,77 @@
#!/usr/bin/env python
import hashlib
import sys
import struct
import socket
import time
from optparse import OptionParser
import dns.message
import dns.name
import dns.rdataclass
import dns.rdatatype
def _calc_hashkey(qname, secret, qtype):
qclass = 'IN' # CLASS is fixed for simplicity
hobj = hashlib.sha256()
hobj.update(dns.name.from_text(qname).to_wire())
hobj.update(struct.pack('HH',
socket.htons(dns.rdatatype.from_text(qtype)),
socket.htons(dns.rdataclass.from_text(qclass))))
hobj.update(secret)
return hobj.hexdigest().upper()
def _redis_get(options, key):
import redis
return redis.Redis(options.address, int(options.port)).get(key)
def _dump_value(options, qname, key, value):
print(';; query=%s/IN/%s' % (qname, options.qtype))
print(';; key=%s' % key)
if value is None:
print(';; no value')
return
if len(value) < 16:
print(';; broken value, short length: %d' % len(value))
return
now = int(time.time())
timestamp = struct.unpack('!Q', value[-16:-8])[0]
expire = struct.unpack('!Q', value[-8:])[0]
print(';; Now=%d, TimeStamp=%d, Expire=%d, TTL=%d' %
(now, timestamp, expire, max(expire - now, 0)))
print(dns.message.from_wire(value[:-16]))
def main():
parser = OptionParser(usage='usage: %prog [options] query_name')
parser.add_option("-a", "--address", dest="address", action="store",
default='127.0.0.1', help="backend-server address",
metavar='ADDRESS')
parser.add_option("-b", "--backend", dest="backend", action="store",
default='redis', help="backend name",
metavar='BACKEND')
parser.add_option("-p", "--port", dest="port", action="store",
default='6379', help="backend-server port",
metavar='PORT')
parser.add_option("-s", "--secret", dest="secret", action="store",
default='default', help="secret seed", metavar='SECRET')
parser.add_option("-t", "--qtype", dest="qtype", action="store",
default='A', help="query RR type", metavar='QTYPE')
(options, args) = parser.parse_args()
if len(args) < 1:
parser.error('qname is missing')
if options.backend == 'redis':
get_func = _redis_get
else:
raise Exception('unknown backend name: %s\n' % options.backend)
key = _calc_hashkey(args[0], options.secret, options.qtype)
value = get_func(options, key)
_dump_value(options, args[0], key, value)
if __name__ == '__main__':
try:
main()
except Exception as e:
sys.stderr.write('%s\n' % e)
exit(1)

View File

@ -1,3 +1,11 @@
15 March 2018: Wouter
- Add --with-libhiredis, unbound support for a new cached backend
that uses a Redis server as the storage. This implementation
depends on the hiredis client library (https://redislabs.com/lp/hiredis/).
And unbound should be built with both --enable-cachedb and
--with-libhiredis[=PATH] (where $PATH/include/hiredis/hiredis.h
should exist). Patch from Jinmei Tatuya (Infoblox).
13 March 2018: Wouter
- Fix typo in documentation.
- Fix #3736: Fix 0 TTL domains stuck on SERVFAIL unless manually

View File

@ -886,3 +886,11 @@ remote-control:
# backend: "testframe"
# # secret seed string to calculate hashed keys
# secret-seed: "default"
#
# # For "redis" backend:
# # redis server's IP address or host name
# redis-server-host: 127.0.0.1
# # redis server's TCP port
# redis-server-port: 6379
# # timeout (in ms) for communication with the redis server
# redis-timeout: 100

View File

@ -1781,15 +1781,42 @@ If it finds a valid answer in the backend, Unbound uses it to respond
to the query without performing iterative DNS resolution.
If Unbound cannot even find an answer in the backend, it resolves the
query as usual, and stores the answer in the backend.
.P
If Unbound was built with
\fB\-\-with\-libhiredis\fR
on a system that has installed the hiredis C client library of Redis,
then the "redis" backend can be used.
This backend communicates with the specified Redis server over a TCP
connection to store and retrive cache data.
It can be used as a persistent and/or shared cache backend.
It should be noted that Unbound never removes data stored in the Redis server,
even if some data have expired in terms of DNS TTL or the Redis server has
cached too much data;
if necessary the Redis server must be configured to limit the cache size,
preferably with some kind of least-recently-used eviction policy.
This backend uses synchronous communication with the Redis server
based on the assumption that the communication is stable and sufficiently
fast.
The thread waiting for a response from the Redis server cannot handle
other DNS queries.
Although the backend has the ability to reconnect to the server when
the connection is closed unexpectedly and there is a configurable timeout
in case the server is overly slow or hangs up, these cases are assumed
to be very rare.
If connection close or timeout happens too often, Unbound will be
effectively unusable with this backend.
It's the administrator's responsibility to make the assumption hold.
.P
The
.B cachedb:
clause gives custom settings of the cache DB module.
.TP
.B backend: \fI<backend name>\fR
Specify the backend database name.
Currently, only the in-memory "testframe" backend is supported.
As the name suggests this backend is not of any practical use.
This option defaults to "testframe".
The default database is the in-memory backend named "testframe", which,
as the name suggests, is not of any practical use.
Depending on the build-time configuration, "redis" backend may also be
used as described above.
.TP
.B secret-seed: \fI<"secret string">\fR
Specify a seed to calculate a hash value from query information.
@ -1799,6 +1826,28 @@ operationally.
If the backend database is shared by multiple Unbound instances,
all instances must use the same secret seed.
This option defaults to "default".
.P
The following
.B cachedb
otions are specific to the redis backend.
.TP
.B redis-server-host: \fI<server address or name>\fR
The IP (either v6 or v4) address or domain name of the Redis server.
In general an IP address should be specified as otherwise Unbound will have to
resolve the name of the server every time it establishes a connection
to the server.
This option defaults to "127.0.0.1".
.TP
.B redis-server-port: \fI<port number>\fR
The TCP port number of the Redis server.
This option defaults to 6379.
.TP
.B redis-timeout: \fI<msec>\fR
The period until when Unbound waits for a response from the Redis sever.
If this timeout expires Unbound closes the connection, treats it as
if the Redis server does not have the requested data, and will try to
re-establish a new connection later.
This option defaults to 100 milliseconds.
.SH "MEMORY CONTROL EXAMPLE"
In the example config settings below memory usage is reduced. Some service
levels are lower, notable very large data and a high TCP load are no longer

View File

@ -507,6 +507,14 @@ struct config_file {
char* cachedb_backend;
/** secret seed for hash key calculation */
char* cachedb_secret;
#ifdef USE_REDIS
/** redis server's IP address or host name */
char* redis_server_host;
/** redis server's TCP port */
int redis_server_port;
/** timeout (in ms) for communication with the redis server */
int redis_timeout;
#endif
#endif
};

File diff suppressed because it is too large Load Diff

View File

@ -448,6 +448,9 @@ ipsecmod-strict{COLON} { YDVAR(1, VAR_IPSECMOD_STRICT) }
cachedb{COLON} { YDVAR(0, VAR_CACHEDB) }
backend{COLON} { YDVAR(1, VAR_CACHEDB_BACKEND) }
secret-seed{COLON} { YDVAR(1, VAR_CACHEDB_SECRETSEED) }
redis-server-host{COLON} { YDVAR(1, VAR_CACHEDB_REDISHOST) }
redis-server-port{COLON} { YDVAR(1, VAR_CACHEDB_REDISPORT) }
redis-timeout{COLON} { YDVAR(1, VAR_CACHEDB_REDISTIMEOUT) }
udp-upstream-without-downstream{COLON} { YDVAR(1, VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM) }
<INITIAL,val>{NEWLINE} { LEXOUT(("NL\n")); cfg_parser->line++; }

File diff suppressed because it is too large Load Diff

View File

@ -268,14 +268,17 @@ extern int yydebug;
VAR_CACHEDB = 478,
VAR_CACHEDB_BACKEND = 479,
VAR_CACHEDB_SECRETSEED = 480,
VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM = 481,
VAR_FOR_UPSTREAM = 482,
VAR_AUTH_ZONE = 483,
VAR_ZONEFILE = 484,
VAR_MASTER = 485,
VAR_URL = 486,
VAR_FOR_DOWNSTREAM = 487,
VAR_FALLBACK_ENABLED = 488
VAR_CACHEDB_REDISHOST = 481,
VAR_CACHEDB_REDISPORT = 482,
VAR_CACHEDB_REDISTIMEOUT = 483,
VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM = 484,
VAR_FOR_UPSTREAM = 485,
VAR_AUTH_ZONE = 486,
VAR_ZONEFILE = 487,
VAR_MASTER = 488,
VAR_URL = 489,
VAR_FOR_DOWNSTREAM = 490,
VAR_FALLBACK_ENABLED = 491
};
#endif
/* Tokens. */
@ -502,14 +505,17 @@ extern int yydebug;
#define VAR_CACHEDB 478
#define VAR_CACHEDB_BACKEND 479
#define VAR_CACHEDB_SECRETSEED 480
#define VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM 481
#define VAR_FOR_UPSTREAM 482
#define VAR_AUTH_ZONE 483
#define VAR_ZONEFILE 484
#define VAR_MASTER 485
#define VAR_URL 486
#define VAR_FOR_DOWNSTREAM 487
#define VAR_FALLBACK_ENABLED 488
#define VAR_CACHEDB_REDISHOST 481
#define VAR_CACHEDB_REDISPORT 482
#define VAR_CACHEDB_REDISTIMEOUT 483
#define VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM 484
#define VAR_FOR_UPSTREAM 485
#define VAR_AUTH_ZONE 486
#define VAR_ZONEFILE 487
#define VAR_MASTER 488
#define VAR_URL 489
#define VAR_FOR_DOWNSTREAM 490
#define VAR_FALLBACK_ENABLED 491
/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
@ -520,7 +526,7 @@ union YYSTYPE
char* str;
#line 524 "util/configparser.h" /* yacc.c:1909 */
#line 530 "util/configparser.h" /* yacc.c:1909 */
};
typedef union YYSTYPE YYSTYPE;

View File

@ -152,6 +152,7 @@ extern struct config_parser_state* cfg_parser;
%token VAR_IPSECMOD_ENABLED VAR_IPSECMOD_HOOK VAR_IPSECMOD_IGNORE_BOGUS
%token VAR_IPSECMOD_MAX_TTL VAR_IPSECMOD_WHITELIST VAR_IPSECMOD_STRICT
%token VAR_CACHEDB VAR_CACHEDB_BACKEND VAR_CACHEDB_SECRETSEED
%token VAR_CACHEDB_REDISHOST VAR_CACHEDB_REDISPORT VAR_CACHEDB_REDISTIMEOUT
%token VAR_UDP_UPSTREAM_WITHOUT_DOWNSTREAM VAR_FOR_UPSTREAM
%token VAR_AUTH_ZONE VAR_ZONEFILE VAR_MASTER VAR_URL VAR_FOR_DOWNSTREAM
%token VAR_FALLBACK_ENABLED
@ -2551,7 +2552,8 @@ cachedbstart: VAR_CACHEDB
;
contents_cachedb: contents_cachedb content_cachedb
| ;
content_cachedb: cachedb_backend_name | cachedb_secret_seed
content_cachedb: cachedb_backend_name | cachedb_secret_seed |
redis_server_host | redis_server_port | redis_timeout
;
cachedb_backend_name: VAR_CACHEDB_BACKEND STRING_ARG
{
@ -2582,6 +2584,46 @@ cachedb_secret_seed: VAR_CACHEDB_SECRETSEED STRING_ARG
#endif
}
;
redis_server_host: VAR_CACHEDB_REDISHOST STRING_ARG
{
#if defined(USE_CACHEDB) && defined(USE_REDIS)
OUTYY(("P(redis_server_host:%s)\n", $2));
free(cfg_parser->cfg->redis_server_host);
cfg_parser->cfg->redis_server_host = $2;
#else
OUTYY(("P(Compiled without cachedb or redis, ignoring)\n"));
free($2);
#endif
}
;
redis_server_port: VAR_CACHEDB_REDISPORT STRING_ARG
{
#if defined(USE_CACHEDB) && defined(USE_REDIS)
int port;
OUTYY(("P(redis_server_port:%s)\n", $2));
port = atoi($2);
if(port == 0 || port < 0 || port > 65535)
yyerror("valid redis server port number expected");
else cfg_parser->cfg->redis_server_port = port;
#else
OUTYY(("P(Compiled without cachedb or redis, ignoring)\n"));
#endif
free($2);
}
;
redis_timeout: VAR_CACHEDB_REDISTIMEOUT STRING_ARG
{
#if defined(USE_CACHEDB) && defined(USE_REDIS)
OUTYY(("P(redis_timeout:%s)\n", $2));
if(atoi($2) == 0)
yyerror("redis timeout value expected");
else cfg_parser->cfg->redis_timeout = atoi($2);
#else
OUTYY(("P(Compiled without cachedb or redis, ignoring)\n"));
#endif
free($2);
}
;
%%
/* parse helper routines could be here */