From 283d6cfd58600a8c9e90584a67f1ca90d85f5669 Mon Sep 17 00:00:00 2001 From: Jim Brunner Date: Wed, 13 Mar 2019 16:31:24 +0000 Subject: [PATCH 001/116] Addition of OnUnload function --- src/module.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/module.c b/src/module.c index 5ad99975..ba38ed8d 100644 --- a/src/module.c +++ b/src/module.c @@ -4799,6 +4799,23 @@ int moduleUnload(sds name) { errno = EBUSY; return REDISMODULE_ERR; } + + /* Give module a chance to clean up. */ + int (*onunload)(void *); + onunload = (int (*)(void *))(unsigned long) dlsym(module->handle, "RedisModule_OnUnload"); + if (onunload) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = module; + ctx.client = moduleFreeContextReusedClient; + int unload_status = onunload((void*)&ctx); + moduleFreeContext(&ctx); + + if (unload_status == REDISMODULE_ERR) { + serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", name); + errno = ECANCELED; + return REDISMODULE_ERR; + } + } moduleUnregisterCommands(module); From 68fd59056b19c930b7b118d1f531f25947e026b8 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 3 Jun 2018 15:37:48 +0300 Subject: [PATCH 002/116] Add RedisModule_Assert() API call. --- src/module.c | 10 ++++++++++ src/redismodule.h | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/src/module.c b/src/module.c index 8954fcdf..c6e6e598 100644 --- a/src/module.c +++ b/src/module.c @@ -3461,6 +3461,15 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ... va_end(ap); } +/* Redis-like assert function. + * + * A failed assertion will shut down the server and produce logging information + * that looks identical to information generated by Redis itself. + */ +void RM__Assert(const char *estr, const char *file, int line) { + _serverAssert(estr, file, line); +} + /* -------------------------------------------------------------------------- * Blocking clients from modules * -------------------------------------------------------------------------- */ @@ -4993,6 +5002,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); + REGISTER_API(_Assert); REGISTER_API(StringAppendBuffer); REGISTER_API(RetainString); REGISTER_API(StringCompare); diff --git a/src/redismodule.h b/src/redismodule.h index d18c3888..db32df04 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -271,6 +271,7 @@ void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value) float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str); int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b); @@ -433,6 +434,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(EmitAOF); REDISMODULE_GET_API(Log); REDISMODULE_GET_API(LogIOError); + REDISMODULE_GET_API(_Assert); REDISMODULE_GET_API(StringAppendBuffer); REDISMODULE_GET_API(RetainString); REDISMODULE_GET_API(StringCompare); @@ -499,6 +501,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int return REDISMODULE_OK; } +#define RedisModule_Assert(_e) ((_e)?(void)0 : (RedisModule__Assert(#_e,__FILE__,__LINE__),exit(1))) + #else /* Things only defined for the modules core, not exported to modules From 52686f48664e8a01e556e6a7ee52013816514a26 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Tue, 16 Apr 2019 22:16:12 +0300 Subject: [PATCH 003/116] Adds a "Modules" section to `INFO` Fixes #6012. As long as "INFO is broken", this should be adequate IMO. Once we rework `INFO`, perhaps into RESP3, this implementation should be revisited. --- src/module.c | 19 +++++++++++++++++++ src/server.c | 7 +++++++ src/server.h | 1 + 3 files changed, 27 insertions(+) diff --git a/src/module.c b/src/module.c index c2952167..60c9a046 100644 --- a/src/module.c +++ b/src/module.c @@ -5244,6 +5244,25 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper function for the INFO command: adds loaded modules as to info's + * output. + * + * After the call, the passed sds info string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds genModulesInfoString(sds info) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + sds name = dictGetKey(de); + struct RedisModule *module = dictGetVal(de); + + info = sdscatprintf(info, "module:name=%s,ver=%d\r\n", name, module->ver); + } + dictReleaseIterator(di); + return info; +} + /* Redis MODULE command. * * MODULE LOAD [args...] */ diff --git a/src/server.c b/src/server.c index fb5d679c..49a65ef5 100644 --- a/src/server.c +++ b/src/server.c @@ -4291,6 +4291,13 @@ sds genRedisInfoString(char *section) { (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec); } + /* Modules */ + if (allsections || defsections || !strcasecmp(section,"modules")) { + if (sections++) info = sdscat(info,"\r\n"); + info = sdscatprintf(info,"# Modules\r\n"); + info = genModulesInfoString(info); + } + /* Command statistics */ if (allsections || !strcasecmp(section,"commandstats")) { if (sections++) info = sdscat(info,"\r\n"); diff --git a/src/server.h b/src/server.h index dfd9f769..d832c646 100644 --- a/src/server.h +++ b/src/server.h @@ -2268,6 +2268,7 @@ void bugReportStart(void); void serverLogObjectDebugInfo(const robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); sds genRedisInfoString(char *section); +sds genModulesInfoString(sds info); void enableWatchdog(int period); void disableWatchdog(void); void watchdogScheduleSignal(int period); From 2fec7d9c6c630db3bcb13a07a08c39404abad447 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 30 May 2019 11:51:58 +0300 Subject: [PATCH 004/116] Jemalloc: Avoid blocking on background thread lock for stats. Background threads may run for a long time, especially when the # of dirty pages is high. Avoid blocking stats calls because of this (which may cause latency spikes). see https://github.com/jemalloc/jemalloc/issues/1502 cherry picked from commit 1a71533511027dbe3f9d989659efeec446915d6b --- deps/jemalloc/src/background_thread.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deps/jemalloc/src/background_thread.c b/deps/jemalloc/src/background_thread.c index 3517a3bb..457669c9 100644 --- a/deps/jemalloc/src/background_thread.c +++ b/deps/jemalloc/src/background_thread.c @@ -787,7 +787,13 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { nstime_init(&stats->run_interval, 0); for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; - malloc_mutex_lock(tsdn, &info->mtx); + if (malloc_mutex_trylock(tsdn, &info->mtx)) { + /* + * Each background thread run may take a long time; + * avoid waiting on the stats if the thread is active. + */ + continue; + } if (info->state != background_thread_stopped) { num_runs += info->tot_n_runs; nstime_add(&stats->run_interval, &info->tot_sleep_time); From 09f99c2a925a0351985e799c106614082d6053cf Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 30 May 2019 12:51:32 +0300 Subject: [PATCH 005/116] make redis purge jemalloc after flush, and enable background purging thread jemalloc 5 doesn't immediately release memory back to the OS, instead there's a decaying mechanism, which doesn't work when there's no traffic (no allocations). this is most evident if there's no traffic after flushdb, the RSS will remain high. 1) enable jemalloc background purging 2) explicitly purge in flushdb --- src/config.c | 9 ++++++++ src/db.c | 14 ++++++++++++ src/debug.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/server.c | 2 ++ src/server.h | 1 + src/zmalloc.c | 34 ++++++++++++++++++++++++++++ src/zmalloc.h | 2 ++ 7 files changed, 124 insertions(+) diff --git a/src/config.c b/src/config.c index 7f0e9af8..16850a1f 100644 --- a/src/config.c +++ b/src/config.c @@ -474,6 +474,10 @@ void loadServerConfigFromString(char *config) { err = "active defrag can't be enabled without proper jemalloc support"; goto loaderr; #endif } + } else if (!strcasecmp(argv[0],"jemalloc-bg-thread") && argc == 2) { + if ((server.jemalloc_bg_thread = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"daemonize") && argc == 2) { if ((server.daemonize = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -1152,6 +1156,9 @@ void configSetCommand(client *c) { return; } #endif + } config_set_bool_field( + "jemalloc-bg-thread",server.jemalloc_bg_thread) { + set_jemalloc_bg_thread(server.jemalloc_bg_thread); } config_set_bool_field( "protected-mode",server.protected_mode) { } config_set_bool_field( @@ -1487,6 +1494,7 @@ void configGetCommand(client *c) { config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); config_get_bool_field("activedefrag", server.active_defrag_enabled); + config_get_bool_field("jemalloc-bg-thread", server.jemalloc_bg_thread); config_get_bool_field("protected-mode", server.protected_mode); config_get_bool_field("gopher-enabled", server.gopher_enabled); config_get_bool_field("io-threads-do-reads", server.io_threads_do_reads); @@ -2318,6 +2326,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES); rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,CONFIG_DEFAULT_ACTIVE_REHASHING); rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); + rewriteConfigYesNoOption(state,"jemalloc-bg-thread",server.jemalloc_bg_thread,1); rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE); rewriteConfigYesNoOption(state,"gopher-enabled",server.gopher_enabled,CONFIG_DEFAULT_GOPHER_ENABLED); rewriteConfigYesNoOption(state,"io-threads-do-reads",server.io_threads_do_reads,CONFIG_DEFAULT_IO_THREADS_DO_READS); diff --git a/src/db.c b/src/db.c index b537a29a..50e23d6b 100644 --- a/src/db.c +++ b/src/db.c @@ -441,6 +441,13 @@ void flushdbCommand(client *c) { signalFlushedDb(c->db->id); server.dirty += emptyDb(c->db->id,flags,NULL); addReply(c,shared.ok); +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif } /* FLUSHALL [ASYNC] @@ -464,6 +471,13 @@ void flushallCommand(client *c) { server.dirty = saved_dirty; } server.dirty++; +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif } /* This command implements DEL and LAZYDEL. */ diff --git a/src/debug.c b/src/debug.c index 0c6b5630..c82c99b1 100644 --- a/src/debug.c +++ b/src/debug.c @@ -297,6 +297,56 @@ void computeDatasetDigest(unsigned char *final) { } } +#ifdef USE_JEMALLOC +void mallctl_int(client *c, robj **argv, int argc) { + int ret; + /* start with the biggest size (int64), and if that fails, try smaller sizes (int32, bool) */ + int64_t old = 0, val; + if (argc > 1) { + long long ll; + if (getLongLongFromObjectOrReply(c, argv[1], &ll, NULL) != C_OK) + return; + val = ll; + } + size_t sz = sizeof(old); + while (sz > 0) { + if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, argc > 1? &val: NULL, argc > 1?sz: 0))) { + if (ret==EINVAL) { + /* size might be wrong, try a smaller one */ + sz /= 2; +#if BYTE_ORDER == BIG_ENDIAN + val <<= 8*sz; +#endif + continue; + } + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } else { +#if BYTE_ORDER == BIG_ENDIAN + old >>= 64 - 8*sz; +#endif + addReplyLongLong(c, old); + return; + } + } + addReplyErrorFormat(c,"%s", strerror(EINVAL)); +} + +void mallctl_string(client *c, robj **argv, int argc) { + int ret; + char *old; + size_t sz = sizeof(old); + /* for strings, it seems we need to first get the old value, before overriding it. */ + if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, NULL, 0))) { + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } + addReplyBulkCString(c, old); + if(argc > 1) + je_mallctl(argv[0]->ptr, NULL, 0, &argv[1]->ptr, sizeof(char*)); +} +#endif + void debugCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { @@ -323,6 +373,10 @@ void debugCommand(client *c) { "STRUCTSIZE -- Return the size of different Redis core C structures.", "ZIPLIST -- Show low level info about the ziplist encoding.", "STRINGMATCH-TEST -- Run a fuzz tester against the stringmatchlen() function.", +#ifdef USE_JEMALLOC +"MALLCTL [] -- Get or set a malloc tunning integer.", +"MALLCTL-STR [] -- Get or set a malloc tunning string.", +#endif NULL }; addReplyHelp(c, help); @@ -676,6 +730,14 @@ NULL { stringmatchlen_fuzz_test(); addReplyStatus(c,"Apparently Redis did not crash: test passed"); +#ifdef USE_JEMALLOC + } else if(!strcasecmp(c->argv[1]->ptr,"mallctl") && c->argc >= 3) { + mallctl_int(c, c->argv+2, c->argc-2); + return; + } else if(!strcasecmp(c->argv[1]->ptr,"mallctl-str") && c->argc >= 3) { + mallctl_string(c, c->argv+2, c->argc-2); + return; +#endif } else { addReplySubcommandSyntaxError(c); return; diff --git a/src/server.c b/src/server.c index 4b87b6ac..fa2c7b1e 100644 --- a/src/server.c +++ b/src/server.c @@ -2230,6 +2230,7 @@ void initServerConfig(void) { server.maxidletime = CONFIG_DEFAULT_CLIENT_TIMEOUT; server.tcpkeepalive = CONFIG_DEFAULT_TCP_KEEPALIVE; server.active_expire_enabled = 1; + server.jemalloc_bg_thread = 1; server.active_defrag_enabled = CONFIG_DEFAULT_ACTIVE_DEFRAG; server.active_defrag_ignore_bytes = CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES; server.active_defrag_threshold_lower = CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER; @@ -2866,6 +2867,7 @@ void initServer(void) { latencyMonitorInit(); bioInit(); initThreadedIO(); + set_jemalloc_bg_thread(server.jemalloc_bg_thread); server.initial_memory_usage = zmalloc_used_memory(); } diff --git a/src/server.h b/src/server.h index 0813f8bd..4ae079ff 100644 --- a/src/server.h +++ b/src/server.h @@ -1129,6 +1129,7 @@ struct redisServer { int tcpkeepalive; /* Set SO_KEEPALIVE if non-zero. */ int active_expire_enabled; /* Can be disabled for testing purposes. */ int active_defrag_enabled; + int jemalloc_bg_thread; /* Enable jemalloc background thread */ size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */ int active_defrag_threshold_lower; /* minimum percentage of fragmentation to start active defrag */ int active_defrag_threshold_upper; /* maximum percentage of fragmentation at which we use maximum effort */ diff --git a/src/zmalloc.c b/src/zmalloc.c index 5e601027..58896a72 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -306,6 +306,7 @@ size_t zmalloc_get_rss(void) { #endif #if defined(USE_JEMALLOC) + int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident) { @@ -327,13 +328,46 @@ int zmalloc_get_allocator_info(size_t *allocated, je_mallctl("stats.allocated", allocated, &sz, NULL, 0); return 1; } + +void set_jemalloc_bg_thread(int enable) { + /* let jemalloc do purging asynchronously, required when there's no traffic + * after flushdb */ + if (enable) { + char val = 1; + je_mallctl("background_thread", NULL, 0, &val, 1); + } +} + +int jemalloc_purge() { + /* return all unused (reserved) pages to the OS */ + char tmp[32]; + unsigned narenas = 0; + size_t sz = sizeof(unsigned); + if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) { + sprintf(tmp, "arena.%d.purge", narenas); + if (!je_mallctl(tmp, NULL, 0, NULL, 0)) + return 0; + } + return -1; +} + #else + int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident) { *allocated = *resident = *active = 0; return 1; } + +void set_jemalloc_bg_thread(int enable) { + ((void)(enable)); +} + +int jemalloc_purge() { + return 0; +} + #endif /* Get the sum of the specified field (converted form kb to bytes) in diff --git a/src/zmalloc.h b/src/zmalloc.h index 6fb19b04..b136a910 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -86,6 +86,8 @@ size_t zmalloc_used_memory(void); void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); size_t zmalloc_get_rss(void); int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident); +void set_jemalloc_bg_thread(int enable); +int jemalloc_purge(); size_t zmalloc_get_private_dirty(long pid); size_t zmalloc_get_smap_bytes_by_field(char *field, long pid); size_t zmalloc_get_memory_size(void); From 56258c6b7d26b6ee44bf31b18e9aad95d0a0142a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 17 Jul 2019 08:51:02 +0300 Subject: [PATCH 006/116] Module API for Forking * create module API for forking child processes. * refactor duplicate code around creating and tracking forks by AOF and RDB. * child processes listen to SIGUSR1 and dies exitFromChild in order to eliminate a valgrind warning of unhandled signal. * note that BGSAVE error reply has changed. valgrind error is: Process terminating with default action of signal 10 (SIGUSR1) --- runtest-moduleapi | 3 +- src/aof.c | 35 +++--------- src/childinfo.c | 2 + src/db.c | 5 +- src/defrag.c | 2 +- src/module.c | 100 ++++++++++++++++++++++++++++++++++ src/rdb.c | 49 +++-------------- src/redismodule.h | 7 +++ src/replication.c | 6 +- src/scripting.c | 3 +- src/server.c | 90 ++++++++++++++++++++++++++---- src/server.h | 9 +++ tests/modules/Makefile | 6 +- tests/modules/fork.c | 84 ++++++++++++++++++++++++++++ tests/unit/moduleapi/fork.tcl | 32 +++++++++++ 15 files changed, 342 insertions(+), 91 deletions(-) create mode 100644 tests/modules/fork.c create mode 100644 tests/unit/moduleapi/fork.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb..da61ab25 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,5 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork "${@}" + diff --git a/src/aof.c b/src/aof.c index 565ee807..fc62d86e 100644 --- a/src/aof.c +++ b/src/aof.c @@ -264,9 +264,9 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (server.rdb_child_pid != -1) { + if (hasForkChild() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; - serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); + serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not @@ -397,7 +397,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { + } else if (hasForkChild()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -493,9 +493,8 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && - (server.aof_child_pid != -1 || server.rdb_child_pid != -1)) - return; + if (server.aof_no_fsync_on_rewrite && hasForkChild()) + return; /* Perform the fsync if needed. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { @@ -1562,39 +1561,24 @@ void aofClosePipes(void) { */ int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { char tmpfile[256]; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "AOF rewrite: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_AOF); + sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); } } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); serverLog(LL_WARNING, @@ -1608,7 +1592,6 @@ int rewriteAppendOnlyFileBackground(void) { server.aof_rewrite_scheduled = 0; server.aof_rewrite_time_start = time(NULL); server.aof_child_pid = childpid; - updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences * accumulated by the parent into server.aof_rewrite_buf will start @@ -1623,7 +1606,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (server.rdb_child_pid != -1) { + } else if (hasForkChild()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { diff --git a/src/childinfo.c b/src/childinfo.c index 719025e8..fa060055 100644 --- a/src/childinfo.c +++ b/src/childinfo.c @@ -80,6 +80,8 @@ void receiveChildInfo(void) { server.stat_rdb_cow_bytes = server.child_info_data.cow_size; } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) { server.stat_aof_cow_bytes = server.child_info_data.cow_size; + } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_MODULE) { + server.stat_module_cow_bytes = server.child_info_data.cow_size; } } } diff --git a/src/db.c b/src/db.c index 51f5a12b..4a489036 100644 --- a/src/db.c +++ b/src/db.c @@ -60,10 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && - !(flags & LOOKUP_NOTOUCH)) - { + if (!hasForkChild() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { diff --git a/src/defrag.c b/src/defrag.c index ecf0255d..93c6a461 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (server.aof_child_pid!=-1 || server.rdb_child_pid!=-1) + if (hasForkChild()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/module.c b/src/module.c index f4f753c0..03cc3670 100644 --- a/src/module.c +++ b/src/module.c @@ -30,6 +30,7 @@ #include "server.h" #include "cluster.h" #include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" @@ -291,6 +292,14 @@ typedef struct RedisModuleCommandFilter { /* Registered filters */ static list *moduleCommandFilters; +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); + +static struct RedisModuleForkInfo { + RedisModuleForkDoneHandler done_handler; + void* done_handler_user_data; +} moduleForkInfo = {0}; + + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -5028,6 +5037,94 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * Module fork API + * -------------------------------------------------------------------------- */ + +/* Create a background child process with the current frozen snaphost of the + * main process where you can do some processing in the background without + * affecting / freezing the traffic and no need for threads and GIL locking. + * Note that Redis allows for only one concurrent fork. + * When the child wants to exit, it should call RedisModule_ExitFromChild. + * If the parent wants to kill the child it should call RedisModule_KillForkChild + * The done handler callback will be executed on the parent process when the + * child existed (but not when killed) + * Return: -1 on failure, on success the parent process will get a positive PID + * of the child, and the child process will get 0. + */ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) +{ + pid_t childpid; + if (hasForkChild()) { + return -1; + } + + openChildInfoPipe(); + if ((childpid = redisFork()) == 0) { + /* Child */ + redisSetProcTitle("redis-module-fork"); + } else if (childpid == -1) { + closeChildInfoPipe(); + serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno)); + } else { + /* Parent */ + server.module_child_pid = childpid; + moduleForkInfo.done_handler = cb; + moduleForkInfo.done_handler_user_data = user_data; + serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid); + } + return childpid; +} + +/* Call from the child process when you want to terminate it. + * retcode will be provided to the done handler executed on the parent process. + */ +int RM_ExitFromChild(int retcode) +{ + sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); + exitFromChild(retcode); + return REDISMODULE_OK; +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) +{ + int statloc; + /* No module child? return. */ + if (server.module_child_pid == -1) return REDISMODULE_ERR; + /* Make sure the module knows the pid it wants to kill (not trying to + * randomly kill other module's forks) */ + if (server.module_child_pid != child_pid) return REDISMODULE_ERR; + /* Kill module child, wait for child exit. */ + serverLog(LL_NOTICE,"Killing running module fork child: %ld", + (long) server.module_child_pid); + if (kill(server.module_child_pid,SIGUSR1) != -1) { + while(wait3(&statloc,0,NULL) != server.module_child_pid); + } + /* Reset the buffer accumulating changes while the child saves. */ + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; + closeChildInfoPipe(); + updateDictResizePolicy(); + return REDISMODULE_OK; +} + +void ModuleForkDoneHandler(int exitcode, int bysignal) +{ + serverLog(LL_NOTICE, + "Module fork exited pid: %d, retcode: %d, bysignal: %d", + server.module_child_pid, exitcode, bysignal); + if (moduleForkInfo.done_handler) { + moduleForkInfo.done_handler(exitcode, bysignal, + moduleForkInfo.done_handler_user_data); + } + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -5490,4 +5587,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(Fork); + REGISTER_API(ExitFromChild); + REGISTER_API(KillForkChild); } diff --git a/src/rdb.c b/src/rdb.c index c566378f..0c3a80d0 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1293,40 +1293,25 @@ werr: int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { int retval; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename,rsi); if (retval == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "RDB: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_RDB); + sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB"); } exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); server.lastbgsave_status = C_ERR; @@ -1338,7 +1323,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_DISK; - updateDictResizePolicy(); return C_OK; } return C_OK; /* unreached */ @@ -2279,10 +2263,9 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { listNode *ln; listIter li; pid_t childpid; - long long start; int pipefds[2]; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; /* Before to fork, create a pipe that will be used in order to * send back to the parent the IDs of the slaves that successfully @@ -2318,8 +2301,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { /* Create the child process. */ openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { /* Child */ int retval; rio slave_sockets; @@ -2327,7 +2309,6 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { rioInitWithFdset(&slave_sockets,fds,numfds); zfree(fds); - closeListeningSockets(0); redisSetProcTitle("redis-rdb-to-slaves"); retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi); @@ -2335,16 +2316,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { retval = C_ERR; if (retval == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "RDB: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_RDB); + sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB"); /* If we are returning OK, at least one slave was served * with the RDB file as expected, so we need to send a report @@ -2413,16 +2385,11 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { close(pipefds[1]); closeChildInfoPipe(); } else { - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); - serverLog(LL_NOTICE,"Background RDB transfer started by pid %d", childpid); server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; - updateDictResizePolicy(); } zfree(clientids); zfree(fds); @@ -2465,13 +2432,13 @@ void bgsaveCommand(client *c) { if (server.rdb_child_pid != -1) { addReplyError(c,"Background save already in progress"); - } else if (server.aof_child_pid != -1) { + } else if (hasForkChild()) { if (schedule) { server.rdb_bgsave_scheduled = 1; addReplyStatus(c,"Background saving scheduled"); } else { addReplyError(c, - "An AOF log rewriting in progress: can't BGSAVE right now. " + "Another BG operation is in progress: can't BGSAVE right now. " "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "possible."); } diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957..60681da7 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -173,6 +173,7 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value); typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); #define REDISMODULE_TYPE_METHOD_VERSION 1 typedef struct RedisModuleTypeMethods { @@ -357,6 +358,9 @@ const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(R int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg); int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg); int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos); +int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); +int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); +int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); #endif /* This is included inline inside each Redis module. */ @@ -528,6 +532,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CommandFilterArgInsert); REDISMODULE_GET_API(CommandFilterArgReplace); REDISMODULE_GET_API(CommandFilterArgDelete); + REDISMODULE_GET_API(Fork); + REDISMODULE_GET_API(ExitFromChild); + REDISMODULE_GET_API(KillForkChild); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/src/replication.c b/src/replication.c index 26e7cf8f..7adf5ba3 100644 --- a/src/replication.c +++ b/src/replication.c @@ -751,11 +751,11 @@ void syncCommand(client *c) { /* Target is disk (or the slave is not capable of supporting * diskless replication) and we don't have a BGSAVE in progress, * let's start one. */ - if (server.aof_child_pid == -1) { + if (!hasForkChild()) { startBgsaveForReplication(c->slave_capa); } else { serverLog(LL_NOTICE, - "No BGSAVE in progress, but an AOF rewrite is active. " + "No BGSAVE in progress, but another BG operation is active. " "BGSAVE for replication delayed"); } } @@ -2899,7 +2899,7 @@ void replicationCron(void) { * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other slaves * have the time to arrive before we start streaming. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + if (!hasForkChild()) { time_t idle, max_idle = 0; int slaves_waiting = 0; int mincapa = -1; diff --git a/src/scripting.c b/src/scripting.c index 032bfdf1..deb40645 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1693,7 +1693,7 @@ void ldbSendLogs(void) { int ldbStartSession(client *c) { ldb.forked = (c->flags & CLIENT_LUA_DEBUG_SYNC) == 0; if (ldb.forked) { - pid_t cp = fork(); + pid_t cp = redisFork(); if (cp == -1) { addReplyError(c,"Fork() failed: can't run EVAL in debugging mode."); return 0; @@ -1710,7 +1710,6 @@ int ldbStartSession(client *c) { * socket to make sure if the parent crashes a reset is sent * to the clients. */ serverLog(LL_WARNING,"Redis forked for debugging eval"); - closeListeningSockets(0); } else { /* Parent */ listAddNodeTail(ldb.children,(void*)(unsigned long)cp); diff --git a/src/server.c b/src/server.c index 4337b8f0..675b638d 100644 --- a/src/server.c +++ b/src/server.c @@ -1447,12 +1447,18 @@ int incrementallyRehash(int dbid) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) + if (!hasForkChild()) dictEnableResize(); else dictDisableResize(); } +int hasForkChild() { + return server.rdb_child_pid != -1 || + server.aof_child_pid != -1 || + server.module_child_pid != -1; +} + /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ @@ -1689,7 +1695,7 @@ void databasesCron(void) { /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad * as will cause a lot of copy-on-write of memory pages. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + if (!hasForkChild()) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ @@ -1886,15 +1892,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + if (!hasForkChild() && server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (server.rdb_child_pid != -1 || server.aof_child_pid != -1 || - ldbPendingChildren()) + if (hasForkChild() || ldbPendingChildren()) { int statloc; pid_t pid; @@ -1907,16 +1912,20 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (pid == -1) { serverLog(LL_WARNING,"wait3() returned an error: %s. " - "rdb_child_pid = %d, aof_child_pid = %d", + "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d", strerror(errno), (int) server.rdb_child_pid, - (int) server.aof_child_pid); + (int) server.aof_child_pid, + (int) server.module_child_pid); } else if (pid == server.rdb_child_pid) { backgroundSaveDoneHandler(exitcode,bysignal); if (!bysignal && exitcode == 0) receiveChildInfo(); } else if (pid == server.aof_child_pid) { backgroundRewriteDoneHandler(exitcode,bysignal); if (!bysignal && exitcode == 0) receiveChildInfo(); + } else if (pid == server.module_child_pid) { + ModuleForkDoneHandler(exitcode,bysignal); + if (!bysignal && exitcode == 0) receiveChildInfo(); } else { if (!ldbRemoveChild(pid)) { serverLog(LL_WARNING, @@ -1954,8 +1963,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Trigger an AOF rewrite if needed. */ if (server.aof_state == AOF_ON && - server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && + !hasForkChild() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { @@ -2013,7 +2021,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + if (!hasForkChild() && server.rdb_bgsave_scheduled && (server.unixtime-server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) @@ -2784,6 +2792,7 @@ void initServer(void) { server.cronloops = 0; server.rdb_child_pid = -1; server.aof_child_pid = -1; + server.module_child_pid = -1; server.rdb_child_type = RDB_CHILD_TYPE_NONE; server.rdb_bgsave_scheduled = 0; server.child_info_pipe[0] = -1; @@ -2802,6 +2811,7 @@ void initServer(void) { server.stat_peak_memory = 0; server.stat_rdb_cow_bytes = 0; server.stat_aof_cow_bytes = 0; + server.stat_module_cow_bytes = 0; server.cron_malloc_stats.zmalloc_used = 0; server.cron_malloc_stats.process_rss = 0; server.cron_malloc_stats.allocator_allocated = 0; @@ -4040,7 +4050,9 @@ sds genRedisInfoString(char *section) { "aof_current_rewrite_time_sec:%jd\r\n" "aof_last_bgrewrite_status:%s\r\n" "aof_last_write_status:%s\r\n" - "aof_last_cow_size:%zu\r\n", + "aof_last_cow_size:%zu\r\n" + "module_fork_in_progress:%d\r\n" + "module_fork_last_cow_size:%zu\r\n", server.loading, server.dirty, server.rdb_child_pid != -1, @@ -4058,7 +4070,9 @@ sds genRedisInfoString(char *section) { -1 : time(NULL)-server.aof_rewrite_time_start), (server.aof_lastbgrewrite_status == C_OK) ? "ok" : "err", (server.aof_last_write_status == C_OK) ? "ok" : "err", - server.stat_aof_cow_bytes); + server.stat_aof_cow_bytes, + server.module_child_pid != -1, + server.stat_module_cow_bytes); if (server.aof_enabled) { info = sdscatprintf(info, @@ -4554,6 +4568,58 @@ void setupSignalHandlers(void) { return; } +static void sigKillChildHandler(int sig) { + UNUSED(sig); + /* this handler is needed to resolve a valgrind warning */ + serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); + exitFromChild(1); +} + +void setupChildSignalHandlers(void) { + struct sigaction act; + + /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction is used. + * Otherwise, sa_handler is used. */ + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = sigKillChildHandler; + sigaction(SIGUSR1, &act, NULL); + return; +} + +int redisFork() { + int childpid; + long long start = ustime(); + if ((childpid = fork()) == 0) { + /* Child */ + closeListeningSockets(0); + setupChildSignalHandlers(); + } else { + /* Parent */ + server.stat_fork_time = ustime()-start; + server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ + latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); + if (childpid == -1) { + return -1; + } + updateDictResizePolicy(); + } + return childpid; +} + +void sendChildCOWInfo(int ptype, char *pname) { + size_t private_dirty = zmalloc_get_private_dirty(-1); + + if (private_dirty) { + serverLog(LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, private_dirty/(1024*1024)); + } + + server.child_info_data.cow_size = private_dirty; + sendChildInfo(ptype); +} + void memtest(size_t megabytes, int passes); /* Returns 1 if there is --sentinel among the arguments or if diff --git a/src/server.h b/src/server.h index f81b1010..deaaf263 100644 --- a/src/server.h +++ b/src/server.h @@ -1030,6 +1030,7 @@ struct clusterState; #define CHILD_INFO_MAGIC 0xC17DDA7A12345678LL #define CHILD_INFO_TYPE_RDB 0 #define CHILD_INFO_TYPE_AOF 1 +#define CHILD_INFO_TYPE_MODULE 3 struct redisServer { /* General */ @@ -1065,6 +1066,7 @@ struct redisServer { int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs to be processed. */ + pid_t module_child_pid; /* PID of module child */ /* Networking */ int port; /* TCP listening port */ int tcp_backlog; /* TCP listen() backlog */ @@ -1138,6 +1140,7 @@ struct redisServer { _Atomic long long stat_net_output_bytes; /* Bytes written to network. */ size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ + size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ /* The following two are used to track instantaneous metrics, like * number of operations per second, network traffic. */ struct { @@ -1528,6 +1531,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +void ModuleForkDoneHandler(int exitcode, int bysignal); /* Utils */ long long ustime(void); @@ -1786,6 +1790,11 @@ void closeChildInfoPipe(void); void sendChildInfo(int process_type); void receiveChildInfo(void); +/* Fork helpers */ +int redisFork(); +int hasForkChild(); +void sendChildCOWInfo(int ptype, char *pname); + /* acl.c -- Authentication related prototypes. */ extern rax *Users; extern user *DefaultUser; diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20af..846d4c87 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,16 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so fork.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +fork.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +fork.so: fork.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/fork.c b/tests/modules/fork.c new file mode 100644 index 00000000..0804e435 --- /dev/null +++ b/tests/modules/fork.c @@ -0,0 +1,84 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +#include +#include +#include + +#define UNUSED(V) ((void) V) + +int child_pid = -1; +int exitted_with_code = -1; + +void done_handler(int exitcode, int bysignal, void *user_data) { + child_pid = -1; + exitted_with_code = exitcode; + assert(user_data==(void*)0xdeadbeef); + UNUSED(bysignal); +} + +int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + long long code_to_exit_with; + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModule_StringToLongLong(argv[1], &code_to_exit_with); + exitted_with_code = -1; + child_pid = RedisModule_Fork(done_handler, (void*)0xdeadbeef); + if (child_pid < 0) { + RedisModule_ReplyWithError(ctx, "Fork failed"); + return REDISMODULE_OK; + } else if (child_pid > 0) { + /* parent */ + RedisModule_ReplyWithLongLong(ctx, child_pid); + return REDISMODULE_OK; + } + + /* child */ + RedisModule_Log(ctx, "notice", "fork child started"); + usleep(200000); + RedisModule_Log(ctx, "notice", "fork child exiting"); + RedisModule_ExitFromChild(code_to_exit_with); + /* unreachable */ + return 0; +} + +int fork_exitcode(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + UNUSED(argv); + UNUSED(argc); + RedisModule_ReplyWithLongLong(ctx, exitted_with_code); + return REDISMODULE_OK; +} + +int fork_kill(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + UNUSED(argv); + UNUSED(argc); + if (RedisModule_KillForkChild(child_pid) != REDISMODULE_OK) + RedisModule_ReplyWithError(ctx, "KillForkChild failed"); + else + RedisModule_ReplyWithLongLong(ctx, 1); + child_pid = -1; + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); + if (RedisModule_Init(ctx,"fork",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.create", fork_create,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.exitcode", fork_exitcode,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.kill", fork_kill,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl new file mode 100644 index 00000000..f7d7e47d --- /dev/null +++ b/tests/unit/moduleapi/fork.tcl @@ -0,0 +1,32 @@ +set testmodule [file normalize tests/modules/fork.so] + +proc count_log_message {pattern} { + set result [exec grep -c $pattern < [srv 0 stdout]] +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module fork} { + # the argument to fork.create is the exitcode on termination + r fork.create 3 + wait_for_condition 20 100 { + [r fork.exitcode] != -1 + } else { + fail "fork didn't terminate" + } + r fork.exitcode + } {3} + + test {Module fork kill} { + r fork.create 3 + after 20 + r fork.kill + after 100 + + assert {[count_log_message "fork child started"] eq "2"} + assert {[count_log_message "Received SIGUSR1 in child"] eq "1"} + assert {[count_log_message "fork child exiting"] eq "1"} + } + +} From d7d028a7a72388cf3908a5f40c8188e68a447dee Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 14 Jul 2019 17:34:13 +0300 Subject: [PATCH 007/116] Allow modules to handle RDB loading errors. This is especially needed in diskless loading, were a short read could have caused redis to exit. now the module can handle the error and return to the caller gracefully. this fixes #5326 --- src/module.c | 41 +++++++++++++++++++++++++++++++++++------ src/redismodule.h | 9 +++++++++ src/server.h | 2 ++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index f4f753c0..68952b1e 100644 --- a/src/module.c +++ b/src/module.c @@ -3139,9 +3139,14 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * RDB loading and saving functions * -------------------------------------------------------------------------- */ -/* Called when there is a load error in the context of a module. This cannot - * be recovered like for the built-in types. */ +/* Called when there is a load error in the context of a module. On some + * modules this cannot be recovered, but if the module declared capability + * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { + if (io->flags & REDISMODULE_HANDLE_IO_ERRORS) { + io->error = 1; + return; + } serverLog(LL_WARNING, "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " @@ -3152,6 +3157,23 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } +/* Set flags defining capabilities or behavior */ +void RM_SetIOFlags(RedisModuleIO *io, int flags) { + io->flags = flags; +} + +/* Get flags which were set by RedisModule_SetIOFlags */ +int RM_GetIOFlags(RedisModuleIO *io) { + return io->flags; +} + +/* Returns true if any previous IO API failed. + * for Load* APIs the REDISMODULE_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetIOFlags first. */ +int RM_IsIOError(RedisModuleIO *io) { + return io->error; +} + /* Save an unsigned 64 bit value into the RDB file. This function should only * be called in the context of the rdb_save method of modules implementing new * data types. */ @@ -3175,6 +3197,7 @@ saveerr: * be called in the context of the rdb_load method of modules implementing * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; @@ -3186,7 +3209,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */ @@ -3245,6 +3268,7 @@ saveerr: /* Implements RM_LoadString() and RM_LoadStringBuffer() */ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { + if (io->error) return NULL; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; @@ -3256,7 +3280,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { loaderr: moduleRDBLoadError(io); - return NULL; /* Never reached. */ + return NULL; } /* In the context of the rdb_load method of a module data type, loads a string @@ -3305,6 +3329,7 @@ saveerr: /* In the context of the rdb_save method of a module data type, loads back the * double value saved by RedisModule_SaveDouble(). */ double RM_LoadDouble(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; @@ -3316,7 +3341,7 @@ double RM_LoadDouble(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* In the context of the rdb_save method of a module data type, saves a float @@ -3341,6 +3366,7 @@ saveerr: /* In the context of the rdb_save method of a module data type, loads back the * float value saved by RedisModule_SaveFloat(). */ float RM_LoadFloat(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; @@ -3352,7 +3378,7 @@ float RM_LoadFloat(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* -------------------------------------------------------------------------- @@ -5408,6 +5434,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); + REGISTER_API(GetIOFlags); + REGISTER_API(SetIOFlags); + REGISTER_API(IsIOError); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957..154b5b40 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -140,6 +140,9 @@ typedef uint64_t RedisModuleTimerID; /* Do filter RedisModule_Call() commands initiated by module itself. */ #define REDISMODULE_CMDFILTER_NOSELF (1<<0) +/* Declare that the module can handle errors with RedisModule_SetIOFlags */ +#define REDISMODULE_HANDLE_IO_ERRORS (1<<0) + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE @@ -271,6 +274,9 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); +void REDISMODULE_API_FUNC(RedisModule_SetIOFlags)(RedisModuleIO *io, int flags); +int REDISMODULE_API_FUNC(RedisModule_GetIOFlags)(RedisModuleIO *io); +int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -444,6 +450,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeSetValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); + REDISMODULE_GET_API(SetIOFlags); + REDISMODULE_GET_API(GetIOFlags); + REDISMODULE_GET_API(IsIOError); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); diff --git a/src/server.h b/src/server.h index f81b1010..9957c3b5 100644 --- a/src/server.h +++ b/src/server.h @@ -599,6 +599,7 @@ typedef struct RedisModuleIO { * 2 (current version with opcodes annotation). */ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct redisObject *key; /* Optional name of key processed */ + int flags; /* flags declaring capabilities or behavior */ } RedisModuleIO; /* Macro to initialize an IO context. Note that the 'ver' field is populated @@ -611,6 +612,7 @@ typedef struct RedisModuleIO { iovar.ver = 0; \ iovar.key = keyptr; \ iovar.ctx = NULL; \ + iovar.flags = 0; \ } while(0); /* This is a structure used to export DEBUG DIGEST capabilities to Redis From e91d9a6fffcac20adfb4fdf2d8fb365ec0816261 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 24 Jul 2019 12:58:15 +0300 Subject: [PATCH 008/116] Extend modules API to allow modules report to redis INFO this implements #6012 --- runtest-moduleapi | 2 +- src/debug.c | 6 ++ src/module.c | 130 ++++++++++++++++++++++++++++++ src/redismodule.h | 16 ++++ src/server.c | 16 +++- src/server.h | 1 + tests/modules/Makefile | 7 +- tests/modules/infotest.c | 32 ++++++++ tests/unit/moduleapi/infotest.tcl | 53 ++++++++++++ 9 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 tests/modules/infotest.c create mode 100644 tests/unit/moduleapi/infotest.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb..9bf12677 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/infotest "${@}" diff --git a/src/debug.c b/src/debug.c index 1f1157d4..daf00b14 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1337,6 +1337,12 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { /* Log dump of processor registers */ logRegisters(uc); + /* Log Modules INFO */ + serverLogRaw(LL_WARNING|LL_RAW, "\n------ MODULES INFO OUTPUT ------\n"); + infostring = modulesCollectInfo(sdsempty(), NULL, 1, 0); + serverLogRaw(LL_WARNING|LL_RAW, infostring); + sdsfree(infostring); + #if defined(HAVE_PROC_MAPS) /* Test memory */ serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n"); diff --git a/src/module.c b/src/module.c index f4f753c0..0e19c11e 100644 --- a/src/module.c +++ b/src/module.c @@ -40,6 +40,16 @@ * pointers that have an API the module can call with them) * -------------------------------------------------------------------------- */ +typedef struct RedisModuleInfoCtx { + struct RedisModule *module; + sds requested_section; + sds info; /* info string we collected so far */ + int sections; /* number of sections we collected so far */ + int in_section; /* indication if we're in an active section or not */ +} RedisModuleInfoCtx; + +typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); + /* This structure represents a module inside the system. */ struct RedisModule { void *handle; /* Module dlopen() handle. */ @@ -51,6 +61,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + RedisModuleInfoFunc info_cb; /* callback for module to add INFO fields. */ }; typedef struct RedisModule RedisModule; @@ -4686,6 +4697,118 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k return res ? REDISMODULE_OK : REDISMODULE_ERR; } + + + +/* -------------------------------------------------------------------------- + * Modules Info fields + * -------------------------------------------------------------------------- */ + +/* Used to start a new section, before adding any fields. the section name will + * be prefixed by "_" and must only include A-Z,a-z,0-9. + * When return value is REDISMODULE_ERR, the section should and will be skipped. */ +int RM_AddInfoSection(RedisModuleInfoCtx *ctx, char *name) { + sds full_name = sdscatprintf(sdsdup(ctx->module->name), "_%s", name); + + /* proceed only if: + * 1) no section was requested (emit all) + * 2) the module name was requested (emit all) + * 3) this specific section was requested. */ + if (ctx->requested_section) { + if (strcasecmp(ctx->requested_section, full_name) && + strcasecmp(ctx->requested_section, ctx->module->name)) { + sdsfree(full_name); + ctx->in_section = 0; + return REDISMODULE_ERR; + } + } + if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n"); + ctx->info = sdscatprintf(ctx->info, "# %s\r\n", full_name); + ctx->in_section = 1; + sdsfree(full_name); + return REDISMODULE_OK; +} + +/* Used by RedisModuleInfoFunc to add info fields. + * Each field will be automatically prefixed by "_". + * Field names or values must not include \r\n of ":" */ +int RM_AddInfoFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%s\r\n", + ctx->module->name, + field, + (sds)value->ptr); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%s\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%.17g\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%lld\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%llu\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { + ctx->module->info_cb = cb; + return REDISMODULE_OK; +} + +sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (!module->info_cb) + continue; + RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0}; + module->info_cb(&info_ctx, for_crash_report); + info = info_ctx.info; + sections = info_ctx.sections; + } + dictReleaseIterator(di); + return info; +} + /* -------------------------------------------------------------------------- * Modules utility APIs * -------------------------------------------------------------------------- */ @@ -5490,4 +5613,11 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(RegisterInfoFunc); + REGISTER_API(AddInfoSection); + REGISTER_API(AddInfoFieldString); + REGISTER_API(AddInfoFieldCString); + REGISTER_API(AddInfoFieldDouble); + REGISTER_API(AddInfoFieldLongLong); + REGISTER_API(AddInfoFieldULongLong); } diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957..1feb14a6 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -160,6 +160,7 @@ typedef struct RedisModuleDict RedisModuleDict; typedef struct RedisModuleDictIter RedisModuleDictIter; typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx; typedef struct RedisModuleCommandFilter RedisModuleCommandFilter; +typedef struct RedisModuleInfoCtx RedisModuleInfoCtx; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -173,6 +174,7 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value); typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); +typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); #define REDISMODULE_TYPE_METHOD_VERSION 1 typedef struct RedisModuleTypeMethods { @@ -317,6 +319,13 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictNext)(RedisModuleCtx *ct RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictPrev)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr); int REDISMODULE_API_FUNC(RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen); int REDISMODULE_API_FUNC(RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key); +int REDISMODULE_API_FUNC(RedisModule_RegisterInfoFunc)(RedisModuleCtx *ctx, RedisModuleInfoFunc cb); +int REDISMODULE_API_FUNC(RedisModule_AddInfoSection)(RedisModuleInfoCtx *ctx, char *name); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -490,6 +499,13 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(DictPrev); REDISMODULE_GET_API(DictCompare); REDISMODULE_GET_API(DictCompareC); + REDISMODULE_GET_API(RegisterInfoFunc); + REDISMODULE_GET_API(AddInfoSection); + REDISMODULE_GET_API(AddInfoFieldString); + REDISMODULE_GET_API(AddInfoFieldCString); + REDISMODULE_GET_API(AddInfoFieldDouble); + REDISMODULE_GET_API(AddInfoFieldLongLong); + REDISMODULE_GET_API(AddInfoFieldULongLong); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/src/server.c b/src/server.c index 4337b8f0..c671e375 100644 --- a/src/server.c +++ b/src/server.c @@ -3809,12 +3809,15 @@ sds genRedisInfoString(char *section) { time_t uptime = server.unixtime-server.stat_starttime; int j; struct rusage self_ru, c_ru; - int allsections = 0, defsections = 0; + int allsections = 0, defsections = 0, everything = 0, modules = 0; int sections = 0; if (section == NULL) section = "default"; allsections = strcasecmp(section,"all") == 0; defsections = strcasecmp(section,"default") == 0; + everything = strcasecmp(section,"everything") == 0; + modules = strcasecmp(section,"modules") == 0; + if (everything) allsections = 1; getrusage(RUSAGE_SELF, &self_ru); getrusage(RUSAGE_CHILDREN, &c_ru); @@ -4357,6 +4360,17 @@ sds genRedisInfoString(char *section) { } } } + + /* Get info from modules. + * if user asked for "everything" or "modules", or a specific section + * that's not found yet. */ + if (everything || modules || + (!allsections && !defsections && sections==0)) { + info = modulesCollectInfo(info, + everything || modules ? NULL: section, + 0, /* not a crash report */ + sections); + } return info; } diff --git a/src/server.h b/src/server.h index b200a669..ddddd84f 100644 --- a/src/server.h +++ b/src/server.h @@ -1528,6 +1528,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); /* Utils */ long long ustime(void); diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20af..66bf6de3 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,17 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so infotest.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +infotest.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +infotest.so: infotest.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c new file mode 100644 index 00000000..d53ea212 --- /dev/null +++ b/tests/modules/infotest.c @@ -0,0 +1,32 @@ +#include "redismodule.h" + +#include + +void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { + RedisModule_AddInfoSection(ctx, "Spanish"); + RedisModule_AddInfoFieldCString(ctx, "uno", "one"); + RedisModule_AddInfoFieldLongLong(ctx, "dos", 2); + + RedisModule_AddInfoSection(ctx, "Italian"); + RedisModule_AddInfoFieldLongLong(ctx, "due", 2); + RedisModule_AddInfoFieldDouble(ctx, "tre", 3.3); + + if (for_crash_report) { + RedisModule_AddInfoSection(ctx, "Klingon"); + RedisModule_AddInfoFieldCString(ctx, "one", "wa’"); + RedisModule_AddInfoFieldCString(ctx, "two", "cha’"); + RedisModule_AddInfoFieldCString(ctx, "three", "wej"); + } + +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + if (RedisModule_Init(ctx,"infotest",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_RegisterInfoFunc(ctx, InfoFunc) == REDISMODULE_ERR) return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl new file mode 100644 index 00000000..143f9050 --- /dev/null +++ b/tests/unit/moduleapi/infotest.tcl @@ -0,0 +1,53 @@ +set testmodule [file normalize tests/modules/infotest.so] + +# Return value for INFO property +proc field {info property} { + if {[regexp "\r\n$property:(.*?)\r\n" $info _ value]} { + set _ $value + } +} + +start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + + test {module info all} { + set info [r info all] + # info all does not contain modules + assert { ![string match "*Spanish*" $info] } + assert { [string match "*used_memory*" $info] } + } + + test {module info everything} { + set info [r info everything] + # info everything contains all default sections, but not ones for crash report + assert { [string match "*Spanish*" $info] } + assert { [string match "*Italian*" $info] } + assert { [string match "*used_memory*" $info] } + assert { ![string match "*Klingon*" $info] } + field $info infotest_dos + } {2} + + test {module info modules} { + set info [r info modules] + # info all does not contain modules + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + } + + test {module info one module} { + set info [r info INFOTEST] + # info all does not contain modules + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + } + + test {module info one section} { + set info [r info INFOTEST_SPANISH] + assert { ![string match "*used_memory*" $info] } + assert { ![string match "*Italian*" $info] } + field $info infotest_uno + } {one} + + # TODO: test crash report. + +} From 4339706e07e15fe5a228d175756c4e4be83e2867 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 30 Jul 2019 15:11:57 +0300 Subject: [PATCH 009/116] Avoid diskelss-load if modules did not declare they handle read errors --- src/module.c | 47 +++++++++++++++++++++++++++++++++++------------ src/redismodule.h | 10 ++++------ src/replication.c | 6 +++++- src/server.h | 3 +-- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/module.c b/src/module.c index 68952b1e..36384c01 100644 --- a/src/module.c +++ b/src/module.c @@ -51,6 +51,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + int options; /* Moduile options and capabilities. */ }; typedef struct RedisModule RedisModule; @@ -771,6 +772,19 @@ long long RM_Milliseconds(void) { return mstime(); } +/* Set flags defining capabilities or behavior bit flags. + * + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS: + * Generally, modules don't need to bother with this, as the process will just + * terminate if a read error happens, however, setting this flag would allow + * repl-diskless-load to work if enabled. + * The module should use RedisModule_IsIOError after reads, before using the + * data that was read, and in case of error, propagate it upwards, and also be + * able to release the partially populated value and all it's allocations. */ +void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { + ctx->module->options = options; +} + /* -------------------------------------------------------------------------- * Automatic memory management for modules * -------------------------------------------------------------------------- */ @@ -3143,7 +3157,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - if (io->flags & REDISMODULE_HANDLE_IO_ERRORS) { + if (io->ctx->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } @@ -3157,19 +3171,29 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } -/* Set flags defining capabilities or behavior */ -void RM_SetIOFlags(RedisModuleIO *io, int flags) { - io->flags = flags; -} +/* Returns 0 if there's at least one registered data type that did not declare + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should + * be avoided since it could cause data loss. */ +int moduleAllDatatypesHandleErrors() { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; -/* Get flags which were set by RedisModule_SetIOFlags */ -int RM_GetIOFlags(RedisModuleIO *io) { - return io->flags; + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (listLength(module->types) && + !(module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS)) + { + dictReleaseIterator(di); + return 0; + } + } + dictReleaseIterator(di); + return 1; } /* Returns true if any previous IO API failed. - * for Load* APIs the REDISMODULE_HANDLE_IO_ERRORS flag must be set with - * RediModule_SetIOFlags first. */ + * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetModuleOptions first. */ int RM_IsIOError(RedisModuleIO *io) { return io->error; } @@ -5434,9 +5458,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); - REGISTER_API(GetIOFlags); - REGISTER_API(SetIOFlags); REGISTER_API(IsIOError); + REGISTER_API(SetModuleOptions); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index 154b5b40..f0f27c06 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -140,8 +140,8 @@ typedef uint64_t RedisModuleTimerID; /* Do filter RedisModule_Call() commands initiated by module itself. */ #define REDISMODULE_CMDFILTER_NOSELF (1<<0) -/* Declare that the module can handle errors with RedisModule_SetIOFlags */ -#define REDISMODULE_HANDLE_IO_ERRORS (1<<0) +/* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ +#define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) /* ------------------------- End of common defines ------------------------ */ @@ -274,9 +274,8 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); -void REDISMODULE_API_FUNC(RedisModule_SetIOFlags)(RedisModuleIO *io, int flags); -int REDISMODULE_API_FUNC(RedisModule_GetIOFlags)(RedisModuleIO *io); int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_SetModuleOptions)(RedisModuleCtx *ctx, int options); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -450,9 +449,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeSetValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); - REDISMODULE_GET_API(SetIOFlags); - REDISMODULE_GET_API(GetIOFlags); REDISMODULE_GET_API(IsIOError); + REDISMODULE_GET_API(SetModuleOptions); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); diff --git a/src/replication.c b/src/replication.c index 26e7cf8f..14ae2f96 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1115,8 +1115,12 @@ void restartAOFAfterSYNC() { static int useDisklessLoad() { /* compute boolean decision to use diskless load */ - return server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || + int enabled = server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || (server.repl_diskless_load == REPL_DISKLESS_LOAD_WHEN_DB_EMPTY && dbTotalServerKeyCount()==0); + /* Check all modules handle read errors, otherwise it's not safe to use diskless load. */ + if (enabled && !moduleAllDatatypesHandleErrors()) + enabled = 0; + return enabled; } /* Helper function for readSyncBulkPayload() to make backups of the current diff --git a/src/server.h b/src/server.h index 9957c3b5..5991cfa6 100644 --- a/src/server.h +++ b/src/server.h @@ -599,7 +599,6 @@ typedef struct RedisModuleIO { * 2 (current version with opcodes annotation). */ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct redisObject *key; /* Optional name of key processed */ - int flags; /* flags declaring capabilities or behavior */ } RedisModuleIO; /* Macro to initialize an IO context. Note that the 'ver' field is populated @@ -612,7 +611,6 @@ typedef struct RedisModuleIO { iovar.ver = 0; \ iovar.key = keyptr; \ iovar.ctx = NULL; \ - iovar.flags = 0; \ } while(0); /* This is a structure used to export DEBUG DIGEST capabilities to Redis @@ -1530,6 +1528,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +int moduleAllDatatypesHandleErrors(); /* Utils */ long long ustime(void); From 07b1abab9543205de4ed2db7bbf28d094a08960b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 21 Jul 2019 18:18:11 +0300 Subject: [PATCH 010/116] Add test for module diskless short reads --- tests/modules/testrdb.c | 13 ++++++- tests/unit/moduleapi/testrdb.tcl | 62 +++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index 415497a2..d73c8bfd 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -15,6 +15,8 @@ RedisModuleString *after_str = NULL; void *testrdb_type_load(RedisModuleIO *rdb, int encver) { int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return NULL; assert(count==1); assert(encver==1); RedisModuleString *str = RedisModule_LoadString(rdb); @@ -57,6 +59,8 @@ int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { RedisModule_FreeString(ctx, before_str); before_str = NULL; int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; if (count) before_str = RedisModule_LoadString(rdb); } else { @@ -64,14 +68,19 @@ int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { RedisModule_FreeString(ctx, after_str); after_str = NULL; int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; if (count) after_str = RedisModule_LoadString(rdb); } + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; return REDISMODULE_OK; } void testrdb_type_free(void *value) { - RedisModule_FreeString(NULL, (RedisModuleString*)value); + if (value) + RedisModule_FreeString(NULL, (RedisModuleString*)value); } int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) @@ -171,6 +180,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) return REDISMODULE_ERR; + RedisModule_SetModuleOptions(ctx, REDISMODULE_OPTIONS_HANDLE_IO_ERRORS); + if (argc > 0) RedisModule_StringToLongLong(argv[0], &conf_aux_count); diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index 22201a08..c7257000 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -56,7 +56,67 @@ tags "modules" { } } + tags {repl} { + test {diskless loading short read with module} { + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] - # TODO: test short read handling + # Set master and replica to use diskless replication + $master config set repl-diskless-sync yes + $master config set rdbcompression no + $replica config set repl-diskless-load swapdb + for {set k 0} {$k < 30} {incr k} { + r testrdb.set.key key$k [string repeat A [expr {int(rand()*1000000)}]] + } + # Start the replication process... + $master config set repl-diskless-sync-delay 0 + $replica replicaof $master_host $master_port + + # kill the replication at various points + set attempts 3 + if {$::accurate} { set attempts 10 } + for {set i 0} {$i < $attempts} {incr i} { + # wait for the replica to start reading the rdb + # using the log file since the replica only responds to INFO once in 2mb + wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1 + + # add some additional random sleep so that we kill the master on a different place each time + after [expr {int(rand()*100)}] + + # kill the replica connection on the master + set killed [$master client kill type replica] + + if {[catch { + set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10] + if {$::verbose} { + puts $res + } + }]} { + puts "failed triggering short read" + # force the replica to try another full sync + $master client kill type replica + $master set asdf asdf + # the side effect of resizing the backlog is that it is flushed (16k is the min size) + $master config set repl-backlog-size [expr {16384 + $i}] + } + # wait for loading to stop (fail) + wait_for_condition 100 10 { + [s -1 loading] eq 0 + } else { + fail "Replica didn't disconnect" + } + } + # enable fast shutdown + $master config set rdb-key-save-delay 0 + } + } + } + } } From 40c4183196b8c6e1c44c27400965786b9f010c74 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 30 Jul 2019 16:32:58 +0300 Subject: [PATCH 011/116] Log message when modules prevent diskless-load --- src/replication.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 14ae2f96..614aaec8 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1118,8 +1118,11 @@ static int useDisklessLoad() { int enabled = server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || (server.repl_diskless_load == REPL_DISKLESS_LOAD_WHEN_DB_EMPTY && dbTotalServerKeyCount()==0); /* Check all modules handle read errors, otherwise it's not safe to use diskless load. */ - if (enabled && !moduleAllDatatypesHandleErrors()) + if (enabled && !moduleAllDatatypesHandleErrors()) { + serverLog(LL_WARNING, + "Skipping diskless-load because there are modules that don't handle read errors."); enabled = 0; + } return enabled; } From b27f388344860394d02e988aa2a6c79b15a4a85a Mon Sep 17 00:00:00 2001 From: Diego Bendersky Date: Mon, 5 Aug 2019 17:35:50 -0300 Subject: [PATCH 012/116] pass len to alloc in clusterManagerMigrateKeysInReply --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index e363a279..90433985 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -3222,7 +3222,7 @@ static redisReply *clusterManagerMigrateKeysInReply(clusterManagerNode *source, redisReply *entry = reply->element[i]; size_t idx = i + offset; assert(entry->type == REDIS_REPLY_STRING); - argv[idx] = (char *) sdsnew(entry->str); + argv[idx] = (char *) sdsnewlen(entry->str, entry->len); argv_len[idx] = entry->len; if (dots) dots[i] = '.'; } From 1026d2caf89eecedfec099031870b1535185d491 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 8 Aug 2019 14:53:36 +0300 Subject: [PATCH 013/116] fix error handling on config parsing of repl-diskless-load --- src/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.c b/src/config.c index a72df2e7..090fc818 100644 --- a/src/config.c +++ b/src/config.c @@ -438,6 +438,7 @@ void loadServerConfigFromString(char *config) { server.repl_diskless_load = configEnumGetValue(repl_diskless_load_enum,argv[1]); if (server.repl_diskless_load == INT_MIN) { err = "argument must be 'disabled', 'on-empty-db', 'swapdb' or 'flushdb'"; + goto loaderr; } } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { server.repl_diskless_sync_delay = atoi(argv[1]); From 1d6e5dc4dcacb1af4698d16e695494f62129071e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 18 Aug 2019 09:41:45 +0300 Subject: [PATCH 014/116] Module INFO, add support for dict fields, rename API to have common prefix --- src/module.c | 106 +++++++++++++++++++++++++----- src/redismodule.h | 28 ++++---- tests/modules/infotest.c | 26 +++++--- tests/unit/moduleapi/infotest.tcl | 7 +- 4 files changed, 129 insertions(+), 38 deletions(-) diff --git a/src/module.c b/src/module.c index 0e19c11e..c48b2f04 100644 --- a/src/module.c +++ b/src/module.c @@ -43,9 +43,10 @@ typedef struct RedisModuleInfoCtx { struct RedisModule *module; sds requested_section; - sds info; /* info string we collected so far */ - int sections; /* number of sections we collected so far */ - int in_section; /* indication if we're in an active section or not */ + sds info; /* info string we collected so far */ + int sections; /* number of sections we collected so far */ + int in_section; /* indication if we're in an active section or not */ + int in_dict_field; /* indication that we're curreintly appending to a dict */ } RedisModuleInfoCtx; typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); @@ -4704,12 +4705,18 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k * Modules Info fields * -------------------------------------------------------------------------- */ +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); + /* Used to start a new section, before adding any fields. the section name will * be prefixed by "_" and must only include A-Z,a-z,0-9. * When return value is REDISMODULE_ERR, the section should and will be skipped. */ -int RM_AddInfoSection(RedisModuleInfoCtx *ctx, char *name) { +int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { sds full_name = sdscatprintf(sdsdup(ctx->module->name), "_%s", name); + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + /* proceed only if: * 1) no section was requested (emit all) * 2) the module name was requested (emit all) @@ -4729,12 +4736,48 @@ int RM_AddInfoSection(RedisModuleInfoCtx *ctx, char *name) { return REDISMODULE_OK; } +/* Starts a dict field, similar to the ones in INFO KEYSPACE. Use normal + * RedisModule_InfoAddField* functions to add the items to this field, and + * terminate with RedisModule_InfoEndDictField. */ +int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) { + if (!ctx->in_section) + return REDISMODULE_ERR; + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + ctx->info = sdscatprintf(ctx->info, + "%s_%s:", + ctx->module->name, + name); + ctx->in_dict_field = 1; + return REDISMODULE_OK; +} + +/* Ends a dict field, see RedisModule_InfoBeginDictField */ +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { + if (!ctx->in_dict_field) + return REDISMODULE_ERR; + /* trim the last ',' if found. */ + if (ctx->info[sdslen(ctx->info)-1]==',') + sdsIncrLen(ctx->info, -1); + ctx->info = sdscatprintf(ctx->info, "\r\n"); + ctx->in_dict_field = 0; + return REDISMODULE_OK; +} + /* Used by RedisModuleInfoFunc to add info fields. * Each field will be automatically prefixed by "_". * Field names or values must not include \r\n of ":" */ -int RM_AddInfoFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { +int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%s,", + field, + (sds)value->ptr); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%s\r\n", ctx->module->name, @@ -4743,9 +4786,16 @@ int RM_AddInfoFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleStrin return REDISMODULE_OK; } -int RM_AddInfoFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { +int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%s,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%s\r\n", ctx->module->name, @@ -4754,9 +4804,16 @@ int RM_AddInfoFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { return REDISMODULE_OK; } -int RM_AddInfoFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { +int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%.17g,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%.17g\r\n", ctx->module->name, @@ -4765,9 +4822,16 @@ int RM_AddInfoFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { return REDISMODULE_OK; } -int RM_AddInfoFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { +int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%lld,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%lld\r\n", ctx->module->name, @@ -4776,9 +4840,16 @@ int RM_AddInfoFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long valu return REDISMODULE_OK; } -int RM_AddInfoFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { +int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%llu,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%llu\r\n", ctx->module->name, @@ -4802,6 +4873,9 @@ sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections continue; RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0}; module->info_cb(&info_ctx, for_crash_report); + /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ + if (info_ctx.in_dict_field) + RM_InfoEndDictField(&info_ctx); info = info_ctx.info; sections = info_ctx.sections; } @@ -5614,10 +5688,12 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); REGISTER_API(RegisterInfoFunc); - REGISTER_API(AddInfoSection); - REGISTER_API(AddInfoFieldString); - REGISTER_API(AddInfoFieldCString); - REGISTER_API(AddInfoFieldDouble); - REGISTER_API(AddInfoFieldLongLong); - REGISTER_API(AddInfoFieldULongLong); + REGISTER_API(InfoAddSection); + REGISTER_API(InfoBeginDictField); + REGISTER_API(InfoEndDictField); + REGISTER_API(InfoAddFieldString); + REGISTER_API(InfoAddFieldCString); + REGISTER_API(InfoAddFieldDouble); + REGISTER_API(InfoAddFieldLongLong); + REGISTER_API(InfoAddFieldULongLong); } diff --git a/src/redismodule.h b/src/redismodule.h index 1feb14a6..26690c17 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -320,12 +320,14 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictPrev)(RedisModuleCtx *ct int REDISMODULE_API_FUNC(RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen); int REDISMODULE_API_FUNC(RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key); int REDISMODULE_API_FUNC(RedisModule_RegisterInfoFunc)(RedisModuleCtx *ctx, RedisModuleInfoFunc cb); -int REDISMODULE_API_FUNC(RedisModule_AddInfoSection)(RedisModuleInfoCtx *ctx, char *name); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddSection)(RedisModuleInfoCtx *ctx, char *name); +int REDISMODULE_API_FUNC(RedisModule_InfoBeginDictField)(RedisModuleInfoCtx *ctx, char *name); +int REDISMODULE_API_FUNC(RedisModule_InfoEndDictField)(RedisModuleInfoCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -500,12 +502,14 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(DictCompare); REDISMODULE_GET_API(DictCompareC); REDISMODULE_GET_API(RegisterInfoFunc); - REDISMODULE_GET_API(AddInfoSection); - REDISMODULE_GET_API(AddInfoFieldString); - REDISMODULE_GET_API(AddInfoFieldCString); - REDISMODULE_GET_API(AddInfoFieldDouble); - REDISMODULE_GET_API(AddInfoFieldLongLong); - REDISMODULE_GET_API(AddInfoFieldULongLong); + REDISMODULE_GET_API(InfoAddSection); + REDISMODULE_GET_API(InfoBeginDictField); + REDISMODULE_GET_API(InfoEndDictField); + REDISMODULE_GET_API(InfoAddFieldString); + REDISMODULE_GET_API(InfoAddFieldCString); + REDISMODULE_GET_API(InfoAddFieldDouble); + REDISMODULE_GET_API(InfoAddFieldLongLong); + REDISMODULE_GET_API(InfoAddFieldULongLong); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index d53ea212..1a75d160 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -3,19 +3,25 @@ #include void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { - RedisModule_AddInfoSection(ctx, "Spanish"); - RedisModule_AddInfoFieldCString(ctx, "uno", "one"); - RedisModule_AddInfoFieldLongLong(ctx, "dos", 2); + RedisModule_InfoAddSection(ctx, "Spanish"); + RedisModule_InfoAddFieldCString(ctx, "uno", "one"); + RedisModule_InfoAddFieldLongLong(ctx, "dos", 2); - RedisModule_AddInfoSection(ctx, "Italian"); - RedisModule_AddInfoFieldLongLong(ctx, "due", 2); - RedisModule_AddInfoFieldDouble(ctx, "tre", 3.3); + RedisModule_InfoAddSection(ctx, "Italian"); + RedisModule_InfoAddFieldLongLong(ctx, "due", 2); + RedisModule_InfoAddFieldDouble(ctx, "tre", 3.3); + + RedisModule_InfoAddSection(ctx, "keyspace"); + RedisModule_InfoBeginDictField(ctx, "db0"); + RedisModule_InfoAddFieldLongLong(ctx, "keys", 3); + RedisModule_InfoAddFieldLongLong(ctx, "expires", 1); + RedisModule_InfoEndDictField(ctx); if (for_crash_report) { - RedisModule_AddInfoSection(ctx, "Klingon"); - RedisModule_AddInfoFieldCString(ctx, "one", "wa’"); - RedisModule_AddInfoFieldCString(ctx, "two", "cha’"); - RedisModule_AddInfoFieldCString(ctx, "three", "wej"); + RedisModule_InfoAddSection(ctx, "Klingon"); + RedisModule_InfoAddFieldCString(ctx, "one", "wa’"); + RedisModule_InfoAddFieldCString(ctx, "two", "cha’"); + RedisModule_InfoAddFieldCString(ctx, "three", "wej"); } } diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 143f9050..7f756f0e 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -48,6 +48,11 @@ start_server {tags {"modules"}} { field $info infotest_uno } {one} - # TODO: test crash report. + test {module info dict} { + set info [r info infotest_keyspace] + set keyspace [field $info infotest_db0] + set keys [scan [regexp -inline {keys\=([\d]*)} $keyspace] keys=%d] + } {3} + # TODO: test crash report. } From 61853ad8dea0a3ddd8da77a257405452a114bd65 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 18 Aug 2019 10:01:57 +0300 Subject: [PATCH 015/116] Module INFO, support default section for simple modules --- src/module.c | 5 ++++- tests/modules/infotest.c | 3 +++ tests/unit/moduleapi/infotest.tcl | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index c48b2f04..046b9bc8 100644 --- a/src/module.c +++ b/src/module.c @@ -4709,9 +4709,12 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); /* Used to start a new section, before adding any fields. the section name will * be prefixed by "_" and must only include A-Z,a-z,0-9. + * NULL or empty string indicates the default section (only ) is used. * When return value is REDISMODULE_ERR, the section should and will be skipped. */ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { - sds full_name = sdscatprintf(sdsdup(ctx->module->name), "_%s", name); + sds full_name = sdsdup(ctx->module->name); + if (name != NULL && strlen(name) > 0) + full_name = sdscatprintf(full_name, "_%s", name); /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index 1a75d160..d2841093 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -3,6 +3,9 @@ #include void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { + RedisModule_InfoAddSection(ctx, ""); + RedisModule_InfoAddFieldLongLong(ctx, "global", -2); + RedisModule_InfoAddSection(ctx, "Spanish"); RedisModule_InfoAddFieldCString(ctx, "uno", "one"); RedisModule_InfoAddFieldLongLong(ctx, "dos", 2); diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 7f756f0e..a64d729f 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -14,12 +14,14 @@ start_server {tags {"modules"}} { set info [r info all] # info all does not contain modules assert { ![string match "*Spanish*" $info] } + assert { ![string match "*infotest*" $info] } assert { [string match "*used_memory*" $info] } } test {module info everything} { set info [r info everything] # info everything contains all default sections, but not ones for crash report + assert { [string match "*infotest_global*" $info] } assert { [string match "*Spanish*" $info] } assert { [string match "*Italian*" $info] } assert { [string match "*used_memory*" $info] } @@ -31,6 +33,7 @@ start_server {tags {"modules"}} { set info [r info modules] # info all does not contain modules assert { [string match "*Spanish*" $info] } + assert { [string match "*infotest_global*" $info] } assert { ![string match "*used_memory*" $info] } } @@ -39,12 +42,14 @@ start_server {tags {"modules"}} { # info all does not contain modules assert { [string match "*Spanish*" $info] } assert { ![string match "*used_memory*" $info] } - } + field $info infotest_global + } {-2} test {module info one section} { set info [r info INFOTEST_SPANISH] assert { ![string match "*used_memory*" $info] } assert { ![string match "*Italian*" $info] } + assert { ![string match "*infotest_global*" $info] } field $info infotest_uno } {one} From 78bbb9b58d7e7b0816f11fe1d4c97ad5a37df01e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 25 Aug 2019 10:11:48 +0300 Subject: [PATCH 016/116] Modlue fork is killed when the parent exists --- src/module.c | 26 +++++++++++++++----------- src/server.c | 6 ++++++ src/server.h | 1 + 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/module.c b/src/module.c index 03cc3670..6f3be61a 100644 --- a/src/module.c +++ b/src/module.c @@ -5086,20 +5086,11 @@ int RM_ExitFromChild(int retcode) return REDISMODULE_OK; } -/* Can be used to kill the forked child process from the parent process. - * child_pid whould be the return value of RedisModule_Fork. */ -int RM_KillForkChild(int child_pid) -{ +void TerminateModuleForkChild(int wait) { int statloc; - /* No module child? return. */ - if (server.module_child_pid == -1) return REDISMODULE_ERR; - /* Make sure the module knows the pid it wants to kill (not trying to - * randomly kill other module's forks) */ - if (server.module_child_pid != child_pid) return REDISMODULE_ERR; - /* Kill module child, wait for child exit. */ serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); - if (kill(server.module_child_pid,SIGUSR1) != -1) { + if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { while(wait3(&statloc,0,NULL) != server.module_child_pid); } /* Reset the buffer accumulating changes while the child saves. */ @@ -5108,6 +5099,19 @@ int RM_KillForkChild(int child_pid) moduleForkInfo.done_handler_user_data = NULL; closeChildInfoPipe(); updateDictResizePolicy(); +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) +{ + /* No module child? return. */ + if (server.module_child_pid == -1) return REDISMODULE_ERR; + /* Make sure the module knows the pid it wants to kill (not trying to + * randomly kill other module's forks) */ + if (server.module_child_pid != child_pid) return REDISMODULE_ERR; + /* Kill module child, wait for child exit. */ + TerminateModuleForkChild(1); return REDISMODULE_OK; } diff --git a/src/server.c b/src/server.c index 675b638d..31418e01 100644 --- a/src/server.c +++ b/src/server.c @@ -3551,6 +3551,12 @@ int prepareForShutdown(int flags) { killRDBChild(); } + /* Kill module child if there is one. */ + if (server.module_child_pid != -1) { + serverLog(LL_WARNING,"There is a module fork child. Killing it!"); + TerminateModuleForkChild(0); + } + if (server.aof_state != AOF_OFF) { /* Kill the AOF saving child as the AOF we already have may be longer * but contains the full dataset anyway. */ diff --git a/src/server.h b/src/server.h index deaaf263..7aa4bc2b 100644 --- a/src/server.h +++ b/src/server.h @@ -1532,6 +1532,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); +void TerminateModuleForkChild(int wait); /* Utils */ long long ustime(void); From d7c25806c0b40c096bbb2ea3cc36d2e0491ec7a5 Mon Sep 17 00:00:00 2001 From: zhudacai 00228490 Date: Sat, 31 Aug 2019 07:47:11 +0000 Subject: [PATCH 017/116] src/debug.c do not support aarch64 dump utcontext, so add the context of aarch64. The content comes from the definition of the sigcontext and tested on my aarch64 server. sigcontext defined at the linux kernel code: arch/arm64/include/uapi/asm/sigcontext.h --- src/debug.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/debug.c b/src/debug.c index 1f1157d4..595e45f7 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1110,6 +1110,33 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.mc_cs ); logStackContent((void**)uc->uc_mcontext.mc_rsp); +#elif defined(__aarch64__) /* Linux AArch64 */ + serverLog(LL_WARNING, + "\n" + "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" + "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" + "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" + "X30:%016lx\n" + "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.regs[18], + (unsigned long) uc->uc_mcontext.regs[19], + (unsigned long) uc->uc_mcontext.regs[20], + (unsigned long) uc->uc_mcontext.regs[21], + (unsigned long) uc->uc_mcontext.regs[22], + (unsigned long) uc->uc_mcontext.regs[23], + (unsigned long) uc->uc_mcontext.regs[24], + (unsigned long) uc->uc_mcontext.regs[25], + (unsigned long) uc->uc_mcontext.regs[26], + (unsigned long) uc->uc_mcontext.regs[27], + (unsigned long) uc->uc_mcontext.regs[28], + (unsigned long) uc->uc_mcontext.regs[29], + (unsigned long) uc->uc_mcontext.regs[30], + (unsigned long) uc->uc_mcontext.pc, + (unsigned long) uc->uc_mcontext.sp, + (unsigned long) uc->uc_mcontext.pstate, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.sp); #else serverLog(LL_WARNING, " Dumping of registers not supported for this OS/arch"); From 66a5c1cd7e557187bf3a1913e836d0c9be8adbb7 Mon Sep 17 00:00:00 2001 From: zhudacai 00228490 Date: Wed, 4 Sep 2019 12:14:25 +0000 Subject: [PATCH 018/116] The aarch64 architecture is support normal memory unaligned accesses, so add the UNALIGNED_LE_CPU to the aarch64 . --- src/siphash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/siphash.c b/src/siphash.c index 6b941903..35774113 100644 --- a/src/siphash.c +++ b/src/siphash.c @@ -58,7 +58,8 @@ int siptlw(int c) { /* Test of the CPU is Little Endian and supports not aligned accesses. * Two interesting conditions to speedup the function that happen to be * in most of x86 servers. */ -#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) +#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) \ + || defined (__aarch64__) || defined (__arm64__) #define UNALIGNED_LE_CPU #endif From cc8f06ece996238111150f6730d8339e3603b5ff Mon Sep 17 00:00:00 2001 From: Doug Nelson Date: Thu, 5 Sep 2019 16:25:06 +0100 Subject: [PATCH 019/116] Typo fixes in API documentation --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index ab614c52..85674ba0 100644 --- a/src/module.c +++ b/src/module.c @@ -2389,7 +2389,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { * * REDISMODULE_HASH_EXISTS: instead of setting the value of the field * expecting a RedisModuleString pointer to pointer, the function just - * reports if the field esists or not and expects an integer pointer + * reports if the field exists or not and expects an integer pointer * as the second element of each pair. * * Example of REDISMODULE_HASH_CFIELD: @@ -3288,7 +3288,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) { * RedisModule_Realloc() or RedisModule_Free(). * * The size of the string is stored at '*lenptr' if not NULL. - * The returned string is not automatically NULL termianted, it is loaded + * The returned string is not automatically NULL terminated, it is loaded * exactly as it was stored inisde the RDB file. */ char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { return moduleLoadString(io,1,lenptr); From 781f3fd8be048773d5a0052150df478db8d0979e Mon Sep 17 00:00:00 2001 From: suntiawnen Date: Fri, 6 Sep 2019 12:01:44 +0800 Subject: [PATCH 020/116] fix rdb function rdbLoadIntegerObject comment --- src/rdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index e4dfc46b..d11fb93f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -260,7 +260,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) { /* Loads an integer-encoded object with the specified encoding type "enctype". * The returned value changes according to the flags, see - * rdbGenerincLoadStringObject() for more info. */ + * rdbGenericLoadStringObject() for more info. */ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) { int plain = flags & RDB_LOAD_PLAIN; int sds = flags & RDB_LOAD_SDS; From 9d2ecf6be364dc5686e5f8d3269dc77c64014e35 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:21:34 +0200 Subject: [PATCH 021/116] ACL: add slightly modified version of sha256.c for password hashing. memory.h include removed, types substituted with stdint types. --- src/Makefile | 2 +- src/sha256.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/sha256.h | 35 ++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 src/sha256.c create mode 100644 src/sha256.h diff --git a/src/Makefile b/src/Makefile index b6cc69e2..198d85cd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -164,7 +164,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o sha256.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/sha256.c b/src/sha256.c new file mode 100644 index 00000000..d644d2d4 --- /dev/null +++ b/src/sha256.c @@ -0,0 +1,158 @@ +/********************************************************************* +* Filename: sha256.c +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Implementation of the SHA-256 hashing algorithm. + SHA-256 is one of the three algorithms in the SHA2 + specification. The others, SHA-384 and SHA-512, are not + offered in this implementation. + Algorithm specification can be found here: + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf + This implementation uses little endian byte order. +*********************************************************************/ + +/*************************** HEADER FILES ***************************/ +#include +#include +#include "sha256.h" + +/****************************** MACROS ******************************/ +#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) +#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) +#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) +#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) + +/**************************** VARIABLES *****************************/ +static const WORD k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +/*********************** FUNCTION DEFINITIONS ***********************/ +void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) +{ + WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; + + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + for ( ; i < 64; ++i) + m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + + a = ctx->state[0]; + b = ctx->state[1]; + c = ctx->state[2]; + d = ctx->state[3]; + e = ctx->state[4]; + f = ctx->state[5]; + g = ctx->state[6]; + h = ctx->state[7]; + + for (i = 0; i < 64; ++i) { + t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i]; + t2 = EP0(a) + MAJ(a,b,c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; + ctx->state[4] += e; + ctx->state[5] += f; + ctx->state[6] += g; + ctx->state[7] += h; +} + +void sha256_init(SHA256_CTX *ctx) +{ + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; + ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; + ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; + ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; + ctx->state[7] = 0x5be0cd19; +} + +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) +{ + WORD i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void sha256_final(SHA256_CTX *ctx, BYTE hash[]) +{ + WORD i; + + i = ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[63] = ctx->bitlen; + ctx->data[62] = ctx->bitlen >> 8; + ctx->data[61] = ctx->bitlen >> 16; + ctx->data[60] = ctx->bitlen >> 24; + ctx->data[59] = ctx->bitlen >> 32; + ctx->data[58] = ctx->bitlen >> 40; + ctx->data[57] = ctx->bitlen >> 48; + ctx->data[56] = ctx->bitlen >> 56; + sha256_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and SHA uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; + } +} diff --git a/src/sha256.h b/src/sha256.h new file mode 100644 index 00000000..dc53ead2 --- /dev/null +++ b/src/sha256.h @@ -0,0 +1,35 @@ +/********************************************************************* +* Filename: sha256.h +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Defines the API for the corresponding SHA1 implementation. +*********************************************************************/ + +#ifndef SHA256_H +#define SHA256_H + +/*************************** HEADER FILES ***************************/ +#include +#include + +/****************************** MACROS ******************************/ +#define SHA256_BLOCK_SIZE 32 // SHA256 outputs a 32 byte digest + +/**************************** DATA TYPES ****************************/ +typedef uint8_t BYTE; // 8-bit byte +typedef uint32_t WORD; // 32-bit word + +typedef struct { + BYTE data[64]; + WORD datalen; + unsigned long long bitlen; + WORD state[8]; +} SHA256_CTX; + +/*********************** FUNCTION DECLARATIONS **********************/ +void sha256_init(SHA256_CTX *ctx); +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len); +void sha256_final(SHA256_CTX *ctx, BYTE hash[]); + +#endif // SHA256_H From ae5054b47620957b5e74074b1afcdf042e3f57f7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:33:22 +0200 Subject: [PATCH 022/116] ACL: SHA256 based password hashing function implemented. --- src/acl.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/acl.c b/src/acl.c index a2ee65dd..aea4fe97 100644 --- a/src/acl.c +++ b/src/acl.c @@ -28,6 +28,7 @@ */ #include "server.h" +#include "sha256.h" #include /* ============================================================================= @@ -139,6 +140,25 @@ int time_independent_strcmp(char *a, char *b) { return diff; /* If zero strings are the same. */ } +/* Given an SDS string, returns the SHA256 hex representation as a + * new SDS string. */ +sds ACLHashPassword(sds cleartext) { + SHA256_CTX ctx; + unsigned char hash[SHA256_BLOCK_SIZE]; + char hex[SHA256_BLOCK_SIZE*2]; + char *cset = "0123456789abcdef"; + + sha256_init(&ctx); + sha256_update(&ctx,(unsigned char*)cleartext,sdslen(cleartext)); + sha256_final(&ctx,hash); + + for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { + hex[j*2] = cset[((hash[j]&0xF0)>>4)]; + hex[j*2+1] = cset[(hash[j]&0xF)]; + } + return sdsnewlen(hex,SHA256_BLOCK_SIZE*2); +} + /* ============================================================================= * Low level ACL API * ==========================================================================*/ From b170a01fa77cb043e9c9e6185e706a20ddfe0608 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:54:57 +0200 Subject: [PATCH 023/116] ACL: store hashed passwords in memory. Note that this breaks API compatibility with Redis < 6: CONFIG GET requirepass Will no longer return a cleartext password as well, but the SHA256 hash of the password set. --- src/acl.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/acl.c b/src/acl.c index aea4fe97..2cd729e7 100644 --- a/src/acl.c +++ b/src/acl.c @@ -142,14 +142,14 @@ int time_independent_strcmp(char *a, char *b) { /* Given an SDS string, returns the SHA256 hex representation as a * new SDS string. */ -sds ACLHashPassword(sds cleartext) { +sds ACLHashPassword(unsigned char *cleartext, size_t len) { SHA256_CTX ctx; unsigned char hash[SHA256_BLOCK_SIZE]; char hex[SHA256_BLOCK_SIZE*2]; char *cset = "0123456789abcdef"; sha256_init(&ctx); - sha256_update(&ctx,(unsigned char*)cleartext,sdslen(cleartext)); + sha256_update(&ctx,(unsigned char*)cleartext,len); sha256_final(&ctx,hash); for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { @@ -721,13 +721,16 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { u->flags &= ~USER_FLAG_NOPASS; listEmpty(u->passwords); } else if (op[0] == '>') { - sds newpass = sdsnewlen(op+1,oplen-1); + sds newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); listNode *ln = listSearchKey(u->passwords,newpass); /* Avoid re-adding the same password multiple times. */ - if (ln == NULL) listAddNodeTail(u->passwords,newpass); + if (ln == NULL) + listAddNodeTail(u->passwords,newpass); + else + sdsfree(newpass); u->flags &= ~USER_FLAG_NOPASS; } else if (op[0] == '<') { - sds delpass = sdsnewlen(op+1,oplen-1); + sds delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); listNode *ln = listSearchKey(u->passwords,delpass); sdsfree(delpass); if (ln) { @@ -744,7 +747,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); /* Avoid re-adding the same pattern multiple times. */ - if (ln == NULL) listAddNodeTail(u->patterns,newpat); + if (ln == NULL) + listAddNodeTail(u->patterns,newpat); + else + sdsfree(newpat); u->flags &= ~USER_FLAG_ALLKEYS; } else if (op[0] == '+' && op[1] != '@') { if (strchr(op,'|') == NULL) { @@ -899,11 +905,15 @@ int ACLCheckUserCredentials(robj *username, robj *password) { listIter li; listNode *ln; listRewind(u->passwords,&li); + sds hashed = ACLHashPassword(password->ptr,sdslen(password->ptr)); while((ln = listNext(&li))) { sds thispass = listNodeValue(ln); - if (!time_independent_strcmp(password->ptr, thispass)) + if (!time_independent_strcmp(hashed, thispass)) { + sdsfree(hashed); return C_OK; + } } + sdsfree(hashed); /* If we reached this point, no password matched. */ errno = EINVAL; From 93c52ff5ff8145b570969f695e960cb502c79bb1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:01:39 +0200 Subject: [PATCH 024/116] RESP3: implement lua.setresp(). --- src/scripting.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 032bfdf1..d5da1433 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -42,7 +42,7 @@ char *redisProtocolToLuaType_Int(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); -char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype); +char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -132,9 +132,9 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '$': p = redisProtocolToLuaType_Bulk(lua,reply); break; case '+': p = redisProtocolToLuaType_Status(lua,reply); break; case '-': p = redisProtocolToLuaType_Error(lua,reply); break; - case '*': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; - case '%': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; - case '~': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; + case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; } return p; } @@ -182,7 +182,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply) { return p+2; } -char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) { +char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { char *p = strchr(reply+1,'\r'); long long mbulklen; int j = 0; @@ -859,6 +859,25 @@ int luaLogCommand(lua_State *lua) { return 0; } +/* redis.setresp() */ +int luaSetResp(lua_State *lua) { + int argc = lua_gettop(lua); + + if (argc != 1) { + lua_pushstring(lua, "redis.setresp() requires one argument."); + return lua_error(lua); + } + + int resp = lua_tonumber(lua,-argc); + if (resp != 2 && resp != 3) { + lua_pushstring(lua, "RESP version must be 2 or 3."); + return lua_error(lua); + } + + server.lua_client->resp = resp; + return 0; +} + /* --------------------------------------------------------------------------- * Lua engine initialization and reset. * ------------------------------------------------------------------------- */ @@ -986,6 +1005,11 @@ void scriptingInit(int setup) { lua_pushcfunction(lua,luaLogCommand); lua_settable(lua,-3); + /* redis.setresp */ + lua_pushstring(lua,"setresp"); + lua_pushcfunction(lua,luaSetResp); + lua_settable(lua,-3); + lua_pushstring(lua,"LOG_DEBUG"); lua_pushnumber(lua,LL_DEBUG); lua_settable(lua,-3); @@ -1379,8 +1403,9 @@ void evalGenericCommand(client *c, int evalsha) { luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys); luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys); - /* Select the right DB in the context of the Lua client */ + /* Set the Lua client database and protocol. */ selectDb(server.lua_client,c->db->id); + server.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */ /* Set a hook in order to be able to stop the script execution if it * is running for too much time. From 26b6c697d3d9b09b49ad5a1128190d5ba6dba09a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:19:10 +0200 Subject: [PATCH 025/116] RESP3: Lua debugger support for printing sets and maps. --- src/scripting.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index d5da1433..68c7ce00 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2078,6 +2078,8 @@ char *ldbRedisProtocolToHuman_Int(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2092,6 +2094,8 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '+': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '-': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; + case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break; + case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; } return p; } @@ -2146,6 +2150,40 @@ char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply) { return p; } +char *ldbRedisProtocolToHuman_Set(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + long long mbulklen; + int j = 0; + + string2ll(reply+1,p-reply-1,&mbulklen); + p += 2; + *o = sdscatlen(*o,"~(",2); + for (j = 0; j < mbulklen; j++) { + p = ldbRedisProtocolToHuman(o,p); + if (j != mbulklen-1) *o = sdscatlen(*o,",",1); + } + *o = sdscatlen(*o,")",1); + return p; +} + +char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + long long mbulklen; + int j = 0; + + string2ll(reply+1,p-reply-1,&mbulklen); + p += 2; + *o = sdscatlen(*o,"{",1); + for (j = 0; j < mbulklen; j++) { + p = ldbRedisProtocolToHuman(o,p); + *o = sdscatlen(*o," => ",4); + p = ldbRedisProtocolToHuman(o,p); + if (j != mbulklen-1) *o = sdscatlen(*o,",",1); + } + *o = sdscatlen(*o,"}",1); + return p; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From 888efc1b3642959b05815473663a5f0971af53ac Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:38:35 +0200 Subject: [PATCH 026/116] RESP3: Lua parsing should depend on lua client, not lua caller. We want all the scripts to run in RESP2 mode by default. It's up to the caller to switch to V3 using redis.setresp() if it is really needed. This way most scripts written for past Redis versions will continue to work with Redis >= 6 even if the client is in RESP3 mode. --- src/scripting.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 68c7ce00..ee37ac23 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -188,7 +188,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { int j = 0; string2ll(reply+1,p-reply-1,&mbulklen); - if (server.lua_caller->resp == 2 || atype == '*') { + if (server.lua_client->resp == 2 || atype == '*') { p += 2; if (mbulklen == -1) { lua_pushboolean(lua,0); @@ -200,7 +200,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { p = redisProtocolToLuaType(lua,p); lua_settable(lua,-3); } - } else if (server.lua_caller->resp == 3) { + } else if (server.lua_client->resp == 3) { /* Here we handle only Set and Map replies in RESP3 mode, since arrays * follow the above RESP2 code path. */ p += 2; From 4a30a26f8fbb54854f5b29b77fd3c744e8be4290 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 15 Sep 2019 21:16:30 +0100 Subject: [PATCH 027/116] [add] improved performance of RM_ReplyWithSimpleString and RM_ReplyWithError by making usage addReplyProto instead of addReplySds --- src/module.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/module.c b/src/module.c index ab614c52..ad7e6864 100644 --- a/src/module.c +++ b/src/module.c @@ -1120,19 +1120,6 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { return REDISMODULE_OK; } -/* Reply with an error or simple string (status message). Used to implement - * ReplyWithSimpleString() and ReplyWithError(). - * The function always returns REDISMODULE_OK. */ -int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - sds strmsg = sdsnewlen(prefix,1); - strmsg = sdscat(strmsg,msg); - strmsg = sdscatlen(strmsg,"\r\n",2); - addReplySds(c,strmsg); - return REDISMODULE_OK; -} - /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including @@ -1148,7 +1135,13 @@ int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { * The function always returns REDISMODULE_OK. */ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { - return replyWithStatus(ctx,err,"-"); + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(err); + addReplyProto(c,"-",1); + addReplyProto(c,err,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; } /* Reply with a simple string (+... \r\n in RESP protocol). This replies @@ -1157,7 +1150,13 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) { - return replyWithStatus(ctx,msg,"+"); + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(msg); + addReplyProto(c,"+",1); + addReplyProto(c,msg,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; } /* Reply with an array type of 'len' elements. However 'len' other calls From dad38c19c811ca222abd7f4cbb9473bceb82b69c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 11:49:42 +0200 Subject: [PATCH 028/116] RESP3: report set/map as nested tables to Lua. --- src/scripting.c | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index ee37ac23..39f75abf 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -202,9 +202,13 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { } } else if (server.lua_client->resp == 3) { /* Here we handle only Set and Map replies in RESP3 mode, since arrays - * follow the above RESP2 code path. */ + * follow the above RESP2 code path. Note that those are represented + * as a table with the "map" or "set" field populated with the actual + * table representing the set or the map type. */ p += 2; lua_newtable(lua); + lua_pushstring(lua,atype == '%' ? "map" : "set"); + lua_newtable(lua); for (j = 0; j < mbulklen; j++) { p = redisProtocolToLuaType(lua,p); if (atype == '%') { @@ -214,6 +218,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { } lua_settable(lua,-3); } + lua_settable(lua,-3); } return p; } @@ -310,6 +315,8 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { * Error are returned as a single element table with 'err' field. * Status replies are returned as single element table with 'ok' * field. */ + + /* Handle error reply. */ lua_pushstring(lua,"err"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -321,8 +328,9 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { lua_pop(lua,2); return; } + lua_pop(lua,1); /* Discard field name pushed before. */ - lua_pop(lua,1); + /* Handle status reply. */ lua_pushstring(lua,"ok"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -332,24 +340,25 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); sdsfree(ok); lua_pop(lua,1); - } else { - void *replylen = addReplyDeferredLen(c); - int j = 1, mbulklen = 0; - - lua_pop(lua,1); /* Discard the 'ok' field value we popped */ - while(1) { - lua_pushnumber(lua,j++); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TNIL) { - lua_pop(lua,1); - break; - } - luaReplyToRedisReply(c, lua); - mbulklen++; - } - setDeferredArrayLen(c,replylen,mbulklen); + return; } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle the array reply. */ + void *replylen = addReplyDeferredLen(c); + int j = 1, mbulklen = 0; + while(1) { + lua_pushnumber(lua,j++); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNIL) { + lua_pop(lua,1); + break; + } + luaReplyToRedisReply(c, lua); + mbulklen++; + } + setDeferredArrayLen(c,replylen,mbulklen); break; default: addReplyNull(c); From c792504fabe6c64248546a63b5356cd29e7a6438 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 12:15:39 +0200 Subject: [PATCH 029/116] RESP3: handle map Lua -> Redis conversion. --- src/scripting.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 39f75abf..cd55000d 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -297,6 +297,8 @@ void luaSortArray(lua_State *lua) { * Lua reply to Redis reply conversion functions. * ------------------------------------------------------------------------- */ +/* Reply to client 'c' converting the top element in the Lua stack to a + * Redis reply. As a side effect the element is consumed from the stack. */ void luaReplyToRedisReply(client *c, lua_State *lua) { int t = lua_type(lua,-1); @@ -339,7 +341,29 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { sdsmapchars(ok,"\r\n"," ",2); addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); sdsfree(ok); - lua_pop(lua,1); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle map reply. */ + lua_pushstring(lua,"map"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int maplen = 0; + void *replylen = addReplyDeferredLen(c); + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, value */ + luaReplyToRedisReply(c, lua); /* Return value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + maplen++; + } + setDeferredMapLen(c,replylen,maplen); + lua_pop(lua,2); return; } lua_pop(lua,1); /* Discard field name pushed before. */ From ca81d4900647f2cca71499fb4a46b43f5a90a24c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 12:19:19 +0200 Subject: [PATCH 030/116] RESP3: handle set Lua -> Redis conversion. --- src/scripting.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index cd55000d..2c90bb7a 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -368,6 +368,28 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { } lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle set reply. */ + lua_pushstring(lua,"set"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int setlen = 0; + void *replylen = addReplyDeferredLen(c); + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, true */ + lua_pop(lua,1); /* Discard the boolean value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + setlen++; + } + setDeferredSetLen(c,replylen,setlen); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle the array reply. */ void *replylen = addReplyDeferredLen(c); int j = 1, mbulklen = 0; From 2cc4d0286c4d92f5f055c157fcc5673c7f90cff0 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 17:49:40 +0200 Subject: [PATCH 031/116] RESP3: implement new NULL representation parsing in Lua. --- src/scripting.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 2c90bb7a..564dad8c 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -43,6 +43,7 @@ char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); +char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -135,6 +136,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '_': p = redisProtocolToLuaType_Null(lua,reply); break; } return p; } @@ -223,6 +225,12 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { return p; } +char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) { + char *p = strchr(reply+1,'\r'); + lua_pushboolean(lua,0); + return p+2; +} + /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table * with a single "err" field set to the error string. Note that this From 6931004969d23dc6233a94e422090f35ebec143d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 18:18:17 +0200 Subject: [PATCH 032/116] RESP3: change behavior of Lua returning true/false for RESP3. Here we introduce a change in the way we convert values from Lua to Redis when RESP3 is selected: this is possible without breaking the fact we can return directly what a command returned, because there is no Redis command in RESP2 that returns true or false to Lua, so the conversion in the case of RESP2 is totally arbitrary. When a script is written selecting RESP3 from Lua, it totally makes sense to change such behavior and return RESP3 true/false when Lua true/false is returned. --- src/scripting.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 564dad8c..3ad228a6 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -315,7 +315,11 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); break; case LUA_TBOOLEAN: - addReply(c,lua_toboolean(lua,-1) ? shared.cone : shared.null[c->resp]); + if (server.lua_client->resp == 2) + addReply(c,lua_toboolean(lua,-1) ? shared.cone : + shared.null[c->resp]); + else + addReplyBool(c,lua_toboolean(lua,-1)); break; case LUA_TNUMBER: addReplyLongLong(c,(long long)lua_tonumber(lua,-1)); From f01f0c02d14149e826532ba20f084244fdf29d20 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 18:36:16 +0200 Subject: [PATCH 033/116] RESP3: convert RESP3 null as Lua nil. Implement RESP3->Lua bools. --- src/scripting.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 3ad228a6..9ab1e7ab 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -44,6 +44,7 @@ char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); +char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -137,6 +138,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; + case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); } return p; } @@ -227,7 +229,13 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) { char *p = strchr(reply+1,'\r'); - lua_pushboolean(lua,0); + lua_pushnil(lua); + return p+2; +} + +char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) { + char *p = strchr(reply+1,'\r'); + lua_pushboolean(lua,tf == 't'); return p+2; } From c1cc5ca7671a5d2b35e1d7e1b7c0e67ab8c0bc91 Mon Sep 17 00:00:00 2001 From: Okada Haruki Date: Tue, 17 Sep 2019 06:18:01 +0900 Subject: [PATCH 034/116] Fix typo --- src/hyperloglog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 5d4afaa2..a44d1564 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1242,7 +1242,7 @@ void pfcountCommand(client *c) { if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ if (isHLLObjectOrReply(c,o) != C_OK) return; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(registers,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); @@ -1329,7 +1329,7 @@ void pfmergeCommand(client *c) { hdr = o->ptr; if (hdr->encoding == HLL_DENSE) use_dense = 1; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(max,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); From e8e30bc40289e43cbd07f532d4ca4c16e03f239c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 18:57:24 +0200 Subject: [PATCH 035/116] RESP3: bool and null values in RESP -> human readable conversion. --- src/scripting.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 9ab1e7ab..b76ff278 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2155,6 +2155,8 @@ char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Null(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2171,6 +2173,8 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break; case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; + case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break; + case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break; } return p; } @@ -2259,6 +2263,21 @@ char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) { return p; } +char *ldbRedisProtocolToHuman_Null(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + *o = sdscatlen(*o,"(null)",6); + return p+2; +} + +char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + if (reply[1] == 't') + *o = sdscatlen(*o,"#true",5); + else + *o = sdscatlen(*o,"#false",6); + return p+2; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From 19aba4ac7828584be3e0621a8db116646f6ece07 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:08:33 +0200 Subject: [PATCH 036/116] RESP3: double -> human readable conversion. --- src/scripting.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index b76ff278..45621053 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2157,6 +2157,7 @@ char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); char *ldbRedisProtocolToHuman_Null(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Double(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2175,6 +2176,7 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break; case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break; + case ',': p = ldbRedisProtocolToHuman_Double(o,reply); break; } return p; } @@ -2278,6 +2280,13 @@ char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) { return p+2; } +char *ldbRedisProtocolToHuman_Double(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + *o = sdscatlen(*o,"(double) ",9); + *o = sdscatlen(*o,reply+1,p-reply-1); + return p+2; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From 89f929b12a3ff27593d25ec039dc0e9d6a049084 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:20:30 +0200 Subject: [PATCH 037/116] RESP3: RESP3 double -> Lua conversion. --- src/scripting.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 45621053..7230f743 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -45,6 +45,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf); +char *redisProtocolToLuaType_Double(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -139,6 +140,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); + case ',': p = redisProtocolToLuaType_Double(lua,reply); } return p; } @@ -239,6 +241,27 @@ char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) { return p+2; } +char *redisProtocolToLuaType_Double(lua_State *lua, char *reply) { + char *p = strchr(reply+1,'\r'); + char buf[MAX_LONG_DOUBLE_CHARS+1]; + size_t len = p-reply-1; + double d; + + if (len <= MAX_LONG_DOUBLE_CHARS) { + memcpy(buf,reply+1,len); + buf[len] = '\0'; + d = strtod(buf,NULL); /* We expect a valid representation. */ + } else { + d = 0; + } + + lua_newtable(lua); + lua_pushstring(lua,"double"); + lua_pushnumber(lua,d); + lua_settable(lua,-3); + return p+2; +} + /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table * with a single "err" field set to the error string. Note that this From 8ea4bdd91d600aa1e9e1ebf7dace19172a7ce23a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:26:46 +0200 Subject: [PATCH 038/116] RESP3: Lua double -> RESP3 conversion. --- src/scripting.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 7230f743..2153233f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -389,6 +389,17 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { } lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle double reply. */ + lua_pushstring(lua,"double"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNUMBER) { + addReplyDouble(c,lua_tonumber(lua,-1)); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle map reply. */ lua_pushstring(lua,"map"); lua_gettable(lua,-2); From 68519b703461bb0890cf347ef575c21423746f0c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:28:39 +0200 Subject: [PATCH 039/116] RESP3: Verbatim conversion test in the LOLWUT command. redis-cli is currently not able to handle it after going in RESP3 mode, because of hiredis limitations. --- src/lolwut.c | 3 ++- src/lolwut5.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index 19cbcf64..ba7e1069 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -43,7 +43,8 @@ void lolwutUnstableCommand(client *c) { sds rendered = sdsnew("Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); } void lolwutCommand(client *c) { diff --git a/src/lolwut5.c b/src/lolwut5.c index 8408b378..52a98c0d 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -277,6 +277,7 @@ void lolwut5Command(client *c) { "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); lwFreeCanvas(canvas); } From aca5482fbf1d958f7bacfb150ce5825a25cf91bd Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:33:01 +0200 Subject: [PATCH 040/116] RESP3: Use verbatim in INFO output. --- src/server.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index fb308e5f..7882b0d9 100644 --- a/src/server.c +++ b/src/server.c @@ -4391,7 +4391,9 @@ void infoCommand(client *c) { addReply(c,shared.syntaxerr); return; } - addReplyBulkSds(c, genRedisInfoString(section)); + sds info = genRedisInfoString(section); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } void monitorCommand(client *c) { From ff9a5d231b7c36e8aa485fb9a765a82fe0cde462 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:46:11 +0200 Subject: [PATCH 041/116] RESP3: Use verbatim in DEBUG HTSTATS / HTSTATS-KEY. --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 1f1157d4..93173f3b 100644 --- a/src/debug.c +++ b/src/debug.c @@ -638,7 +638,8 @@ NULL dictGetStats(buf,sizeof(buf),server.db[dbid].expires); stats = sdscat(stats,buf); - addReplyBulkSds(c,stats); + addReplyVerbatim(c,stats,sdslen(stats),"txt"); + sdsfree(stats); } else if (!strcasecmp(c->argv[1]->ptr,"htstats-key") && c->argc == 3) { robj *o; dict *ht = NULL; @@ -665,7 +666,7 @@ NULL } else { char buf[4096]; dictGetStats(buf,sizeof(buf),ht); - addReplyBulkCString(c,buf); + addReplyVerbatim(c,buf,strlen(buf),"txt"); } } else if (!strcasecmp(c->argv[1]->ptr,"change-repl-id") && c->argc == 2) { serverLog(LL_WARNING,"Changing replication IDs after receiving DEBUG change-repl-id"); From 1b3cb3b0de0f3049d3d2793b34a5d9c247ccc4d8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:48:14 +0200 Subject: [PATCH 042/116] RESP3: Use verbatim in MEMORY subcommands. --- src/object.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index fb011d31..4534d7c0 100644 --- a/src/object.c +++ b/src/object.c @@ -1440,13 +1440,15 @@ NULL #if defined(USE_JEMALLOC) sds info = sdsempty(); je_malloc_stats_print(inputCatSds, &info, NULL); - addReplyBulkSds(c, info); + addReplyVerbatim(c,info,sdslen(info),"txt") + sdsfree(info); #else addReplyBulkCString(c,"Stats not supported for the current allocator"); #endif } else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) { sds report = getMemoryDoctorReport(); - addReplyBulkSds(c,report); + addReplyVerbatim(c,report,sdslen(report),"txt"); + sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) { #if defined(USE_JEMALLOC) char tmp[32]; From dd2f695d7e52185728d0d3a215b7fdae1986eb9a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:51:15 +0200 Subject: [PATCH 043/116] RESP3: Use verbatim in CLUSTER subcommands. --- src/cluster.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index a2615fdc..1e7dcd50 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4252,7 +4252,9 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - addReplyBulkSds(c,clusterGenNodesDescription(0)); + sds nodes = clusterGenNodesDescription(0); + addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); + sdsfree(nodes); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4494,10 +4496,8 @@ NULL "cluster_stats_messages_received:%lld\r\n", tot_msg_received); /* Produce the reply protocol. */ - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); From cb384127d1ba8e85ccd507599df46541f1349b4c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:52:13 +0200 Subject: [PATCH 044/116] RESP3: Use verbatim in CLIENT LIST. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 7555ca77..a959d557 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1990,7 +1990,7 @@ NULL return; } sds o = getAllClientsInfoString(type); - addReplyBulkCBuffer(c,o,sdslen(o)); + addReplyVerbatim(c,o,sdslen(o),"txt"); sdsfree(o); } else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) { /* CLIENT REPLY ON|OFF|SKIP */ From 474a9231605f94a1265583282ccc7c123956e30b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:53:22 +0200 Subject: [PATCH 045/116] RESP3: Use verbatim in LATENCY subcommands. --- src/latency.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/latency.c b/src/latency.c index 33aa1245..b834da5c 100644 --- a/src/latency.c +++ b/src/latency.c @@ -599,7 +599,7 @@ NULL event = dictGetKey(de); graph = latencyCommandGenSparkeline(event,ts); - addReplyBulkCString(c,graph); + addReplyVerbatim(c,graph,sdslen(graph),"txt"); sdsfree(graph); } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { /* LATENCY LATEST */ @@ -608,7 +608,7 @@ NULL /* LATENCY DOCTOR */ sds report = createLatencyReport(); - addReplyBulkCBuffer(c,report,sdslen(report)); + addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { /* LATENCY RESET */ From 74e3a622a2af2c7f561b058de1249c081ae82afd Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Fri, 20 Sep 2019 08:14:36 +0800 Subject: [PATCH 046/116] Fix bad handling of unexpected option while loading config "lua-replicate-commands". --- src/config.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config.c b/src/config.c index a72df2e7..ae4d4a0e 100644 --- a/src/config.c +++ b/src/config.c @@ -672,6 +672,9 @@ void loadServerConfigFromString(char *config) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { server.lua_always_replicate_commands = yesnotoi(argv[1]); + if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { From 30a3644e643ed60bedd839b2a3f9ec59ff25c00b Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Fri, 20 Sep 2019 08:37:23 +0800 Subject: [PATCH 047/116] RESP3: Fix function redisProtocolToLuaType about RESP3->Lua bools. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 2153233f..fa896de2 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -139,7 +139,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; - case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); + case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break; case ',': p = redisProtocolToLuaType_Double(lua,reply); } return p; From a1a65d238c4b144b34960a1e7be94766da35a81f Mon Sep 17 00:00:00 2001 From: Evgeny Date: Fri, 20 Sep 2019 01:11:20 -0700 Subject: [PATCH 048/116] Fix compilation error --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index 4534d7c0..697429b8 100644 --- a/src/object.c +++ b/src/object.c @@ -1440,7 +1440,7 @@ NULL #if defined(USE_JEMALLOC) sds info = sdsempty(); je_malloc_stats_print(inputCatSds, &info, NULL); - addReplyVerbatim(c,info,sdslen(info),"txt") + addReplyVerbatim(c,info,sdslen(info),"txt"); sdsfree(info); #else addReplyBulkCString(c,"Stats not supported for the current allocator"); From 0a146a8be4aea201351d4292a936bbf508c48d4b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:18:59 +0200 Subject: [PATCH 049/116] Add useless break for uniformity / future protection. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index fa896de2..9ac8af2e 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -140,7 +140,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break; - case ',': p = redisProtocolToLuaType_Double(lua,reply); + case ',': p = redisProtocolToLuaType_Double(lua,reply); break; } return p; } From 630638dcdeafd5b285b82c1e004abf690e620d67 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:44:32 +0200 Subject: [PATCH 050/116] Remove redundant statement in config.c. Thanks to @guybe7 for spotting the error in the original PR I merged. --- src/config.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index ae4d4a0e..fe002bac 100644 --- a/src/config.c +++ b/src/config.c @@ -671,8 +671,9 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - server.lua_always_replicate_commands = yesnotoi(argv[1]); - if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) == -1) { + if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) + == -1) + { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && From 2e4fa7bb48875984c285a63475fbdb8b26e279f3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:46:35 +0200 Subject: [PATCH 051/116] Make config.c always_replicate_commands more uniform. Better if it resembles the other similar code paths. --- src/config.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fe002bac..d37e3c56 100644 --- a/src/config.c +++ b/src/config.c @@ -671,10 +671,10 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) - == -1) - { - err = "argument must be 'yes' or 'no'"; goto loaderr; + server.lua_always_replicate_commands = yesnotoi(argv[1]); + if (server.lua_always_replicate_commands == -1) { + err = "argument must be 'yes' or 'no'"; + goto loaderr; } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) From c3899720f64e62920c7d90bc009afdca5824c087 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 20:06:47 +0200 Subject: [PATCH 052/116] hiredis updated to master version. --- deps/hiredis/.gitignore | 1 + deps/hiredis/.travis.yml | 74 ++- deps/hiredis/CHANGELOG.md | 15 +- deps/hiredis/CMakeLists.txt | 90 ++++ deps/hiredis/Makefile | 106 ++++- deps/hiredis/README.md | 3 +- deps/hiredis/adapters/libevent.h | 112 ++++- deps/hiredis/appveyor.yml | 7 +- deps/hiredis/async.c | 176 +++++--- deps/hiredis/async.h | 8 + deps/hiredis/async_private.h | 72 +++ deps/hiredis/examples/CMakeLists.txt | 46 ++ deps/hiredis/examples/example-libevent-ssl.c | 73 +++ deps/hiredis/examples/example-libevent.c | 15 +- deps/hiredis/examples/example-ssl.c | 97 ++++ deps/hiredis/examples/example.c | 17 +- deps/hiredis/hiredis.c | 253 ++++++----- deps/hiredis/hiredis.h | 103 ++++- deps/hiredis/hiredis.pc.in | 11 + deps/hiredis/hiredis_ssl.h | 53 +++ deps/hiredis/hiredis_ssl.pc.in | 12 + deps/hiredis/net.c | 122 +++-- deps/hiredis/net.h | 4 + deps/hiredis/read.c | 28 +- deps/hiredis/read.h | 5 +- deps/hiredis/sds.c | 2 +- deps/hiredis/sds.h | 31 +- deps/hiredis/sockcompat.c | 248 ++++++++++ deps/hiredis/sockcompat.h | 91 ++++ deps/hiredis/ssl.c | 448 +++++++++++++++++++ deps/hiredis/test.c | 93 +++- deps/hiredis/test.sh | 70 +++ deps/hiredis/win32.h | 18 +- 33 files changed, 2178 insertions(+), 326 deletions(-) create mode 100644 deps/hiredis/CMakeLists.txt create mode 100644 deps/hiredis/async_private.h create mode 100644 deps/hiredis/examples/CMakeLists.txt create mode 100644 deps/hiredis/examples/example-libevent-ssl.c create mode 100644 deps/hiredis/examples/example-ssl.c create mode 100644 deps/hiredis/hiredis.pc.in create mode 100644 deps/hiredis/hiredis_ssl.h create mode 100644 deps/hiredis/hiredis_ssl.pc.in create mode 100644 deps/hiredis/sockcompat.c create mode 100644 deps/hiredis/sockcompat.h create mode 100644 deps/hiredis/ssl.c create mode 100755 deps/hiredis/test.sh diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index c44b5c53..8e50b543 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -5,3 +5,4 @@ /*.dylib /*.a /*.pc +*.dSYM diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index faf2ce68..dd8e0e73 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -26,20 +26,72 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib - valgrind env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" + +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - ctest -V matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" - -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + script: + - mkdir build && cd build + - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && + ninja -v' + - ctest -V diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index a7fe3ac1..d1d37e51 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -12,6 +12,16 @@ compare to other values, casting might be necessary or can be removed, if casting was applied before. +### 0.x.x (unreleased) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. +If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + ### 0.14.0 (2018-09-25) * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) @@ -50,8 +60,9 @@ * Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 * Fix warnings, when compiled with -Wshadow * Make hiredis compile in Cygwin on Windows, now CI-tested - -**BREAKING CHANGES**: +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. * Remove backwards compatibility macro's diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt new file mode 100644 index 00000000..9e78894f --- /dev/null +++ b/deps/hiredis/CMakeLists.txt @@ -0,0 +1,90 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +ADD_LIBRARY(hiredis SHARED + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +INSTALL(FILES hiredis.h read.h sds.h async.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + ADD_LIBRARY(hiredis_ssl SHARED + ssl.c) + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +ENDIF() + +IF(NOT (WIN32 OR MINGW)) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 06ca9946..25ac1546 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o +OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis +SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -39,7 +45,7 @@ export REDIS_TEST_CONFIG CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) REAL_LDFLAGS=$(LDFLAGS) @@ -49,12 +55,30 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=$(AR) rcs # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + SSL_LDFLAGS=-lssl -lcrypto +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) REAL_LDFLAGS+= -ldl -lnsl -lsocket DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) @@ -66,40 +90,61 @@ ifeq ($(uname_S),Darwin) endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) ifndef AE_DIR hiredis-example-ae: @@ -116,7 +161,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -133,32 +178,33 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread +endif +hiredis-test: test.o $(TEST_LIBS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c INSTALL?= cp -pPR @@ -175,6 +221,20 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 01223ea5..c0b432f0 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -286,6 +286,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -406,6 +407,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 7d2bef18..a4952776 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_free(e->rev); - event_free(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,7 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)calloc(1, sizeof(*e)); e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +161,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml index 819efbd5..5b43fdbe 100644 --- a/deps/hiredis/appveyor.yml +++ b/deps/hiredis/appveyor.yml @@ -5,8 +5,9 @@ environment: CC: gcc - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 clone_depth: 1 @@ -20,4 +21,4 @@ install: build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,22 +42,9 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); +#include "async_private.h" /* Forward declaration of function in hiredis.c */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); @@ -126,6 +115,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -150,56 +140,52 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; - + } ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } - __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -328,7 +314,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -344,9 +330,15 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -358,6 +350,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -408,7 +403,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, assert(reply->element[2]->type == REDIS_REPLY_INTEGER); /* Unset subscribed flag only when no pipelined pending subscribe. */ - if (reply->element[2]->integer == 0 + if (reply->element[2]->integer == 0 && dictSize(ac->sub.channels) == 0 && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; @@ -524,6 +519,18 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { } } +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + /* This function should be called when the socket is readable. * It processes all replies that can be read and executes their callbacks. */ @@ -539,28 +546,13 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Always re-schedule reads */ - _EL_ADD_READ(ac); - redisProcessCallbacks(ac); - } + c->funcs->async_read(ac); } -void redisAsyncHandleWrite(redisAsyncContext *ac) { +void redisAsyncWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); int done = 0; - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { @@ -575,6 +567,51 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { } } +void redisAsyncHandleWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + c->funcs->async_write(ac); +} + +void __redisSetError(redisContext *c, int type, const char *str); + +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; + } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); +} + /* Sets a pointer to the first argument and its length starting at p. Returns * the number of bytes to skip to get to the following argument. */ static const char *nextArgument(const char *start, const char **str, size_t *len) { @@ -714,3 +751,16 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.timeout) { + ac->c.timeout = calloc(1, sizeof(tv)); + } + + if (tv.tv_sec == ac->c.timeout->tv_sec && + tv.tv_usec == ac->c.timeout->tv_usec) { + return; + } + + *ac->c.timeout = tv; +} diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 740555c2..4f6b3b78 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -57,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -81,6 +82,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -106,6 +108,7 @@ typedef struct redisAsyncContext { } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -113,12 +116,17 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h new file mode 100644 index 00000000..d0133ae1 --- /dev/null +++ b/deps/hiredis/async_private.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0); + +static inline void refreshTimeout(redisAsyncContext *ctx) { + if (ctx->c.timeout && ctx->ev.scheduleTimer && + (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { + ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); + // } else { + // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); + // if (ctx->c.timeout){ + // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, + // ctx->c.timeout->tv_usec); + // } + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 00000000..dd3a313a --- /dev/null +++ b/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,46 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 00000000..1021113b --- /dev/null +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index d333c22b..1fe71ae4 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -35,8 +40,14 @@ void disconnectCallback(const redisAsyncContext *c, int status) { int main (int argc, char **argv) { signal(SIGPIPE, SIG_IGN); struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.timeout = &tv; - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c new file mode 100644 index 00000000..81f4648c --- /dev/null +++ b/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,97 @@ +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 4d494c55..0e93fc8b 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -5,14 +5,27 @@ #include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 0947d1ed..282595bd 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,10 +41,20 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privdata = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); @@ -112,19 +121,31 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len if (r == NULL) return NULL; - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); + task->type == REDIS_REPLY_STRING || + task->type == REDIS_REPLY_VERB); /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; + if (task->type == REDIS_REPLY_VERB) { + buf = malloc(len+4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(r->vtype,buf,3); + r->vtype[3] = '\0'; + memcpy(buf+4,str,len-4); + buf[len-4] = '\0'; + } else { + buf = malloc(len+1); + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(buf,str,len); + buf[len] = '\0'; + } r->str = buf; r->len = len; @@ -138,7 +159,7 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len return r; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; r = createReplyObject(task->type); @@ -649,29 +670,30 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(void) { +static redisContext *redisContextInit(const redisOptions *options) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = calloc(1, sizeof(*c)); if (c == NULL) return NULL; + c->funcs = &redisContextDefaultFuncs; c->obuf = sdsempty(); c->reader = redisReaderCreate(); + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); return NULL; } - + (void)options; /* options are used in other functions */ return c; } void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); + redisNetClose(c); sdsfree(c->obuf); redisReaderFree(c->reader); @@ -680,12 +702,16 @@ void redisFree(redisContext *c) { free(c->unix_sock.path); free(c->timeout); free(c->saddr); + if (c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + } + memset(c, 0xff, sizeof(*c)); free(c); } -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -694,10 +720,13 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privdata && c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + c->privdata = NULL; } + redisNetClose(c); + sdsfree(c->obuf); redisReaderFree(c->reader); @@ -718,112 +747,107 @@ int redisReconnect(redisContext *c) { return REDIS_ERR; } +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(options); + if (c == NULL) { + return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE + return NULL; + } + if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->timeout); + } + return c; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); } redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -853,22 +877,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); + return REDIS_ERR; } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -883,21 +900,15 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + int nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { if (nwritten == (signed)sdslen(c->obuf)) { sdsfree(c->obuf); diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 47d7982e..69dc39c5 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -35,7 +35,11 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +#endif #include /* uintXX_t, etc */ #include "sds.h" /* for sds */ @@ -74,6 +78,12 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and @@ -92,6 +102,8 @@ typedef struct redisReply { size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING and REDIS_REPLY_DOUBLE (in additionl to dval). */ + char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null + terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; @@ -111,14 +123,93 @@ void redisFreeSdsCommand(sds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value. if NULL, no timeout is used */ + const struct timeval *timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +struct redisAsyncContext; +struct redisContext; + +typedef struct redisContextFuncs { + void (*free_privdata)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + int (*read)(struct redisContext *, char *, size_t); + int (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ @@ -139,8 +230,12 @@ typedef struct redisContext { /* For non-blocking connect */ struct sockadr *saddr; size_t addrlen; + + /* Additional private data for hiredis addons such as SSL */ + void *privdata; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -151,7 +246,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -167,7 +262,7 @@ int redisReconnect(redisContext *c); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in new file mode 100644 index 00000000..140b040f --- /dev/null +++ b/deps/hiredis/hiredis.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h new file mode 100644 index 00000000..f844f954 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/** + * Secure the connection using SSL. This should be done before any command is + * executed on the connection. + */ +int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, + const char *keypath, const char *servername); + +/** + * Initiate SSL/TLS negotiation on a provided context. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/hiredis_ssl.pc.in b/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 00000000..588a978a --- /dev/null +++ b/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index a4b3abc6..e5f40b0a 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -34,36 +34,64 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +int redisNetRead(redisContext *c, char *buf, size_t bufcap) { + int nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +int redisNetWrite(redisContext *c) { + int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; @@ -79,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -101,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -108,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -119,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); + return REDIS_ERR; + } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -170,7 +207,7 @@ static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -212,12 +249,12 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -230,7 +267,7 @@ static int redisContextWaitReady(redisContext *c, long msec) { } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -277,11 +314,18 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); +#ifdef _WIN32 + DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; + to_ptr = &timeout_msec; + to_sz = sizeof(timeout_msec); +#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } @@ -291,7 +335,8 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -360,7 +405,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -401,16 +446,14 @@ addrretry: } /* For repeat connection */ - if (c->saddr) { - free(c->saddr); - } + free(c->saddr); c->saddr = malloc(p->ai_addrlen); memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; } else if (errno == EINPROGRESS) { if (blocking) { @@ -424,7 +467,7 @@ addrretry: if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { @@ -471,8 +514,9 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; if (redisCreateSocket(c,AF_UNIX) < 0) @@ -499,9 +543,11 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_UNIX; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -516,4 +562,10 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a11594e6..a4393c06 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -37,6 +37,10 @@ #include "hiredis.h" +void redisNetClose(redisContext *c); +int redisNetRead(redisContext *c, char *buf, size_t bufcap); +int redisNetWrite(redisContext *c); + int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index cc0f3cc7..b9853ea9 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -31,10 +31,10 @@ #include "fmacros.h" #include -#include #include #ifndef _MSC_VER #include +#include #endif #include #include @@ -44,6 +44,7 @@ #include "read.h" #include "sds.h" +#include "win32.h" static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -294,9 +295,9 @@ static int processLineItem(redisReader *r) { buf[len] = '\0'; if (strcasecmp(buf,",inf") == 0) { - d = 1.0/0.0; /* Positive infinite. */ + d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -1.0/0.0; /* Nevative infinite. */ + d = -INFINITY; /* Nevative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -379,10 +380,18 @@ static int processBulkItem(redisReader *r) { /* Only continue when the buffer contains the entire bulk item. */ bytelen += len+2; /* include \r\n */ if (r->pos+bytelen <= r->len) { + if ((cur->type == REDIS_REPLY_VERB && len < 4) || + (cur->type == REDIS_REPLY_VERB && s[5] != ':')) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Verbatim string 4 bytes of content type are " + "missing or incorrectly encoded."); + return REDIS_ERR; + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,s+2,len); else - obj = (void*)REDIS_REPLY_STRING; + obj = (void*)(long)cur->type; success = 1; } } @@ -430,7 +439,7 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || elements > INT_MAX) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -523,6 +532,9 @@ static int processItem(redisReader *r) { case '#': cur->type = REDIS_REPLY_BOOL; break; + case '=': + cur->type = REDIS_REPLY_VERB; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -543,6 +555,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_BOOL: return processLineItem(r); case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: return processBulkItem(r); case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: @@ -657,8 +670,11 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index f3d07584..58105312 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -55,12 +56,12 @@ #define REDIS_REPLY_ERROR 6 #define REDIS_REPLY_DOUBLE 7 #define REDIS_REPLY_BOOL 8 -#define REDIS_REPLY_VERB 9 #define REDIS_REPLY_MAP 9 #define REDIS_REPLY_SET 10 #define REDIS_REPLY_ATTR 11 #define REDIS_REPLY_PUSH 12 #define REDIS_REPLY_BIGNUM 13 +#define REDIS_REPLY_VERB 14 #define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ @@ -79,7 +80,7 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 44777b10..6cf75841 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1035,7 +1035,7 @@ sds *sdssplitargs(const char *line, int *argc) { s_free(vector); return NULL; } - + vector = new_vector; vector[*argc] = current; (*argc)++; diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 13be75a9..3f9a9645 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -34,6 +34,9 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +#endif #include #include @@ -132,20 +135,20 @@ static inline void sdssetlen(sds s, size_t newlen) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + SDS_HDR(8,s)->len = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + SDS_HDR(16,s)->len = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + SDS_HDR(32,s)->len = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } @@ -156,21 +159,21 @@ static inline void sdsinclen(sds s, size_t inc) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + SDS_HDR(8,s)->len += (uint8_t)inc; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + SDS_HDR(16,s)->len += (uint16_t)inc; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + SDS_HDR(32,s)->len += (uint32_t)inc; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + SDS_HDR(64,s)->len += (uint64_t)inc; break; } } @@ -200,16 +203,16 @@ static inline void sdssetalloc(sds s, size_t newlen) { /* Nothing to do, this type has no total allocation info. */ break; case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c new file mode 100644 index 00000000..4cc2f414 --- /dev/null +++ b/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h new file mode 100644 index 00000000..56006c16 --- /dev/null +++ b/deps/hiredis/sockcompat.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include + +#ifdef _MSC_VER +typedef signed long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c new file mode 100644 index 00000000..78ab9e43 --- /dev/null +++ b/deps/hiredis/ssl.c @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#include + +#include +#include + +#include "async_private.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +/* The SSL context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSLContext { + /** + * OpenSSL SSL_CTX; It is optional and will not be set when using + * user-supplied SSL. + */ + SSL_CTX *ssl_ctx; + + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSLContext; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +#ifdef HIREDIS_SSL_TRACE +/** + * Callback used for debugging + */ +static void sslLogCallback(const SSL *ssl, int where, int ret) { + const char *retstr = ""; + int should_log = 1; + /* Ignore low-level SSL stuff */ + + if (where & SSL_CB_ALERT) { + should_log = 1; + } + if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { + should_log = 1; + } + if ((where & SSL_CB_EXIT) && ret == 0) { + should_log = 1; + } + + if (!should_log) { + return; + } + + retstr = SSL_alert_type_string(ret); + printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); + + if (where == SSL_CB_HANDSHAKE_DONE) { + printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); + } +} +#endif + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +static pthread_mutex_t *ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static void initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +/** + * SSL Connection initialization. + */ + +static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { + if (c->privdata) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + c->privdata = calloc(1, sizeof(redisSSLContext)); + + c->funcs = &redisContextSSLFuncs; + redisSSLContext *rssl = c->privdata; + + rssl->ssl_ctx = ssl_ctx; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + return REDIS_ERR; +} + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, NULL, ssl); +} + +int redisSecureConnection(redisContext *c, const char *capath, + const char *certpath, const char *keypath, const char *servername) { + + SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; + + /* Initialize global OpenSSL stuff */ + static int isInit = 0; + if (!isInit) { + isInit = 1; + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + } + + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); + goto error; + } + +#ifdef HIREDIS_SSL_TRACE + SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); +#endif + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); + goto error; + } + + if (capath) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); + goto error; + } + } + if (certpath) { + if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); + goto error; + } + } + + ssl = SSL_new(ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + if (servername) { + if (!SSL_set_tlsext_host_name(ssl, servername)) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + goto error; + } + } + + return redisSSLConnect(c, ssl_ctx, ssl); + +error: + if (ssl) SSL_free(ssl); + if (ssl_ctx) SSL_CTX_free(ssl_ctx); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSLContext *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFreeContext(void *privdata){ + redisSSLContext *rsc = privdata; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + if (rsc->ssl_ctx) { + SSL_CTX_free(rsc->ssl_ctx); + rsc->ssl_ctx = NULL; + } + free(rsc); +} + +static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSLContext *rssl = c->privdata; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static int redisSSLWrite(redisContext *c) { + redisSSLContext *rssl = c->privdata; + + size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privdata = redisSSLFreeContext, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 79cff430..8668e185 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -13,12 +13,16 @@ #include #include "hiredis.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -33,6 +37,14 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; }; /* The following lines make up our testing "framework" :) */ @@ -93,11 +105,27 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } +static void do_ssl_handshake(redisContext *c, struct config config) { +#ifdef HIREDIS_TEST_SSL + redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; + (void) config; +#endif +} + static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -121,9 +149,21 @@ static redisContext *do_connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -360,7 +400,8 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); - test("Set error when array > INT_MAX: "); +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); ret = redisReaderGetReply(reader,&reply); @@ -369,7 +410,6 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); -#if LLONG_MAX > SIZE_MAX test("Set error when bulk > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); @@ -434,22 +474,23 @@ static void test_free_null(void) { test_cond(reply == NULL); } +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; struct addrinfo hints = {.ai_family = AF_INET}; struct addrinfo *ai_tmp = NULL; - const char *bad_domain = "idontexist.com"; - int rv = getaddrinfo(bad_domain, "6379", &hints, &ai_tmp); + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); if (rv != 0) { // Address does *not* exist test("Returns error when host cannot be resolved: "); // First see if this domain name *actually* resolves to NXDOMAIN - c = redisConnect("dontexist.com", 6379); + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); test_cond( c->err == REDIS_ERR_OTHER && (strcmp(c->errstr, "Name or service not known") == 0 || - strcmp(c->errstr, "Can't resolve: sadkfjaskfjsa.com") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || @@ -574,7 +615,8 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); + redisAppendFormattedCommand(c, cmd, strlen(cmd)); + s = c->funcs->write(c); tv.tv_sec = 0; tv.tv_usec = 10000; redisSetTimeout(c, tv); @@ -583,7 +625,7 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -591,7 +633,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -894,6 +936,23 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -922,6 +981,20 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; + + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + } +#endif + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); cfg.type = CONN_FD; diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh new file mode 100755 index 00000000..2cab9e6f --- /dev/null +++ b/deps/hiredis/test.sh @@ -0,0 +1,70 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SSL_TEST_ARGS= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ From 7a73b7f168eb01c4cf43ab0e0348f0f2d9922cb5 Mon Sep 17 00:00:00 2001 From: valentino Date: Sat, 21 Sep 2019 20:58:57 +0300 Subject: [PATCH 053/116] DISCARD should not fail during OOM discard command should not fail during OOM, otherwise client MULTI state will not be cleared. --- src/server.c | 2 +- tests/unit/multi.tcl | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 7882b0d9..3737c335 100644 --- a/src/server.c +++ b/src/server.c @@ -3411,7 +3411,7 @@ int processCommand(client *c) { * is in MULTI/EXEC context? Error. */ if (out_of_memory && (c->cmd->flags & CMD_DENYOOM || - (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand))) { + (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand && c->cmd->proc != discardCommand))) { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 6655bf62..9fcef71d 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -306,4 +306,18 @@ start_server {tags {"multi"}} { } close_replication_stream $repl } + + test {DISCARD should not fail during OOM} { + set rd [redis_deferring_client] + $rd config set maxmemory 1 + assert {[$rd read] eq {OK}} + r multi + catch {r set x 1} e + assert_match {OOM*} $e + r discard + $rd config set maxmemory 0 + assert {[$rd read] eq {OK}} + $rd close + r ping + } {PONG} } From 264708427b87b5c45f8ce5ee9d95bb99ce937732 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:39:42 +0200 Subject: [PATCH 054/116] RESP3: implementation of verbatim output with TTY target. --- src/redis-cli.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index db53fc7d..d08dfeb5 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -819,10 +819,17 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) { out = sdscatprintf(out,"(double) %s\n",r->str); break; case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: /* If you are producing output for the standard output we want - * a more interesting output with quoted characters and so forth */ - out = sdscatrepr(out,r->str,r->len); - out = sdscat(out,"\n"); + * a more interesting output with quoted characters and so forth, + * unless it's a verbatim string type. */ + if (r->type == REDIS_REPLY_STRING) { + out = sdscatrepr(out,r->str,r->len); + out = sdscat(out,"\n"); + } else { + out = sdscatlen(out,r->str,r->len); + out = sdscat(out,"\n"); + } break; case REDIS_REPLY_NIL: out = sdscat(out,"(nil)\n"); From 5e399d5d335b2d4464621cbf6a3e2568dd779178 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:41:04 +0200 Subject: [PATCH 055/116] hiredis udpated (RESP3 WIP). --- deps/hiredis/hiredis.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 282595bd..abd94c01 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -128,15 +128,16 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len /* Copy string value */ if (task->type == REDIS_REPLY_VERB) { - buf = malloc(len+4+1); /* Skip 4 bytes of verbatim type header. */ + buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ if (buf == NULL) { freeReplyObject(r); return NULL; } - memcpy(r->vtype,buf,3); + memcpy(r->vtype,str,3); r->vtype[3] = '\0'; - memcpy(buf+4,str,len-4); + memcpy(buf,str+4,len-4); buf[len-4] = '\0'; + r->len = len-4; } else { buf = malloc(len+1); if (buf == NULL) { @@ -145,9 +146,9 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len } memcpy(buf,str,len); buf[len] = '\0'; + r->len = len; } r->str = buf; - r->len = len; if (task->parent) { parent = task->parent->obj; From bb7546c9134b21b32e2fc9a6850407179bad8a64 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:42:57 +0200 Subject: [PATCH 056/116] RESP3: varbatim handling for other redis-cli outputs. --- src/redis-cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index d08dfeb5..9a5f8172 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -968,6 +968,7 @@ static sds cliFormatReplyRaw(redisReply *r) { break; case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: if (r->type == REDIS_REPLY_STATUS && config.eval_ldb) { /* The Lua debugger replies with arrays of simple (status) * strings. We colorize the output for more fun if this @@ -1021,6 +1022,7 @@ static sds cliFormatReplyCSV(redisReply *r) { out = sdscatprintf(out,"%lld",r->integer); break; case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: out = sdscatrepr(out,r->str,r->len); break; case REDIS_REPLY_NIL: From 8ea185ea30853cd067a44d18895029be76591edb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:47:36 +0200 Subject: [PATCH 057/116] redis-cli: AUTH can now have 3 arguments as well. --- src/redis-cli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 9a5f8172..38a0b0b7 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1222,7 +1222,8 @@ static int cliSendCommand(int argc, char **argv, long repeat) { if (!strcasecmp(command,"select") && argc == 2 && config.last_cmd_type != REDIS_REPLY_ERROR) { config.dbnum = atoi(argv[1]); cliRefreshPrompt(); - } else if (!strcasecmp(command,"auth") && argc == 2) { + } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3)) + { cliSelect(); } } From cc108057ffda25a5421e2acc23dc63fe3c7b13d8 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:55:05 +0200 Subject: [PATCH 058/116] redis-cli: ability to start a session in RESP3 mode. --- src/redis-cli.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 38a0b0b7..47bd11a8 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -230,6 +230,7 @@ static struct config { int verbose; clusterManagerCommand cluster_manager_command; int no_auth_warning; + int resp3; } config; /* User preferences. */ @@ -751,6 +752,21 @@ static int cliSelect(void) { return REDIS_ERR; } +/* Select RESP3 mode if redis-cli was started with the -3 option. */ +static int cliSwitchProto(void) { + redisReply *reply; + if (config.resp3 == 0) return REDIS_OK; + + reply = redisCommand(context,"HELLO 3"); + if (reply != NULL) { + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR; + freeReplyObject(reply); + return result; + } + return REDIS_ERR; +} + /* Connect to the server. It is possible to pass certain flags to the function: * CC_FORCE: The connection is performed even if there is already * a connected socket. @@ -788,11 +804,13 @@ static int cliConnect(int flags) { * errors. */ anetKeepAlive(NULL, context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); - /* Do AUTH and select the right DB. */ + /* Do AUTH, select the right DB, switch to RESP3 if needed. */ if (cliAuth() != REDIS_OK) return REDIS_ERR; if (cliSelect() != REDIS_OK) return REDIS_ERR; + if (cliSwitchProto() != REDIS_OK) + return REDIS_ERR; } return REDIS_OK; } @@ -1449,6 +1467,8 @@ static int parseOptions(int argc, char **argv) { printf("redis-cli %s\n", version); sdsfree(version); exit(0); + } else if (!strcmp(argv[i],"-3")) { + config.resp3 = 1; } else if (CLUSTER_MANAGER_MODE() && argv[i][0] != '-') { if (config.cluster_manager_command.argc == 0) { int j = i + 1; @@ -1529,6 +1549,7 @@ static void usage(void) { " -i When -r is used, waits seconds per command.\n" " It is possible to specify sub-second times like -i 0.1.\n" " -n Database number.\n" +" -3 Start session in RESP3 protocol mode.\n" " -x Read last argument from STDIN.\n" " -d Multi-bulk delimiter in for raw formatting (default: \\n).\n" " -c Enable cluster mode (follow -ASK and -MOVED redirections).\n" @@ -1543,7 +1564,9 @@ static void usage(void) { " --csv is specified, or if you redirect the output to a non\n" " TTY, it samples the latency for 1 second (you can use\n" " -i to change the interval), then produces a single output\n" -" and exits.\n" +" and exits.\n",version); + + fprintf(stderr, " --latency-history Like --latency but tracking latency changes over time.\n" " Default time interval is 15 sec. Change it using -i.\n" " --latency-dist Shows latency as a spectrum, requires xterm 256 colors.\n" @@ -1578,7 +1601,7 @@ static void usage(void) { " --help Output this help and exit.\n" " --version Output version and exit.\n" "\n", - version, REDIS_CLI_DEFAULT_PIPE_TIMEOUT); + REDIS_CLI_DEFAULT_PIPE_TIMEOUT); /* Using another fprintf call to avoid -Woverlength-strings compile warning */ fprintf(stderr, "Cluster Manager Commands:\n" From b21dd082c329352362f309743ec0b27abdf68595 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 19:36:06 +0200 Subject: [PATCH 059/116] redis-cli: CSV and RAW target for more RESP3 types. --- src/redis-cli.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 47bd11a8..7374526e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1006,9 +1006,15 @@ static sds cliFormatReplyRaw(redisReply *r) { out = sdscatlen(out,r->str,r->len); } break; + case REDIS_REPLY_BOOL: + out = sdscat(out,r->integer ? "(true)" : "(false)"); + break; case REDIS_REPLY_INTEGER: out = sdscatprintf(out,"%lld",r->integer); break; + case REDIS_REPLY_DOUBLE: + out = sdscatprintf(out,"%s",r->str); + break; case REDIS_REPLY_ARRAY: for (i = 0; i < r->elements; i++) { if (i > 0) out = sdscat(out,config.mb_delim); @@ -1017,6 +1023,19 @@ static sds cliFormatReplyRaw(redisReply *r) { sdsfree(tmp); } break; + case REDIS_REPLY_MAP: + for (i = 0; i < r->elements; i += 2) { + if (i > 0) out = sdscat(out,config.mb_delim); + tmp = cliFormatReplyRaw(r->element[i]); + out = sdscatlen(out,tmp,sdslen(tmp)); + sdsfree(tmp); + + out = sdscatlen(out," ",1); + tmp = cliFormatReplyRaw(r->element[i+1]); + out = sdscatlen(out,tmp,sdslen(tmp)); + sdsfree(tmp); + } + break; default: fprintf(stderr,"Unknown reply type: %d\n", r->type); exit(1); @@ -1039,14 +1058,21 @@ static sds cliFormatReplyCSV(redisReply *r) { case REDIS_REPLY_INTEGER: out = sdscatprintf(out,"%lld",r->integer); break; + case REDIS_REPLY_DOUBLE: + out = sdscatprintf(out,"%s",r->str); + break; case REDIS_REPLY_STRING: case REDIS_REPLY_VERB: out = sdscatrepr(out,r->str,r->len); break; case REDIS_REPLY_NIL: - out = sdscat(out,"NIL"); + out = sdscat(out,"NULL"); + break; + case REDIS_REPLY_BOOL: + out = sdscat(out,r->integer ? "true" : "false"); break; case REDIS_REPLY_ARRAY: + case REDIS_REPLY_MAP: /* CSV has no map type, just output flat list. */ for (i = 0; i < r->elements; i++) { sds tmp = cliFormatReplyCSV(r->element[i]); out = sdscatlen(out,tmp,sdslen(tmp)); From eda703ab284f1ce491fb2c376fd08fe8aa956c62 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 19:57:13 +0200 Subject: [PATCH 060/116] redis-cli: support for ACL style user/pass AUTH. --- src/redis-cli.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7374526e..c183155c 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -218,6 +218,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; char prompt[128]; @@ -729,8 +730,13 @@ static int cliAuth(void) { redisReply *reply; if (config.auth == NULL) return REDIS_OK; - reply = redisCommand(context,"AUTH %s",config.auth); + if (config.user == NULL) + reply = redisCommand(context,"AUTH %s",config.auth); + else + reply = redisCommand(context,"AUTH %s %s",config.user,config.auth); if (reply != NULL) { + if (reply->type == REDIS_REPLY_ERROR) + fprintf(stderr,"Warning: AUTH failed\n"); freeReplyObject(reply); return REDIS_OK; } @@ -1350,8 +1356,12 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; - } else if (!strcmp(argv[i],"-a") && !lastarg) { + } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) + && !lastarg) + { config.auth = argv[++i]; + } else if (!strcmp(argv[i],"--user") && !lastarg) { + config.user = argv[++i]; } else if (!strcmp(argv[i],"-u") && !lastarg) { parseRedisUri(argv[++i]); } else if (!strcmp(argv[i],"--raw")) { @@ -1570,6 +1580,8 @@ static void usage(void) { " You can also use the " REDIS_CLI_AUTH_ENV " environment\n" " variable to pass this password more safely\n" " (if both are used, this argument takes predecence).\n" +" -user Used to send ACL style 'AUTH username pass'. Needs -a.\n" +" -pass Alias of -a for consistency with the new --user option.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -2409,7 +2421,12 @@ static int clusterManagerNodeConnect(clusterManagerNode *node) { * errors. */ anetKeepAlive(NULL, node->context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); if (config.auth) { - redisReply *reply = redisCommand(node->context,"AUTH %s",config.auth); + redisReply *reply; + if (config.user == NULL) + reply = redisCommand(node->context,"AUTH %s", config.auth); + else + reply = redisCommand(node->context,"AUTH %s %s", + config.user,config.auth); int ok = clusterManagerCheckRedisReply(node, reply, NULL); if (reply != NULL) freeReplyObject(reply); if (!ok) return 0; @@ -7737,6 +7754,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.user = NULL; config.eval = NULL; config.eval_ldb = 0; config.eval_ldb_end = 0; From 733280d9cb1122dd15440429137610a4237869d4 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 23 Sep 2019 23:45:31 +0100 Subject: [PATCH 061/116] [fix] un-refactor the code. [perf] replyWithStatus now makes usage of addReplyProto --- src/module.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/module.c b/src/module.c index ad7e6864..067a6992 100644 --- a/src/module.c +++ b/src/module.c @@ -1120,6 +1120,19 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { return REDISMODULE_OK; } +/* Reply with an error or simple string (status message). Used to implement + * ReplyWithSimpleString() and ReplyWithError(). + * The function always returns REDISMODULE_OK. */ +int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(msg); + addReplyProto(c,"-",1); + addReplyProto(c,msg,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; +} + /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including @@ -1135,13 +1148,7 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { * The function always returns REDISMODULE_OK. */ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(err); - addReplyProto(c,"-",1); - addReplyProto(c,err,len); - addReplyProto(c,"\r\n",2); - return REDISMODULE_OK; + return replyWithStatus(ctx,err,"-"); } /* Reply with a simple string (+... \r\n in RESP protocol). This replies @@ -1150,13 +1157,7 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(msg); - addReplyProto(c,"+",1); - addReplyProto(c,msg,len); - addReplyProto(c,"\r\n",2); - return REDISMODULE_OK; + return replyWithStatus(ctx,msg,"+"); } /* Reply with an array type of 'len' elements. However 'len' other calls From 0a4d2bbd9c07bdaa5979941b5483cebb7419f7a0 Mon Sep 17 00:00:00 2001 From: "Mike A. Owens" Date: Mon, 23 Sep 2019 19:24:09 -0400 Subject: [PATCH 062/116] Seed SipHash with 128-bit key SipHash expects a 128-bit key, and we were indeed generating 128-bits, but restricting them to hex characters 0-9a-f, effectively giving us only 4 bits-per-byte of key material, and 64 bits overall. Now, we skip the hex conversion and supply 128 bits of unfiltered random data. --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 7882b0d9..fc9809b1 100644 --- a/src/server.c +++ b/src/server.c @@ -4787,9 +4787,9 @@ int main(int argc, char **argv) { srand(time(NULL)^getpid()); gettimeofday(&tv,NULL); - char hashseed[16]; - getRandomHexChars(hashseed,sizeof(hashseed)); - dictSetHashFunctionSeed((uint8_t*)hashseed); + uint8_t hashseed[16]; + getRandomBytes(hashseed,sizeof(hashseed)); + dictSetHashFunctionSeed(hashseed); server.sentinel_mode = checkForSentinelMode(argc,argv); initServerConfig(); ACLInit(); /* The ACL subsystem must be initialized ASAP because the From 8a531cedb01215e367d88b600bb9178ebb9eb4b2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Sep 2019 17:45:05 +0200 Subject: [PATCH 063/116] ACL: fix ##6408, default user state affecting all the connections. --- src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 7882b0d9..2bdfd164 100644 --- a/src/server.c +++ b/src/server.c @@ -3341,9 +3341,10 @@ int processCommand(client *c) { /* Check if the user is authenticated. This check is skipped in case * the default user is flagged as "nopass" and is active. */ - int auth_required = !(DefaultUser->flags & USER_FLAG_NOPASS) && + int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || + DefaultUser->flags & USER_FLAG_DISABLED) && !c->authenticated; - if (auth_required || DefaultUser->flags & USER_FLAG_DISABLED) { + if (auth_required) { /* AUTH and HELLO are valid even in non authenticated state. */ if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) { flagTransaction(c); From b3d6cb268a14836e8e0d9a209e2f3b7495b1f22d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Sep 2019 18:08:11 +0200 Subject: [PATCH 064/116] Modify #6401 changes to fit 80 cols. --- src/server.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 42dc8f1e..51106abe 100644 --- a/src/server.c +++ b/src/server.c @@ -3412,7 +3412,10 @@ int processCommand(client *c) { * is in MULTI/EXEC context? Error. */ if (out_of_memory && (c->cmd->flags & CMD_DENYOOM || - (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand && c->cmd->proc != discardCommand))) { + (c->flags & CLIENT_MULTI && + c->cmd->proc != execCommand && + c->cmd->proc != discardCommand))) + { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; From af15b285faa2a9cece8f5a76e05221836b6abae5 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 25 Sep 2019 17:28:42 +0100 Subject: [PATCH 065/116] [fix] fixed the un-refactor bug. --- src/module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 067a6992..d02e2708 100644 --- a/src/module.c +++ b/src/module.c @@ -1126,9 +1126,10 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(msg); - addReplyProto(c,"-",1); - addReplyProto(c,msg,len); + const size_t msgLen = strlen(msg); + const size_t prefixLen = strlen(prefix); + addReplyProto(c,prefix,prefixLen); + addReplyProto(c,msg,msgLen); addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } From b7b23bdfb864efb595868100a8857532cff0094f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Sep 2019 12:18:39 +0200 Subject: [PATCH 066/116] INFO: more info about loaded modules. Related to #6024. --- src/module.c | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 71fdd6b7..b2bddd31 100644 --- a/src/module.c +++ b/src/module.c @@ -52,7 +52,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ - int options; /* Moduile options and capabilities. */ + int options; /* Module options and capabilities. */ }; typedef struct RedisModule RedisModule; @@ -5363,9 +5363,36 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper for genModulesInfoString(): given a list of modules, return + * am SDS string in the form "[modulename|modulename2|...]" */ +sds genModulesInfoStringRenderModulesList(list *l) { + listIter li; + listNode *ln; + listRewind(l,&li); + sds output = sdsnew("["); + while((ln = listNext(&li))) { + RedisModule *module = ln->value; + output = sdscat(output,module->name); + } + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + +/* Helper for genModulesInfoString(): render module options as an SDS string. */ +sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { + sds output = sdsnew("["); + if (module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) + output = sdscat(output,"handle-io-errors|"); + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + + /* Helper function for the INFO command: adds loaded modules as to info's * output. - * + * * After the call, the passed sds info string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ sds genModulesInfoString(sds info) { @@ -5376,7 +5403,17 @@ sds genModulesInfoString(sds info) { sds name = dictGetKey(de); struct RedisModule *module = dictGetVal(de); - info = sdscatprintf(info, "module:name=%s,ver=%d\r\n", name, module->ver); + sds usedby = genModulesInfoStringRenderModulesList(module->usedby); + sds using = genModulesInfoStringRenderModulesList(module->using); + sds options = genModulesInfoStringRenderModuleOptions(module); + info = sdscatprintf(info, + "module:name=%s,ver=%d,api=%d,filters=%d," + "usedby=%s,using=%s,options=%s\r\n", + name, module->ver, module->apiver, + (int)listLength(module->filters), usedby, using, options); + sdsfree(usedby); + sdsfree(using); + sdsfree(options); } dictReleaseIterator(di); return info; From 83e87bac762c84f2a2e9ee6922d038ee09ae9cd4 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 26 Sep 2019 15:16:34 +0300 Subject: [PATCH 067/116] Fix lastbgsave_status, when new child signal handler get intended kill And add a test for that. --- src/server.c | 9 ++++++++- tests/integration/rdb.tcl | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 31418e01..c0e59c86 100644 --- a/src/server.c +++ b/src/server.c @@ -1910,6 +1910,13 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); + /* sigKillChildHandler catches the signal and calls exit(), but we + * must make sure not to flag lastbgsave_status, etc incorrectly. */ + if (exitcode == SIGUSR1) { + bysignal = SIGUSR1; + exitcode = 1; + } + if (pid == -1) { serverLog(LL_WARNING,"wait3() returned an error: %s. " "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d", @@ -4578,7 +4585,7 @@ static void sigKillChildHandler(int sig) { UNUSED(sig); /* this handler is needed to resolve a valgrind warning */ serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); - exitFromChild(1); + exitFromChild(SIGUSR1); } void setupChildSignalHandlers(void) { diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 58a098ed..b364291e 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -115,3 +115,17 @@ start_server_and_kill_it [list "dir" $server_path] { } } } + +start_server {} { + test {Test FLUSHALL aborts bgsave} { + r config set rdb-key-save-delay 1000 + r debug populate 1000 + r bgsave + assert_equal [s rdb_bgsave_in_progress] 1 + r flushall + after 200 + assert_equal [s rdb_bgsave_in_progress] 0 + # make sure the server is still writable + r set x xx + } +} \ No newline at end of file From fddc4757c8c585d384889c1c7efba1ccf2121e6b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Sep 2019 16:14:21 +0200 Subject: [PATCH 068/116] BGREWRITEAOF: improve the generic error message. --- src/aof.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 7237cdfb..91d0d1a7 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1630,7 +1630,8 @@ void bgrewriteaofCommand(client *c) { } else if (rewriteAppendOnlyFileBackground() == C_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { - addReply(c,shared.err); + addReplyError(c,"Can't execute an AOF background rewriting. " + "Please check the server logs for more information."); } } From 6578119a237426a4092c2fc567bd6f4029af2e61 Mon Sep 17 00:00:00 2001 From: nikhilajayk Date: Thu, 26 Sep 2019 21:51:49 +0530 Subject: [PATCH 069/116] Added cluster host and protected mode variables --- utils/create-cluster/README | 2 +- utils/create-cluster/create-cluster | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/create-cluster/README b/utils/create-cluster/README index e682f6dc..37a3080d 100644 --- a/utils/create-cluster/README +++ b/utils/create-cluster/README @@ -16,7 +16,7 @@ To create a cluster, follow these steps: number of instances you want to create. 2. Use "./create-cluster start" in order to run the instances. 3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that -an actual Redis cluster will be created. +an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP) 4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory. In order to stop a cluster: diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 468f924a..9ffd462a 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -1,10 +1,12 @@ #!/bin/bash # Settings +CLUSTER_HOST=127.0.0.1 PORT=30000 TIMEOUT=2000 NODES=6 REPLICAS=1 +PROTECTED_MODE=yes # You may want to put the above config parameters into config.sh in order to # override the defaults without modifying this script. @@ -22,7 +24,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - ../../src/redis-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes + ../../src/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes done exit 0 fi @@ -32,7 +34,7 @@ then HOSTS="" while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) - HOSTS="$HOSTS 127.0.0.1:$PORT" + HOSTS="$HOSTS $CLUSTER_HOST:$PORT" done ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS exit 0 From ae3ef964c1f75999499c2d35f7485aea1c0a70b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:39:40 +0200 Subject: [PATCH 070/116] Modules fork: improve SIGUSR1 handling, fix include. We can't expect SIGUSR1 to have any specific value range, so let's define an exit code that we can handle in a special way. This also fixes an #include that is not standard. --- src/module.c | 2 +- src/server.c | 14 ++++++++++---- src/server.h | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 854989e7..d3b37a3d 100644 --- a/src/module.c +++ b/src/module.c @@ -31,7 +31,7 @@ #include "cluster.h" #include "rdb.h" #include -#include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" diff --git a/src/server.c b/src/server.c index f38ed789..046694e1 100644 --- a/src/server.c +++ b/src/server.c @@ -1913,8 +1913,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); /* sigKillChildHandler catches the signal and calls exit(), but we - * must make sure not to flag lastbgsave_status, etc incorrectly. */ - if (exitcode == SIGUSR1) { + * must make sure not to flag lastbgsave_status, etc incorrectly. + * We could directly terminate the child process via SIGUSR1 + * without handling it, but in this case Valgrind will log an + * annoying error. */ + if (exitcode == SERVER_CHILD_NOERROR_RETVAL) { bysignal = SIGUSR1; exitcode = 1; } @@ -4618,11 +4621,14 @@ void setupSignalHandlers(void) { return; } +/* This is the signal handler for children process. It is currently useful + * in order to track the SIGUSR1, that we send to a child in order to terminate + * it in a clean way, without the parent detecting an error and stop + * accepting writes because of a write error condition. */ static void sigKillChildHandler(int sig) { UNUSED(sig); - /* this handler is needed to resolve a valgrind warning */ serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); - exitFromChild(SIGUSR1); + exitFromChild(SERVER_CHILD_NOERROR_RETVAL); } void setupChildSignalHandlers(void) { diff --git a/src/server.h b/src/server.h index d132cf09..c701d6d2 100644 --- a/src/server.h +++ b/src/server.h @@ -179,6 +179,14 @@ typedef long long mstime_t; /* millisecond time type. */ #define ACTIVE_EXPIRE_CYCLE_SLOW 0 #define ACTIVE_EXPIRE_CYCLE_FAST 1 +/* Children process will exit with this status code to signal that the + * process terminated without an error: this is useful in order to kill + * a saving child (RDB or AOF one), without triggering in the parent the + * write protection that is normally turned on on write errors. + * Usually children that are terminated with SIGUSR1 will exit with this + * special code. */ +#define SERVER_CHILD_NOERROR_RETVAL 255 + /* Instantaneous metrics tracking. */ #define STATS_METRIC_SAMPLES 16 /* Number of samples per metric. */ #define STATS_METRIC_COMMAND 0 /* Number of commands executed. */ From beb1356b5bdeb38eecb5c0bef509d3d4b3f2d3c0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:59:37 +0200 Subject: [PATCH 071/116] Improve error message in BGSAVE. --- src/rdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index d9164b21..e1cedf6f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2590,9 +2590,9 @@ void bgsaveCommand(client *c) { addReplyStatus(c,"Background saving scheduled"); } else { addReplyError(c, - "Another BG operation is in progress: can't BGSAVE right now. " - "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " - "possible."); + "Another child process is active (AOF?): can't BGSAVE right now. " + "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " + "possible."); } } else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) { addReplyStatus(c,"Background saving started"); From 82845f8d045a1fdbac17cf1959ad51906a8840ed Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:59:58 +0200 Subject: [PATCH 072/116] TerminateModuleForkChild(): use wait4 for safety. In theory currently there is only one active child, but the API may change or for bugs in the implementation we may have several (it was like that for years because of a bug). Better to wait for a specific pid and avoid consuing other pending children information. --- src/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d3b37a3d..cee7a85a 100644 --- a/src/module.c +++ b/src/module.c @@ -5193,7 +5193,8 @@ void TerminateModuleForkChild(int wait) { serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { - while(wait3(&statloc,0,NULL) != server.module_child_pid); + while(wait4(server.module_child_pid,&statloc,0,NULL) != + server.module_child_pid); } /* Reset the buffer accumulating changes while the child saves. */ server.module_child_pid = -1; From de1f82aa3302461c21f6cfcb8e8972e865b02174 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:03:09 +0200 Subject: [PATCH 073/116] Function renamed hasForkChild() -> hasActiveChildProcess(). --- src/aof.c | 10 +++++----- src/db.c | 2 +- src/defrag.c | 2 +- src/module.c | 2 +- src/rdb.c | 6 +++--- src/replication.c | 4 ++-- src/server.c | 14 +++++++------- src/server.h | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/aof.c b/src/aof.c index 8bc6c543..4e6af7c1 100644 --- a/src/aof.c +++ b/src/aof.c @@ -264,7 +264,7 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (hasForkChild() && server.aof_child_pid == -1) { + if (hasActiveChildProcess() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { @@ -395,7 +395,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -491,7 +491,7 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && hasForkChild()) + if (server.aof_no_fsync_on_rewrite && hasActiveChildProcess()) return; /* Perform the fsync if needed. */ @@ -1563,7 +1563,7 @@ void aofClosePipes(void) { int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); if ((childpid = redisFork()) == 0) { @@ -1607,7 +1607,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { diff --git a/src/db.c b/src/db.c index a46e0251..afedc6ae 100644 --- a/src/db.c +++ b/src/db.c @@ -60,7 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (!hasForkChild() && !(flags & LOOKUP_NOTOUCH)){ + if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { diff --git a/src/defrag.c b/src/defrag.c index 93c6a461..50f6b41f 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (hasForkChild()) + if (hasActiveChildProcess()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/module.c b/src/module.c index cee7a85a..4e6ad6cf 100644 --- a/src/module.c +++ b/src/module.c @@ -5157,7 +5157,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { pid_t childpid; - if (hasForkChild()) { + if (hasActiveChildProcess()) { return -1; } diff --git a/src/rdb.c b/src/rdb.c index e1cedf6f..e430bcd5 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1336,7 +1336,7 @@ werr: int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { pid_t childpid; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); @@ -2417,7 +2417,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { pid_t childpid; int pipefds[2]; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; /* Before to fork, create a pipe that will be used in order to * send back to the parent the IDs of the slaves that successfully @@ -2584,7 +2584,7 @@ void bgsaveCommand(client *c) { if (server.rdb_child_pid != -1) { addReplyError(c,"Background save already in progress"); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { if (schedule) { server.rdb_bgsave_scheduled = 1; addReplyStatus(c,"Background saving scheduled"); diff --git a/src/replication.c b/src/replication.c index 8039e06a..76090a9e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -751,7 +751,7 @@ void syncCommand(client *c) { /* Target is disk (or the slave is not capable of supporting * diskless replication) and we don't have a BGSAVE in progress, * let's start one. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { startBgsaveForReplication(c->slave_capa); } else { serverLog(LL_NOTICE, @@ -2930,7 +2930,7 @@ void replicationCron(void) { * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other slaves * have the time to arrive before we start streaming. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { time_t idle, max_idle = 0; int slaves_waiting = 0; int mincapa = -1; diff --git a/src/server.c b/src/server.c index 046694e1..b2ab653a 100644 --- a/src/server.c +++ b/src/server.c @@ -1449,13 +1449,13 @@ int incrementallyRehash(int dbid) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (!hasForkChild()) + if (!hasActiveChildProcess()) dictEnableResize(); else dictDisableResize(); } -int hasForkChild() { +int hasActiveChildProcess() { return server.rdb_child_pid != -1 || server.aof_child_pid != -1 || server.module_child_pid != -1; @@ -1697,7 +1697,7 @@ void databasesCron(void) { /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad * as will cause a lot of copy-on-write of memory pages. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ @@ -1894,14 +1894,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (!hasForkChild() && + if (!hasActiveChildProcess() && server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (hasForkChild() || ldbPendingChildren()) + if (hasActiveChildProcess() || ldbPendingChildren()) { int statloc; pid_t pid; @@ -1975,7 +1975,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Trigger an AOF rewrite if needed. */ if (server.aof_state == AOF_ON && - !hasForkChild() && + !hasActiveChildProcess() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { @@ -2033,7 +2033,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (!hasForkChild() && + if (!hasActiveChildProcess() && server.rdb_bgsave_scheduled && (server.unixtime-server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) diff --git a/src/server.h b/src/server.h index c701d6d2..2ab3f2b4 100644 --- a/src/server.h +++ b/src/server.h @@ -1818,7 +1818,7 @@ void receiveChildInfo(void); /* Fork helpers */ int redisFork(); -int hasForkChild(); +int hasActiveChildProcess(); void sendChildCOWInfo(int ptype, char *pname); /* acl.c -- Authentication related prototypes. */ From 721d3c9e0cb86117c9ebd61552facc88a286d47c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:17:47 +0200 Subject: [PATCH 074/116] TerminateModuleForkChild(): move safety checks there. We don't want that the API could be used directly in an unsafe way, without checking if there is an active child. Now the safety checks are moved directly in the function performing the operations. --- src/module.c | 35 ++++++++++++++++++----------------- src/server.c | 2 +- src/server.h | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/module.c b/src/module.c index 4e6ad6cf..27d5eb86 100644 --- a/src/module.c +++ b/src/module.c @@ -5031,7 +5031,7 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi ln = listSearchKey(moduleCommandFilters,filter); if (!ln) return REDISMODULE_ERR; listDelNode(moduleCommandFilters,ln); - + ln = listSearchKey(ctx->module->filters,filter); if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); @@ -5154,8 +5154,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) * Return: -1 on failure, on success the parent process will get a positive PID * of the child, and the child process will get 0. */ -int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) -{ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { pid_t childpid; if (hasActiveChildProcess()) { return -1; @@ -5181,14 +5180,20 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) /* Call from the child process when you want to terminate it. * retcode will be provided to the done handler executed on the parent process. */ -int RM_ExitFromChild(int retcode) -{ +int RM_ExitFromChild(int retcode) { sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); exitFromChild(retcode); return REDISMODULE_OK; } -void TerminateModuleForkChild(int wait) { +/* Kill the active module forked child, if there is one active and the + * pid matches, and returns C_OK. Otherwise if there is no active module + * child or the pid does not match, return C_ERR without doing anything. */ +void TerminateModuleForkChild(int child_pid, int wait) { + /* Module child should be active and pid should match. */ + if (server.module_child_pid == -1 || + server.module_child_pid != child_pid) return C_ERR; + int statloc; serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); @@ -5202,24 +5207,20 @@ void TerminateModuleForkChild(int wait) { moduleForkInfo.done_handler_user_data = NULL; closeChildInfoPipe(); updateDictResizePolicy(); + return C_OK; } /* Can be used to kill the forked child process from the parent process. * child_pid whould be the return value of RedisModule_Fork. */ -int RM_KillForkChild(int child_pid) -{ - /* No module child? return. */ - if (server.module_child_pid == -1) return REDISMODULE_ERR; - /* Make sure the module knows the pid it wants to kill (not trying to - * randomly kill other module's forks) */ - if (server.module_child_pid != child_pid) return REDISMODULE_ERR; +int RM_KillForkChild(int child_pid) { /* Kill module child, wait for child exit. */ - TerminateModuleForkChild(1); - return REDISMODULE_OK; + if (TerminateModuleForkChild(child_pid,1) == C_OK) + return REDISMODULE_OK; + else + return REDISMODULE_ERR; } -void ModuleForkDoneHandler(int exitcode, int bysignal) -{ +void ModuleForkDoneHandler(int exitcode, int bysignal) { serverLog(LL_NOTICE, "Module fork exited pid: %d, retcode: %d, bysignal: %d", server.module_child_pid, exitcode, bysignal); diff --git a/src/server.c b/src/server.c index b2ab653a..a2af7824 100644 --- a/src/server.c +++ b/src/server.c @@ -3589,7 +3589,7 @@ int prepareForShutdown(int flags) { /* Kill module child if there is one. */ if (server.module_child_pid != -1) { serverLog(LL_WARNING,"There is a module fork child. Killing it!"); - TerminateModuleForkChild(0); + TerminateModuleForkChild(server.module_child_pid,0); } if (server.aof_state != AOF_OFF) { diff --git a/src/server.h b/src/server.h index 2ab3f2b4..52abef4b 100644 --- a/src/server.h +++ b/src/server.h @@ -1552,7 +1552,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); -void TerminateModuleForkChild(int wait); +void TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); From 0a07f8ffeed8fc0a422287d847cf0c9de6c88393 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:23:07 +0200 Subject: [PATCH 075/116] TerminateModuleForkChild(): fix function prototype. --- src/module.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 27d5eb86..21747870 100644 --- a/src/module.c +++ b/src/module.c @@ -5189,7 +5189,7 @@ int RM_ExitFromChild(int retcode) { /* Kill the active module forked child, if there is one active and the * pid matches, and returns C_OK. Otherwise if there is no active module * child or the pid does not match, return C_ERR without doing anything. */ -void TerminateModuleForkChild(int child_pid, int wait) { +int TerminateModuleForkChild(int child_pid, int wait) { /* Module child should be active and pid should match. */ if (server.module_child_pid == -1 || server.module_child_pid != child_pid) return C_ERR; diff --git a/src/server.h b/src/server.h index 52abef4b..91c7219a 100644 --- a/src/server.h +++ b/src/server.h @@ -1552,7 +1552,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); -void TerminateModuleForkChild(int child_pid, int wait); +int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); From b394817754ade6151f40b64115f9fbe03a9ca09b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:32:52 +0200 Subject: [PATCH 076/116] Fix memory leak in RM_UnregisterCommandFilter(). --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 21747870..98d4db97 100644 --- a/src/module.c +++ b/src/module.c @@ -5036,6 +5036,8 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); + zfree(filter); + return REDISMODULE_OK; } From 8b5848a4f50f2fb0d9651ea91b74b22e83e6a452 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:33:21 +0200 Subject: [PATCH 077/116] Fix memory leak in moduleLoadFromQueue(). --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 98d4db97..351ffcc4 100644 --- a/src/module.c +++ b/src/module.c @@ -5334,6 +5334,8 @@ void moduleLoadFromQueue(void) { void moduleFreeModuleStructure(struct RedisModule *module) { listRelease(module->types); listRelease(module->filters); + listRelease(module->usedby); + listRelease(module->using); sdsfree(module->name); zfree(module); } From 916c4e5d864f55949e0202d2d9a073e19cf49b72 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:42:38 +0200 Subject: [PATCH 078/116] moduleRDBLoadError(): io->ctx may be NULL. The correct way to access the module about a given IO context is to deference io->type->module, since io->ctx is only populated if the user requests an explicit context from an IO object. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 351ffcc4..8e9ffbf8 100644 --- a/src/module.c +++ b/src/module.c @@ -3177,7 +3177,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - if (io->ctx->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { + if (io->type->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } From 23f5cb423341b014f6a7005e2cbe10c22e7098e3 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 10:58:15 +0200 Subject: [PATCH 079/116] Change a bit the style of #6385. --- src/module.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index 9c2bed08..13353df2 100644 --- a/src/module.c +++ b/src/module.c @@ -1149,10 +1149,8 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - const size_t msgLen = strlen(msg); - const size_t prefixLen = strlen(prefix); - addReplyProto(c,prefix,prefixLen); - addReplyProto(c,msg,msgLen); + addReplyProto(c,prefix,strlen(prefix)); + addReplyProto(c,msg,strlen(msg)); addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } From 6a2831c24affd565b111398846c6cf6e40eca6e7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 17:17:36 +0200 Subject: [PATCH 080/116] Fix comments aesthetics. --- src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index a2af7824..79c458ee 100644 --- a/src/server.c +++ b/src/server.c @@ -4058,8 +4058,9 @@ sds genRedisInfoString(char *section) { mh->allocator_rss_bytes, mh->rss_extra, mh->rss_extra_bytes, - mh->total_frag, /* this is the total RSS overhead, including fragmentation, */ - mh->total_frag_bytes, /* named so for backwards compatibility */ + mh->total_frag, /* This is the total RSS overhead, including + fragmentation. */ + mh->total_frag_bytes, /* Named so for backwards compatibility. */ freeMemoryGetNotCountedMemory(), mh->repl_backlog, mh->clients_slaves, From ea7c3fe7fd6d269e1e09443db3e2148cb445febb Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Tue, 17 Sep 2019 03:32:35 -0700 Subject: [PATCH 081/116] Allowed passing in of password hash and fixed config rewrite --- src/acl.c | 43 +++++++++++++++++++++++++++++++++++++----- tests/support/test.tcl | 6 ++++++ tests/unit/acl.tcl | 22 ++++++++++++++++++++- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/acl.c b/src/acl.c index 2cd729e7..9c5196f4 100644 --- a/src/acl.c +++ b/src/acl.c @@ -94,6 +94,9 @@ void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +/* The length of the string representation of a hashed password. */ +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 + /* ============================================================================= * Helper functions for the rest of the ACL implementation * ==========================================================================*/ @@ -145,7 +148,7 @@ int time_independent_strcmp(char *a, char *b) { sds ACLHashPassword(unsigned char *cleartext, size_t len) { SHA256_CTX ctx; unsigned char hash[SHA256_BLOCK_SIZE]; - char hex[SHA256_BLOCK_SIZE*2]; + char hex[HASH_PASSWORD_LEN]; char *cset = "0123456789abcdef"; sha256_init(&ctx); @@ -156,7 +159,7 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) { hex[j*2] = cset[((hash[j]&0xF0)>>4)]; hex[j*2+1] = cset[(hash[j]&0xF)]; } - return sdsnewlen(hex,SHA256_BLOCK_SIZE*2); + return sdsnewlen(hex,HASH_PASSWORD_LEN); } /* ============================================================================= @@ -522,7 +525,7 @@ sds ACLDescribeUser(user *u) { listRewind(u->passwords,&li); while((ln = listNext(&li))) { sds thispass = listNodeValue(ln); - res = sdscatlen(res,">",1); + res = sdscatlen(res,"#",1); res = sdscatsds(res,thispass); res = sdscatlen(res," ",1); } @@ -649,6 +652,10 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). * < Remove this password from the list of valid passwords. * nopass All the set passwords of the user are removed, and the user * is flagged as requiring no password: it means that every @@ -685,6 +692,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * EEXIST: You are adding a key pattern after "*" was already added. This is * almost surely an error on the user side. * ENODEV: The password you are trying to remove from the user does not exist. + * EBADMSG: The hash you are trying to add is not a valid hash. */ int ACLSetUser(user *u, const char *op, ssize_t oplen) { if (oplen == -1) oplen = strlen(op); @@ -720,8 +728,30 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"resetpass")) { u->flags &= ~USER_FLAG_NOPASS; listEmpty(u->passwords); - } else if (op[0] == '>') { - sds newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else if (op[0] == '>' || op[0] == '#') { + sds newpass; + if (op[0] == '>') { + newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (oplen != HASH_PASSWORD_LEN + 1) { + errno = EBADMSG; + return C_ERR; + } + + /* Password hashes can only be characters that represent + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. + */ + for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) { + char c = op[i]; + if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) { + errno = EBADMSG; + return C_ERR; + } + } + newpass = sdsnewlen(op+1,oplen-1); + } + listNode *ln = listSearchKey(u->passwords,newpass); /* Avoid re-adding the same password multiple times. */ if (ln == NULL) @@ -848,6 +878,9 @@ char *ACLSetUserStringError(void) { else if (errno == ENODEV) errmsg = "The password you are trying to remove from the user does " "not exist"; + else if (errno == EBADMSG) + errmsg = "The password hash must be exactly 64 characters and contain " + "only lowercase hexadecimal characters"; return errmsg; } diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 6f02f2f1..2646acec 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -15,6 +15,12 @@ proc assert {condition} { } } +proc assert_no_match {pattern value} { + if {[string match $pattern $value]} { + error "assertion:Expected '$value' to not match '$pattern'" + } +} + proc assert_match {pattern value} { if {![string match $pattern $value]} { error "assertion:Expected '$value' to match '$pattern'" diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 05844143..d3b72120 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -35,6 +35,26 @@ start_server {tags {"acl"}} { set e } {*WRONGPASS*} + test {Test password hashes can be added} { + r ACL setuser newuser #34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6 + catch {r AUTH newuser passwd4} e + assert {$e eq "OK"} + } + + test {Test password hashes validate input} { + # Validate Length + catch {r ACL setuser newuser #34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e} e + # Validate character outside set + catch {r ACL setuser newuser #34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4eq} e + set e + } {*Error in ACL SETUSER modifier*} + + test {ACL GETUSER returns the password hash instead of the actual password} { + set passstr [dict get [r ACL getuser newuser] passwords] + assert_match {*34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6*} $passstr + assert_no_match {*passwd4*} $passstr + } + test {By default users are not able to access any command} { catch {r SET foo bar} e set e @@ -67,7 +87,7 @@ start_server {tags {"acl"}} { set e } {*NOPERM*} - test {ACLs can include or excluse whole classes of commands} { + test {ACLs can include or exclude whole classes of commands} { r ACL setuser newuser -@all +@set +acl r SADD myset a b c; # Should not raise an error r ACL setuser newuser +@all -@string From 09041b9359dfcdc84cf5570ded44160fbe7e2bfb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 18:22:55 +0200 Subject: [PATCH 082/116] ACLs: change hashed passwords opcode to also remove them. Related to PR #6405 --- src/acl.c | 32 ++++++++++++++++++++++---------- tests/unit/acl.tcl | 6 ++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/acl.c b/src/acl.c index 9c5196f4..4c43add1 100644 --- a/src/acl.c +++ b/src/acl.c @@ -95,7 +95,7 @@ void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); /* The length of the string representation of a hashed password. */ -#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 /* ============================================================================= * Helper functions for the rest of the ACL implementation @@ -652,11 +652,14 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). - * # Add this password hash to the list of valid hashes for - * the user. This is useful if you have previously computed - * the hash, and don't want to store it in plaintext. - * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). * < Remove this password from the list of valid passwords. + * ! Remove this hashed password from the list of valid passwords. + * This is useful when you want to remove a password just by + * hash without knowing its plaintext version at all. * nopass All the set passwords of the user are removed, and the user * is flagged as requiring no password: it means that every * password will work against this user. If this directive is @@ -735,12 +738,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else { if (oplen != HASH_PASSWORD_LEN + 1) { errno = EBADMSG; - return C_ERR; + return C_ERR; } /* Password hashes can only be characters that represent - * hexadecimal values, which are numbers and lowercase - * characters 'a' through 'f'. + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. */ for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) { char c = op[i]; @@ -759,8 +762,17 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { else sdsfree(newpass); u->flags &= ~USER_FLAG_NOPASS; - } else if (op[0] == '<') { - sds delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else if (op[0] == '<' || op[0] == '!') { + sds delpass; + if (op[0] == '<') { + delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (oplen != HASH_PASSWORD_LEN + 1) { + errno = EBADMSG; + return C_ERR; + } + delpass = sdsnewlen(op+1,oplen-1); + } listNode *ln = listSearchKey(u->passwords,delpass); sdsfree(delpass); if (ln) { diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index d3b72120..2205d2d8 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -55,6 +55,12 @@ start_server {tags {"acl"}} { assert_no_match {*passwd4*} $passstr } + test {Test hashed passwords removal} { + r ACL setuser newuser !34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6 + set passstr [dict get [r ACL getuser newuser] passwords] + assert_no_match {*34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6*} $passstr + } + test {By default users are not able to access any command} { catch {r SET foo bar} e set e From a9628142fd3ce5e3d221ba4b60bcb33c394140e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 18:37:59 +0200 Subject: [PATCH 083/116] Clarify a comment about memory total_frag field. --- src/server.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 79c458ee..cacce6dc 100644 --- a/src/server.c +++ b/src/server.c @@ -4058,9 +4058,11 @@ sds genRedisInfoString(char *section) { mh->allocator_rss_bytes, mh->rss_extra, mh->rss_extra_bytes, - mh->total_frag, /* This is the total RSS overhead, including - fragmentation. */ - mh->total_frag_bytes, /* Named so for backwards compatibility. */ + mh->total_frag, /* This is the total RSS overhead, including + fragmentation, but not just it. This field + (and the next one) is named like that just + for backward compatibility. */ + mh->total_frag_bytes, freeMemoryGetNotCountedMemory(), mh->repl_backlog, mh->clients_slaves, From 1b4f888109b90b8982d16402268c6dbb90225430 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 30 Sep 2019 21:13:13 +0300 Subject: [PATCH 084/116] Use sdscatfmt instead of sdscatprintf in module info sdscatfmt is faster --- src/module.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/module.c b/src/module.c index efbf8eb8..18622bd6 100644 --- a/src/module.c +++ b/src/module.c @@ -4824,7 +4824,7 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { sds full_name = sdsdup(ctx->module->name); if (name != NULL && strlen(name) > 0) - full_name = sdscatprintf(full_name, "_%s", name); + full_name = sdscatfmt(full_name, "_%s", name); /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) @@ -4843,7 +4843,7 @@ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { } } if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n"); - ctx->info = sdscatprintf(ctx->info, "# %s\r\n", full_name); + ctx->info = sdscatfmt(ctx->info, "# %S\r\n", full_name); ctx->in_section = 1; sdsfree(full_name); return REDISMODULE_OK; @@ -4858,7 +4858,7 @@ int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) { /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) RM_InfoEndDictField(ctx); - ctx->info = sdscatprintf(ctx->info, + ctx->info = sdscatfmt(ctx->info, "%s_%s:", ctx->module->name, name); @@ -4873,7 +4873,7 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { /* trim the last ',' if found. */ if (ctx->info[sdslen(ctx->info)-1]==',') sdsIncrLen(ctx->info, -1); - ctx->info = sdscatprintf(ctx->info, "\r\n"); + ctx->info = sdscat(ctx->info, "\r\n"); ctx->in_dict_field = 0; return REDISMODULE_OK; } @@ -4885,14 +4885,14 @@ int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleStrin if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, - "%s=%s,", + ctx->info = sdscatfmt(ctx->info, + "%s=%S,", field, (sds)value->ptr); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, - "%s_%s:%s\r\n", + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%S\r\n", ctx->module->name, field, (sds)value->ptr); @@ -4903,13 +4903,13 @@ int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, + ctx->info = sdscatfmt(ctx->info, "%s=%s,", field, value); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, + ctx->info = sdscatfmt(ctx->info, "%s_%s:%s\r\n", ctx->module->name, field, @@ -4939,14 +4939,14 @@ int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long valu if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, - "%s=%lld,", + ctx->info = sdscatfmt(ctx->info, + "%s=%I,", field, value); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, - "%s_%s:%lld\r\n", + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%I\r\n", ctx->module->name, field, value); @@ -4957,14 +4957,14 @@ int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, - "%s=%llu,", + ctx->info = sdscatfmt(ctx->info, + "%s=%U,", field, value); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, - "%s_%s:%llu\r\n", + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%U\r\n", ctx->module->name, field, value); @@ -5712,9 +5712,9 @@ sds genModulesInfoString(sds info) { sds usedby = genModulesInfoStringRenderModulesList(module->usedby); sds using = genModulesInfoStringRenderModulesList(module->using); sds options = genModulesInfoStringRenderModuleOptions(module); - info = sdscatprintf(info, - "module:name=%s,ver=%d,api=%d,filters=%d," - "usedby=%s,using=%s,options=%s\r\n", + info = sdscatfmt(info, + "module:name=%S,ver=%i,api=%i,filters=%i," + "usedby=%S,using=%S,options=%S\r\n", name, module->ver, module->apiver, (int)listLength(module->filters), usedby, using, options); sdsfree(usedby); From 3281ebb495f0c7fb14bf28df3534d800e0c7d6a1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 1 Oct 2019 10:38:56 +0200 Subject: [PATCH 085/116] Fix GEORADIUS replies broken after RESP3 introduction. This commit fixes #6417. --- src/geo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geo.c b/src/geo.c index 826d11ff..470a615f 100644 --- a/src/geo.c +++ b/src/geo.c @@ -466,7 +466,7 @@ void georadiusGeneric(client *c, int flags) { /* Look up the requested zset */ robj *zobj = NULL; - if ((zobj = lookupKeyReadOrReply(c, key, shared.null[c->resp])) == NULL || + if ((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL || checkType(c, zobj, OBJ_ZSET)) { return; } @@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { /* If no matching results, the user gets an empty reply. */ if (ga->used == 0 && storekey == NULL) { - addReplyNull(c); + addReplyNullArray(c); geoArrayFree(ga); return; } From 40acb4412d9409ef4339d07caeeba540256a062a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 1 Oct 2019 19:18:08 +0200 Subject: [PATCH 086/116] GEORADIUS reply: fix of the previous fix about #6417. --- src/geo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geo.c b/src/geo.c index 470a615f..f47f4ee2 100644 --- a/src/geo.c +++ b/src/geo.c @@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { /* If no matching results, the user gets an empty reply. */ if (ga->used == 0 && storekey == NULL) { - addReplyNullArray(c); + addReply(c,shared.emptyarray); geoArrayFree(ga); return; } From 98426e98868e91e8be42849ad13d36ea538ccf25 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 2 Oct 2019 08:40:35 +0300 Subject: [PATCH 087/116] On LUA script timeout, print the script SHA to the log since the slowlog and other means that can help you detect the bad script are only exposed after the script is done. it might be a good idea to at least print the script name (sha) to the log when it timeouts. --- src/scripting.c | 9 ++++++++- src/server.h | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 3129e4f4..8ce090e9 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1083,6 +1083,7 @@ void scriptingInit(int setup) { if (setup) { server.lua_client = NULL; server.lua_caller = NULL; + server.lua_cur_script = NULL; server.lua_timedout = 0; ldbInit(); } @@ -1407,7 +1408,11 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { /* Set the timeout condition if not already set and the maximum * execution time was reached. */ if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { - serverLog(LL_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); + serverLog(LL_WARNING, + "Lua slow script detected: still in execution after %lld milliseconds. " + "You can try killing the script using the SCRIPT KILL command. " + "script SHA is: %s", + elapsed, server.lua_cur_script); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason @@ -1524,6 +1529,7 @@ void evalGenericCommand(client *c, int evalsha) { * If we are debugging, we set instead a "line" hook so that the * debugger is call-back at every line executed by the script. */ server.lua_caller = c; + server.lua_cur_script = funcname + 2; server.lua_time_start = mstime(); server.lua_kill = 0; if (server.lua_time_limit > 0 && ldb.active == 0) { @@ -1550,6 +1556,7 @@ void evalGenericCommand(client *c, int evalsha) { queueClientForReprocessing(server.master); } server.lua_caller = NULL; + server.lua_cur_script = NULL; /* Call the Lua garbage collector from time to time to avoid a * full cycle performed by Lua, which adds too latency. diff --git a/src/server.h b/src/server.h index 6e011a2c..6f3451eb 100644 --- a/src/server.h +++ b/src/server.h @@ -1389,6 +1389,7 @@ struct redisServer { lua_State *lua; /* The Lua interpreter. We use just one for all clients */ client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ + char* lua_cur_script; /* The current script right now, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ From f49f0a6f729720cc1462d30d036c8ba2baef361b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 Oct 2019 11:27:21 +0200 Subject: [PATCH 088/116] SDS: make sdscatfmt() faster by pre-allocating a bit. --- src/sds.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sds.c b/src/sds.c index cd60946b..98bd2e77 100644 --- a/src/sds.c +++ b/src/sds.c @@ -603,6 +603,10 @@ sds sdscatfmt(sds s, char const *fmt, ...) { long i; va_list ap; + /* To avoid continuous reallocations, let's start with a buffer that + * can hold at least two times the format string itself. It's not the + * best heuristic but seems to work in practice. */ + s = sdsMakeRoomFor(s, initlen + strlen(fmt)*2); va_start(ap,fmt); f = fmt; /* Next format specifier byte to process. */ i = initlen; /* Position of the next byte to write to dest str. */ From 758b39be997f16bc084055edb8caa71be2f2ffcc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 Oct 2019 11:21:24 +0200 Subject: [PATCH 089/116] Speedup INFO server section. --- src/release.c | 14 ++++++++++++++ src/server.c | 34 +++++++++++++++++----------------- src/server.h | 1 + 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/release.c b/src/release.c index 4e59c747..e0bd018f 100644 --- a/src/release.c +++ b/src/release.c @@ -32,6 +32,7 @@ * files using this functions. */ #include +#include #include "release.h" #include "version.h" @@ -50,3 +51,16 @@ uint64_t redisBuildId(void) { return crc64(0,(unsigned char*)buildid,strlen(buildid)); } + +/* Return a cached value of the build string in order to avoid recomputing + * and converting it in hex every time: this string is shown in the INFO + * output that should be fast. */ +char *redisBuildIdString(void) { + static char buf[32]; + static int cached = 0; + if (!cached) { + snprintf(buf,sizeof(buf),"%llx",(unsigned long long) redisBuildId()); + cached = 1; + } + return buf; +} diff --git a/src/server.c b/src/server.c index 9380a840..593f98f3 100644 --- a/src/server.c +++ b/src/server.c @@ -3892,32 +3892,32 @@ sds genRedisInfoString(char *section) { call_uname = 0; } - info = sdscatprintf(info, + info = sdscatfmt(info, "# Server\r\n" "redis_version:%s\r\n" "redis_git_sha1:%s\r\n" - "redis_git_dirty:%d\r\n" - "redis_build_id:%llx\r\n" + "redis_git_dirty:%i\r\n" + "redis_build_id:%s\r\n" "redis_mode:%s\r\n" "os:%s %s %s\r\n" - "arch_bits:%d\r\n" + "arch_bits:%i\r\n" "multiplexing_api:%s\r\n" "atomicvar_api:%s\r\n" - "gcc_version:%d.%d.%d\r\n" - "process_id:%ld\r\n" + "gcc_version:%i.%i.%i\r\n" + "process_id:%I\r\n" "run_id:%s\r\n" - "tcp_port:%d\r\n" - "uptime_in_seconds:%jd\r\n" - "uptime_in_days:%jd\r\n" - "hz:%d\r\n" - "configured_hz:%d\r\n" - "lru_clock:%ld\r\n" + "tcp_port:%i\r\n" + "uptime_in_seconds:%I\r\n" + "uptime_in_days:%I\r\n" + "hz:%i\r\n" + "configured_hz:%i\r\n" + "lru_clock:%u\r\n" "executable:%s\r\n" "config_file:%s\r\n", REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, - (unsigned long long) redisBuildId(), + redisBuildIdString(), mode, name.sysname, name.release, name.machine, server.arch_bits, @@ -3928,14 +3928,14 @@ sds genRedisInfoString(char *section) { #else 0,0,0, #endif - (long) getpid(), + (int64_t) getpid(), server.runid, server.port, - (intmax_t)uptime, - (intmax_t)(uptime/(3600*24)), + (int64_t)uptime, + (int64_t)(uptime/(3600*24)), server.hz, server.config_hz, - (unsigned long) server.lruclock, + server.lruclock, server.executable ? server.executable : "", server.configfile ? server.configfile : ""); } diff --git a/src/server.h b/src/server.h index 6e011a2c..4d48fcd1 100644 --- a/src/server.h +++ b/src/server.h @@ -2140,6 +2140,7 @@ void dictSdsDestructor(void *privdata, void *val); char *redisGitSHA1(void); char *redisGitDirty(void); uint64_t redisBuildId(void); +char *redisBuildIdString(void); /* Commands prototypes */ void authCommand(client *c); From 2e19b941136ebd4e1d9c4766a040d3148a55d24e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 9 Sep 2019 14:35:06 +0300 Subject: [PATCH 090/116] RED-31295 - redis: avoid race between dlopen and thread creation It seeems that since I added the creation of the jemalloc thread redis sometimes fails to start with the following error: Inconsistency detected by ld.so: dl-tls.c: 493: _dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen <= GL(dl_tls_generation)' failed! This seems to be due to a race bug in ld.so, in which TLS creation on the thread, collide with dlopen. Move the creation of BIO and jemalloc threads to after modules are loaded. plus small bugfix when trying to disable the jemalloc thread at runtime --- src/server.c | 10 ++++++++++ src/zmalloc.c | 6 ++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index fa2c7b1e..8e99431f 100644 --- a/src/server.c +++ b/src/server.c @@ -2865,6 +2865,14 @@ void initServer(void) { scriptingInit(1); slowlogInit(); latencyMonitorInit(); +} + +/* Some steps in server initialization need to be done last (after modules + * are loaded). + * Specifically, creation of threads due to a race bug in ld.so, in which + * Thread Local Storage initialization collides with dlopen call. + * see: https://sourceware.org/bugzilla/show_bug.cgi?id=19329 */ +void InitServerLast() { bioInit(); initThreadedIO(); set_jemalloc_bg_thread(server.jemalloc_bg_thread); @@ -4876,6 +4884,7 @@ int main(int argc, char **argv) { #endif moduleLoadFromQueue(); ACLLoadUsersAtStartup(); + InitServerLast(); loadDataFromDisk(); if (server.cluster_enabled) { if (verifyClusterConfigWithData() == C_ERR) { @@ -4890,6 +4899,7 @@ int main(int argc, char **argv) { if (server.sofd > 0) serverLog(LL_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); } else { + InitServerLast(); sentinelIsRunning(); } diff --git a/src/zmalloc.c b/src/zmalloc.c index 58896a72..437f43b6 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -332,10 +332,8 @@ int zmalloc_get_allocator_info(size_t *allocated, void set_jemalloc_bg_thread(int enable) { /* let jemalloc do purging asynchronously, required when there's no traffic * after flushdb */ - if (enable) { - char val = 1; - je_mallctl("background_thread", NULL, 0, &val, 1); - } + char val = !!enable; + je_mallctl("background_thread", NULL, 0, &val, 1); } int jemalloc_purge() { From c549513acdf2eb3232d7ad7c8903f10ea85789b6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 Oct 2019 18:33:40 +0200 Subject: [PATCH 091/116] Modules: handle propagation when ctx is freed. Flag modules commands ctx. --- src/module.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/module.c b/src/module.c index 18622bd6..9876a966 100644 --- a/src/module.c +++ b/src/module.c @@ -158,6 +158,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; #define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4) #define REDISMODULE_CTX_THREAD_SAFE (1<<5) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6) +#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<7) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -519,8 +520,29 @@ int RM_GetApi(const char *funcname, void **targetPtrPtr) { return REDISMODULE_OK; } +/* Helper function for when a command callback is called, in order to handle + * details needed to correctly replicate commands. */ +void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { + client *c = ctx->client; + + /* We don't need to do anything here if the context was never used + * in order to propagate commands. */ + if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return; + + if (c->flags & CLIENT_LUA) return; + + /* Handle the replication of the final EXEC, since whatever a command + * emits is always wrapped around MULTI/EXEC. */ + robj *propargv[1]; + propargv[0] = createStringObject("EXEC",4); + alsoPropagate(server.execCommand,c->db->id,propargv,1, + PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(propargv[0]); +} + /* Free the context after the user function was called. */ void moduleFreeContext(RedisModuleCtx *ctx) { + moduleHandlePropagationAfterCommandCallback(ctx); autoMemoryCollect(ctx); poolAllocRelease(ctx); if (ctx->postponed_arrays) { @@ -536,34 +558,16 @@ void moduleFreeContext(RedisModuleCtx *ctx) { if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) freeClient(ctx->client); } -/* Helper function for when a command callback is called, in order to handle - * details needed to correctly replicate commands. */ -void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { - client *c = ctx->client; - - if (c->flags & CLIENT_LUA) return; - - /* Handle the replication of the final EXEC, since whatever a command - * emits is always wrapped around MULTI/EXEC. */ - if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { - robj *propargv[1]; - propargv[0] = createStringObject("EXEC",4); - alsoPropagate(server.execCommand,c->db->id,propargv,1, - PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(propargv[0]); - } -} - /* This Redis command binds the normal Redis command invocation with commands * exported by modules. */ void RedisModuleCommandDispatcher(client *c) { RedisModuleCommandProxy *cp = (void*)(unsigned long)c->cmd->getkeys_proc; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_MODULE_COMMAND_CALL; ctx.module = cp->module; ctx.client = c; cp->func(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to From e938bbc543c7a7aa3d67ee75f87eb5412df0a3ef Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 10:56:37 +0200 Subject: [PATCH 092/116] Modules: implement RM_Replicate() from async callbacks. --- src/module.c | 33 +++++++++++++++++++++++++++++++-- src/server.h | 2 ++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 9876a966..e8fb5adb 100644 --- a/src/module.c +++ b/src/module.c @@ -147,10 +147,14 @@ struct RedisModuleCtx { int keys_count; struct RedisModulePoolAllocBlock *pa_head; + redisOpArray saved_oparray; /* When propagating commands in a callback + we reallocate the "also propagate" op + array. Here we save the old one to + restore it later. */ }; typedef struct RedisModuleCtx RedisModuleCtx; -#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL} +#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL, {0}} #define REDISMODULE_CTX_MULTI_EMITTED (1<<0) #define REDISMODULE_CTX_AUTO_MEMORY (1<<1) #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2) @@ -538,6 +542,24 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { alsoPropagate(server.execCommand,c->db->id,propargv,1, PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(propargv[0]); + + /* If this is not a module command context (but is instead a simple + * callback context), we have to handle directly the "also propagate" + * array and emit it. In a module command call this will be handled + * directly by call(). */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL) && + server.also_propagate.numops) + { + for (int j = 0; j < server.also_propagate.numops; j++) { + redisOp *rop = &server.also_propagate.ops[j]; + int target = rop->target; + if (target) + propagate(rop->cmd,rop->dbid,rop->argv,rop->argc,target); + } + redisOpArrayFree(&server.also_propagate); + } + /* Restore the previous oparray in case of nexted use of the API. */ + server.also_propagate = ctx->saved_oparray; } /* Free the context after the user function was called. */ @@ -1354,9 +1376,16 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* If we already emitted MULTI return ASAP. */ if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return; /* If this is a thread safe context, we do not want to wrap commands - * executed into MUTLI/EXEC, they are executed as single commands + * executed into MULTI/EXEC, they are executed as single commands * from an external client in essence. */ if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) return; + /* If this is a callback context, and not a module command execution + * context, we have to setup the op array for the "also propagate" API + * so that RM_Replicate() will work. */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) { + ctx->saved_oparray = server.also_propagate; + redisOpArrayInit(&server.also_propagate); + } execCommandPropagateMulti(ctx->client); ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; } diff --git a/src/server.h b/src/server.h index 4d48fcd1..c78010ec 100644 --- a/src/server.h +++ b/src/server.h @@ -1922,6 +1922,8 @@ struct redisCommand *lookupCommandOrOriginal(sds name); void call(client *c, int flags); void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags); void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int target); +void redisOpArrayInit(redisOpArray *oa); +void redisOpArrayFree(redisOpArray *oa); void forceCommandPropagation(client *c, int flags); void preventCommandPropagation(client *c); void preventCommandAOF(client *c); From 1bca62c4b751b325c726482dd9a7264125a2664c Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 11:03:46 +0200 Subject: [PATCH 093/116] Modules: RM_Replicate() in thread safe contexts. --- src/module.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index e8fb5adb..c87f81c5 100644 --- a/src/module.c +++ b/src/module.c @@ -1407,6 +1407,20 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * * Please refer to RedisModule_Call() for more information. * + * ## Note about calling this function from a thread safe context: + * + * Normally when you call this function from the callback implementing a + * module command, or any other callback provided by the Redis Module API, + * Redis will accumulate all the calls to this function in the context of + * the callback, and will propagate all the commands wrapped in a MULTI/EXEC + * transaction. However when calling this function from a threaded safe context + * that can live an undefined amount of time, and can be locked/unlocked in + * at will, the behavior is different: MULTI/EXEC wrapper is not emitted + * and the command specified is inserted in the AOF and replication stream + * immediately. + * + * ## Return value + * * The command returns REDISMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { @@ -1424,10 +1438,18 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) va_end(ap); if (argv == NULL) return REDISMODULE_ERR; - /* Replicate! */ - moduleReplicateMultiIfNeeded(ctx); - alsoPropagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + /* Replicate! When we are in a threaded context, we want to just insert + * the replicated command ASAP, since it is not clear when the context + * will stop being used, so accumulating stuff does not make much sense, + * nor we could easily use the alsoPropagate() API from threads. */ + if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) { + propagate(cmd,ctx->client->db->id,argv,argc, + PROPAGATE_AOF|PROPAGATE_REPL); + } else { + moduleReplicateMultiIfNeeded(ctx); + alsoPropagate(cmd,ctx->client->db->id,argv,argc, + PROPAGATE_AOF|PROPAGATE_REPL); + } /* Release the argv. */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); From 45cd8e03cab6c24329c25cbb873405fe86911b5b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 13:06:09 +0200 Subject: [PATCH 094/116] Modules: RM_Replicate() test module: initial implementation. --- tests/modules/Makefile | 5 ++- tests/modules/propagate.c | 82 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/modules/propagate.c diff --git a/tests/modules/Makefile b/tests/modules/Makefile index e669d8e5..c1c8ffa2 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so testrdb.so fork.so infotest.so +all: commandfilter.so testrdb.so fork.so infotest.so propagate.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -22,6 +22,7 @@ commandfilter.xo: ../../src/redismodule.h fork.xo: ../../src/redismodule.h testrdb.xo: ../../src/redismodule.h infotest.xo: ../../src/redismodule.h +propagate.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc @@ -35,3 +36,5 @@ testrdb.so: testrdb.xo infotest.so: infotest.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc +propagate.so: propagate.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c new file mode 100644 index 00000000..e08372dc --- /dev/null +++ b/tests/modules/propagate.c @@ -0,0 +1,82 @@ +/* This module is used to test the propagation (replication + AOF) of + * commands, via the RedisModule_Replicate() interface, in asynchronous + * contexts, such as callbacks not implementing commands, and thread safe + * contexts. + * + * We create a timer callback and a threads using a thread safe context. + * Using both we try to propagate counters increments, and later we check + * if the replica contains the changes as expected. + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +/* Timer callback. */ +void timerHandler(RedisModuleCtx *ctx, void *data) { + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(data); + + static int times = 0; + + printf("Fired!\n"); + RedisModule_Replicate(ctx,"INCR","c","timer"); + times++; + + if (times < 10) + RedisModule_CreateTimer(ctx,100,timerHandler,NULL); +} + +int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModuleTimerID tid = RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + REDISMODULE_NOT_USED(tid); + + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"propagate-test",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"propagate-test", + propagateTestCommand, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; +} From 55a3da87f83c7c912b6c981a3e9fb1259179007a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 13:23:48 +0200 Subject: [PATCH 095/116] Modules: RM_Replicate() test with threads. --- tests/modules/propagate.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index e08372dc..fbd3ec35 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -39,6 +39,7 @@ #define REDISMODULE_EXPERIMENTAL_API #include "redismodule.h" +#include /* Timer callback. */ void timerHandler(RedisModuleCtx *ctx, void *data) { @@ -47,12 +48,26 @@ void timerHandler(RedisModuleCtx *ctx, void *data) { static int times = 0; - printf("Fired!\n"); RedisModule_Replicate(ctx,"INCR","c","timer"); times++; if (times < 10) RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + else + times = 0; +} + +/* The thread entry point. */ +void *threadMain(void *arg) { + REDISMODULE_NOT_USED(arg); + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(NULL); + for (int i = 0; i < 10; i++) { + RedisModule_ThreadSafeContextLock(ctx); + RedisModule_Replicate(ctx,"INCR","c","thread"); + RedisModule_ThreadSafeContextUnlock(ctx); + } + RedisModule_FreeThreadSafeContext(ctx); + return NULL; } int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) @@ -60,7 +75,13 @@ int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); - RedisModuleTimerID tid = RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + RedisModuleTimerID timer_id = + RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + REDISMODULE_NOT_USED(timer_id); + + pthread_t tid; + if (pthread_create(&tid,NULL,threadMain,NULL) != 0) + return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread"); REDISMODULE_NOT_USED(tid); RedisModule_ReplyWithSimpleString(ctx,"OK"); From 1b8b8c029fa09511bfeb01c1808456de7a28565d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 18:44:47 +0200 Subject: [PATCH 096/116] Modules: add RM_Replicate() Tcl test file & fix the module. --- runtest-moduleapi | 2 +- tests/modules/propagate.c | 1 + tests/unit/moduleapi/propagate.tcl | 30 ++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/unit/moduleapi/propagate.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 1f090ff6..a16cca68 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest --single unit/moduleapi/propagate "${@}" diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index fbd3ec35..f83af179 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -61,6 +61,7 @@ void timerHandler(RedisModuleCtx *ctx, void *data) { void *threadMain(void *arg) { REDISMODULE_NOT_USED(arg); RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(NULL); + RedisModule_SelectDb(ctx,9); /* Tests ran in database number 9. */ for (int i = 0; i < 10; i++) { RedisModule_ThreadSafeContextLock(ctx); RedisModule_Replicate(ctx,"INCR","c","thread"); diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl new file mode 100644 index 00000000..71307ce3 --- /dev/null +++ b/tests/unit/moduleapi/propagate.tcl @@ -0,0 +1,30 @@ +set testmodule [file normalize tests/modules/propagate.so] + +tags "modules" { + test {Modules can propagate in async and threaded contexts} { + start_server {} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + + # Start the replication process... + $replica replicaof $master_host $master_port + wait_for_sync $replica + + after 1000 + $master propagate-test + + wait_for_condition 5000 10 { + ([$replica get timer] eq "10") && \ + ([$replica get thread] eq "10") + } else { + fail "The two counters don't match the expected value." + } + } + } + } +} From 2a81e49ddee1db6ae3c09bc8b9e1b05d430571f0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 11:44:39 +0200 Subject: [PATCH 097/116] Modules: RM_Call/Replicate() ability to exclude AOF/replicas. --- src/module.c | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/module.c b/src/module.c index c87f81c5..0e9c22e5 100644 --- a/src/module.c +++ b/src/module.c @@ -318,6 +318,10 @@ static struct RedisModuleForkInfo { void* done_handler_user_data; } moduleForkInfo = {0}; +/* Flags for moduleCreateArgvFromUserFormat(). */ +#define REDISMODULE_ARGV_REPLICATE (1<<0) +#define REDISMODULE_ARGV_NO_AOF (1<<1) +#define REDISMODULE_ARGV_NO_REPLICAS (1<<2) /* -------------------------------------------------------------------------- * Prototypes @@ -1407,6 +1411,10 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * * Please refer to RedisModule_Call() for more information. * + * Using the special "A" and "R" modifiers, the caller can exclude either + * the AOF or the replicas from the propagation of the specified command. + * Otherwise, by default, the command will be propagated in both channels. + * * ## Note about calling this function from a thread safe context: * * Normally when you call this function from the callback implementing a @@ -1438,17 +1446,22 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) va_end(ap); if (argv == NULL) return REDISMODULE_ERR; + /* Select the propagation target. Usually is AOF + replicas, however + * the caller can exclude one or the other using the "A" or "R" + * modifiers. */ + int target = 0; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) target |= PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) target |= PROPAGATE_REPL; + /* Replicate! When we are in a threaded context, we want to just insert * the replicated command ASAP, since it is not clear when the context * will stop being used, so accumulating stuff does not make much sense, * nor we could easily use the alsoPropagate() API from threads. */ if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) { - propagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + propagate(cmd,ctx->client->db->id,argv,argc,target); } else { moduleReplicateMultiIfNeeded(ctx); - alsoPropagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(cmd,ctx->client->db->id,argv,argc,target); } /* Release the argv. */ @@ -2767,12 +2780,11 @@ RedisModuleString *RM_CreateStringFromCallReply(RedisModuleCallReply *reply) { * to special modifiers in "fmt". For now only one exists: * * "!" -> REDISMODULE_ARGV_REPLICATE + * "A" -> REDISMODULE_ARGV_NO_AOF + * "R" -> REDISMODULE_ARGV_NO_REPLICAS * * On error (format specifier error) NULL is returned and nothing is * allocated. On success the argument vector is returned. */ - -#define REDISMODULE_ARGV_REPLICATE (1<<0) - robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) { int argc = 0, argv_size, j; robj **argv = NULL; @@ -2821,6 +2833,10 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int } } else if (*p == '!') { if (flags) (*flags) |= REDISMODULE_ARGV_REPLICATE; + } else if (*p == 'A') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_AOF; + } else if (*p == 'R') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_REPLICAS; } else { goto fmterr; } @@ -2912,8 +2928,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* Run the command */ int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; if (replicate) { - call_flags |= CMD_CALL_PROPAGATE_AOF; - call_flags |= CMD_CALL_PROPAGATE_REPL; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) + call_flags |= CMD_CALL_PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) + call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); From 3eaff2941e6625c77c6eee5a53b8b94f94b78e33 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 11:46:53 +0200 Subject: [PATCH 098/116] Modules: RM_Call(): give pointer to documentation. --- src/module.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 0e9c22e5..82ba9f86 100644 --- a/src/module.c +++ b/src/module.c @@ -2857,7 +2857,10 @@ fmterr: * NULL is returned and errno is set to the following values: * * EINVAL: command non existing, wrong arity, wrong format specifier. - * EPERM: operation in Cluster instance with key in non local slot. */ + * EPERM: operation in Cluster instance with key in non local slot. + * + * This API is documented here: https://redis.io/topics/modules-intro + */ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { struct redisCommand *cmd; client *c = NULL; From ee1cef189fff604f165b2d20a307545840de944e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 12:00:41 +0200 Subject: [PATCH 099/116] Minor aesthetic changes to #6419. --- src/scripting.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 8ce090e9..ec95eb25 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1411,7 +1411,7 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { serverLog(LL_WARNING, "Lua slow script detected: still in execution after %lld milliseconds. " "You can try killing the script using the SCRIPT KILL command. " - "script SHA is: %s", + "Script SHA1 is: %s", elapsed, server.lua_cur_script); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others diff --git a/src/server.h b/src/server.h index f24c428c..2e574990 100644 --- a/src/server.h +++ b/src/server.h @@ -1389,7 +1389,7 @@ struct redisServer { lua_State *lua; /* The Lua interpreter. We use just one for all clients */ client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ - char* lua_cur_script; /* The current script right now, or NULL */ + char* lua_cur_script; /* SHA1 of the script currently running, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ From 0f1969f16f541c4a4b17b0063ed764059fe89bc5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Fri, 4 Oct 2019 14:22:13 +0300 Subject: [PATCH 100/116] trim the double implementation of jemalloc purge --- src/object.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/object.c b/src/object.c index 697429b8..70022f89 100644 --- a/src/object.c +++ b/src/object.c @@ -1450,22 +1450,10 @@ NULL addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) { -#if defined(USE_JEMALLOC) - char tmp[32]; - unsigned narenas = 0; - size_t sz = sizeof(unsigned); - if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) { - sprintf(tmp, "arena.%d.purge", narenas); - if (!je_mallctl(tmp, NULL, 0, NULL, 0)) { - addReply(c, shared.ok); - return; - } - } - addReplyError(c, "Error purging dirty pages"); -#else - addReply(c, shared.ok); - /* Nothing to do for other allocators. */ -#endif + if (jemalloc_purge() == 0) + addReply(c, shared.ok); + else + addReplyError(c, "Error purging dirty pages"); } else { addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr); } From 9073d56eec0d37dda201b8aaa7995652fb8afc33 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 18:52:07 +0200 Subject: [PATCH 101/116] LOLWUT: refactoring + skeleton of LOLWUT 6. --- src/Makefile | 2 +- src/lolwut.c | 121 +++++++++++++++++++++++++++++++++++++++++++++++++- src/lolwut.h | 49 ++++++++++++++++++++ src/lolwut5.c | 110 +-------------------------------------------- src/lolwut6.c | 75 +++++++++++++++++++++++++++++++ 5 files changed, 247 insertions(+), 110 deletions(-) create mode 100644 src/lolwut.h create mode 100644 src/lolwut6.c diff --git a/src/Makefile b/src/Makefile index 198d85cd..a76adbf4 100644 --- a/src/Makefile +++ b/src/Makefile @@ -164,7 +164,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o sha256.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o sha256.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/lolwut.c b/src/lolwut.c index ba7e1069..d3e8fbad 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -34,8 +34,11 @@ */ #include "server.h" +#include "lolwut.h" +#include void lolwut5Command(client *c); +void lolwut6Command(client *c); /* The default target for LOLWUT if no matching version was found. * This is what unstable versions of Redis will display. */ @@ -47,11 +50,127 @@ void lolwutUnstableCommand(client *c) { sdsfree(rendered); } +/* LOLWUT [] */ void lolwutCommand(client *c) { char *v = REDIS_VERSION; - if ((v[0] == '5' && v[1] == '.') || + char verstr[64]; + + if (c->argc == 2) { + long ver; + if (getLongFromObjectOrReply(c,c->argv[1],&ver,NULL) != C_OK) return; + snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver); + v = verstr; + } + + if ((v[0] == '5' && v[1] == '.' && v[2] != '9') || (v[0] == '4' && v[1] == '.' && v[2] == '9')) lolwut5Command(c); + else if ((v[0] == '6' && v[1] == '.' && v[2] != '9') || + (v[0] == '5' && v[1] == '.' && v[2] == '9')) + lolwut6Command(c); else lolwutUnstableCommand(c); } + +/* ========================== LOLWUT Canvase =============================== + * Many LOWUT versions will likely print some computer art to the screen. + * This is the case with LOLWUT 5 and LOLWUT 6, so here there is a generic + * canvas implementation that can be reused. */ + +/* Allocate and return a new canvas of the specified size. */ +lwCanvas *lwCreateCanvas(int width, int height) { + lwCanvas *canvas = zmalloc(sizeof(*canvas)); + canvas->width = width; + canvas->height = height; + canvas->pixels = zmalloc(width*height); + memset(canvas->pixels,0,width*height); + return canvas; +} + +/* Free the canvas created by lwCreateCanvas(). */ +void lwFreeCanvas(lwCanvas *canvas) { + zfree(canvas->pixels); + zfree(canvas); +} + +/* Set a pixel to the specified color. Color is 0 or 1, where zero means no + * dot will be displyed, and 1 means dot will be displayed. + * Coordinates are arranged so that left-top corner is 0,0. You can write + * out of the size of the canvas without issues. */ +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return; + canvas->pixels[x+y*canvas->width] = color; +} + +/* Return the value of the specified pixel on the canvas. */ +int lwGetPixel(lwCanvas *canvas, int x, int y) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return 0; + return canvas->pixels[x+y*canvas->width]; +} + +/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { + int dx = abs(x2-x1); + int dy = abs(y2-y1); + int sx = (x1 < x2) ? 1 : -1; + int sy = (y1 < y2) ? 1 : -1; + int err = dx-dy, e2; + + while(1) { + lwDrawPixel(canvas,x1,y1,color); + if (x1 == x2 && y1 == y2) break; + e2 = err*2; + if (e2 > -dy) { + err -= dy; + x1 += sx; + } + if (e2 < dx) { + err += dx; + y1 += sy; + } + } +} + +/* Draw a square centered at the specified x,y coordinates, with the specified + * rotation angle and size. In order to write a rotated square, we use the + * trivial fact that the parametric equation: + * + * x = sin(k) + * y = cos(k) + * + * Describes a circle for values going from 0 to 2*PI. So basically if we start + * at 45 degrees, that is k = PI/4, with the first point, and then we find + * the other three points incrementing K by PI/2 (90 degrees), we'll have the + * points of the square. In order to rotate the square, we just start with + * k = PI/4 + rotation_angle, and we are done. + * + * Of course the vanilla equations above will describe the square inside a + * circle of radius 1, so in order to draw larger squares we'll have to + * multiply the obtained coordinates, and then translate them. However this + * is much simpler than implementing the abstract concept of 2D shape and then + * performing the rotation/translation transformation, so for LOLWUT it's + * a good approach. */ +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color) { + int px[4], py[4]; + + /* Adjust the desired size according to the fact that the square inscribed + * into a circle of radius 1 has the side of length SQRT(2). This way + * size becomes a simple multiplication factor we can use with our + * coordinates to magnify them. */ + size /= 1.4142135623; + size = round(size); + + /* Compute the four points. */ + float k = M_PI/4 + angle; + for (int j = 0; j < 4; j++) { + px[j] = round(sin(k) * size + x); + py[j] = round(cos(k) * size + y); + k += M_PI/2; + } + + /* Draw the square. */ + for (int j = 0; j < 4; j++) + lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],color); +} diff --git a/src/lolwut.h b/src/lolwut.h new file mode 100644 index 00000000..c049ac90 --- /dev/null +++ b/src/lolwut.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2018-2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* This structure represents our canvas. Drawing functions will take a pointer + * to a canvas to write to it. Later the canvas can be rendered to a string + * suitable to be printed on the screen, using unicode Braille characters. */ + +/* This represents a very simple generic canvas in order to draw stuff. + * It's up to each LOLWUT versions to translate what they draw to the + * screen, depending on the result to accomplish. */ +typedef struct lwCanvas { + int width; + int height; + char *pixels; +} lwCanvas; + +/* Drawing functions implemented inside lolwut.c. */ +lwCanvas *lwCreateCanvas(int width, int height); +void lwFreeCanvas(lwCanvas *canvas); +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color); +int lwGetPixel(lwCanvas *canvas, int x, int y); +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color); +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color); diff --git a/src/lolwut5.c b/src/lolwut5.c index 52a98c0d..4e982814 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -34,17 +34,9 @@ */ #include "server.h" +#include "lolwut.h" #include -/* This structure represents our canvas. Drawing functions will take a pointer - * to a canvas to write to it. Later the canvas can be rendered to a string - * suitable to be printed on the screen, using unicode Braille characters. */ -typedef struct lwCanvas { - int width; - int height; - char *pixels; -} lwCanvas; - /* Translate a group of 8 pixels (2x4 vertical rectangle) to the corresponding * braille character. The byte should correspond to the pixels arranged as * follows, where 0 is the least significant bit, and 7 the most significant @@ -69,104 +61,6 @@ void lwTranslatePixelsGroup(int byte, char *output) { output[2] = 0x80 | (code & 0x3F); /* 10-xxxxxx */ } -/* Allocate and return a new canvas of the specified size. */ -lwCanvas *lwCreateCanvas(int width, int height) { - lwCanvas *canvas = zmalloc(sizeof(*canvas)); - canvas->width = width; - canvas->height = height; - canvas->pixels = zmalloc(width*height); - memset(canvas->pixels,0,width*height); - return canvas; -} - -/* Free the canvas created by lwCreateCanvas(). */ -void lwFreeCanvas(lwCanvas *canvas) { - zfree(canvas->pixels); - zfree(canvas); -} - -/* Set a pixel to the specified color. Color is 0 or 1, where zero means no - * dot will be displyed, and 1 means dot will be displayed. - * Coordinates are arranged so that left-top corner is 0,0. You can write - * out of the size of the canvas without issues. */ -void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { - if (x < 0 || x >= canvas->width || - y < 0 || y >= canvas->height) return; - canvas->pixels[x+y*canvas->width] = color; -} - -/* Return the value of the specified pixel on the canvas. */ -int lwGetPixel(lwCanvas *canvas, int x, int y) { - if (x < 0 || x >= canvas->width || - y < 0 || y >= canvas->height) return 0; - return canvas->pixels[x+y*canvas->width]; -} - -/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ -void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { - int dx = abs(x2-x1); - int dy = abs(y2-y1); - int sx = (x1 < x2) ? 1 : -1; - int sy = (y1 < y2) ? 1 : -1; - int err = dx-dy, e2; - - while(1) { - lwDrawPixel(canvas,x1,y1,color); - if (x1 == x2 && y1 == y2) break; - e2 = err*2; - if (e2 > -dy) { - err -= dy; - x1 += sx; - } - if (e2 < dx) { - err += dx; - y1 += sy; - } - } -} - -/* Draw a square centered at the specified x,y coordinates, with the specified - * rotation angle and size. In order to write a rotated square, we use the - * trivial fact that the parametric equation: - * - * x = sin(k) - * y = cos(k) - * - * Describes a circle for values going from 0 to 2*PI. So basically if we start - * at 45 degrees, that is k = PI/4, with the first point, and then we find - * the other three points incrementing K by PI/2 (90 degrees), we'll have the - * points of the square. In order to rotate the square, we just start with - * k = PI/4 + rotation_angle, and we are done. - * - * Of course the vanilla equations above will describe the square inside a - * circle of radius 1, so in order to draw larger squares we'll have to - * multiply the obtained coordinates, and then translate them. However this - * is much simpler than implementing the abstract concept of 2D shape and then - * performing the rotation/translation transformation, so for LOLWUT it's - * a good approach. */ -void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle) { - int px[4], py[4]; - - /* Adjust the desired size according to the fact that the square inscribed - * into a circle of radius 1 has the side of length SQRT(2). This way - * size becomes a simple multiplication factor we can use with our - * coordinates to magnify them. */ - size /= 1.4142135623; - size = round(size); - - /* Compute the four points. */ - float k = M_PI/4 + angle; - for (int j = 0; j < 4; j++) { - px[j] = round(sin(k) * size + x); - py[j] = round(cos(k) * size + y); - k += M_PI/2; - } - - /* Draw the square. */ - for (int j = 0; j < 4; j++) - lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],1); -} - /* Schotter, the output of LOLWUT of Redis 5, is a computer graphic art piece * generated by Georg Nees in the 60s. It explores the relationship between * caos and order. @@ -200,7 +94,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ sx += r2*square_side/3; sy += r3*square_side/3; } - lwDrawSquare(canvas,sx,sy,square_side,angle); + lwDrawSquare(canvas,sx,sy,square_side,angle,1); } } diff --git a/src/lolwut6.c b/src/lolwut6.c new file mode 100644 index 00000000..68f09036 --- /dev/null +++ b/src/lolwut6.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ---------------------------------------------------------------------------- + * + * This file implements the LOLWUT command. The command should do something + * fun and interesting, and should be replaced by a new implementation at + * each new version of Redis. + */ + +#include "server.h" +#include "lolwut.h" + +/* The LOLWUT 6 command: + * + * LOLWUT [columns] [rows] + * + * By default the command uses 80 columns, 40 squares per row + * per column. + */ +void lolwut6Command(client *c) { + long cols = 80; + long rows = 40; + + /* Parse the optional arguments if any. */ + if (c->argc > 1 && + getLongFromObjectOrReply(c,c->argv[1],&cols,NULL) != C_OK) + return; + + if (c->argc > 2 && + getLongFromObjectOrReply(c,c->argv[2],&rows,NULL) != C_OK) + return; + + /* Limits. We want LOLWUT to be always reasonably fast and cheap to execute + * so we have maximum number of columns, rows, and output resulution. */ + if (cols < 1) cols = 1; + if (cols > 1000) cols = 1000; + if (rows < 1) rows = 1; + if (rows > 1000) rows = 1000; + + /* Generate the city skyline and reply. */ + sds rendered = sdsempty(); + rendered = sdscat(rendered, + "\nDedicated to the 8 bit game developers of the past. Redis ver. "); + rendered = sdscat(rendered,REDIS_VERSION); + rendered = sdscatlen(rendered,"\n",1); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); + // lwFreeCanvas(canvas); +} From bd6044706641fa6370a90dc5347e6cd37f5930eb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 19:18:45 +0200 Subject: [PATCH 102/116] LOLWUT: ability to specify VERSION option. --- src/lolwut.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index d3e8fbad..7e2ceca7 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -50,16 +50,22 @@ void lolwutUnstableCommand(client *c) { sdsfree(rendered); } -/* LOLWUT [] */ +/* LOLWUT [VERSION ] [... version specific arguments ...] */ void lolwutCommand(client *c) { char *v = REDIS_VERSION; char verstr[64]; - if (c->argc == 2) { + if (c->argc >= 3 && !strcasecmp(c->argv[1]->ptr,"version")) { long ver; - if (getLongFromObjectOrReply(c,c->argv[1],&ver,NULL) != C_OK) return; + if (getLongFromObjectOrReply(c,c->argv[2],&ver,NULL) != C_OK) return; snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver); v = verstr; + + /* Adjust argv/argc to filter the "VERSION ..." option, since the + * specific LOLWUT version implementations don't know about it + * and expect their arguments. */ + c->argv += 2; + c->argc -= 2; } if ((v[0] == '5' && v[1] == '.' && v[2] != '9') || @@ -70,6 +76,12 @@ void lolwutCommand(client *c) { lolwut6Command(c); else lolwutUnstableCommand(c); + + /* Fix back argc/argv in case of VERSION argument. */ + if (v == verstr) { + c->argv -= 2; + c->argc += 2; + } } /* ========================== LOLWUT Canvase =============================== From d1a005ab3963c16b65e805675a76f0e40c723158 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 6 Oct 2019 13:55:21 +0300 Subject: [PATCH 103/116] fix issues found by a static analyzer cluster.c - stack buffer memory alignment The pointer 'buf' is cast to a more strictly aligned pointer type evict.c - lazyfree_lazy_eviction, lazyfree_lazy_eviction always called defrag.c - bug in dead code server.c - casting was missing parenthesis rax.c - indentation / newline suggested an 'else if' was intended --- src/cluster.c | 32 +++++++++++++++++--------------- src/defrag.c | 2 +- src/evict.c | 7 +++---- src/rax.c | 3 ++- src/server.c | 2 +- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 1e7dcd50..93be2aa3 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -2140,7 +2140,7 @@ void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; ssize_t nread; clusterMsg *hdr; clusterLink *link = (clusterLink*) privdata; @@ -2517,7 +2517,8 @@ void clusterBroadcastPong(int target) { * * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[sizeof(clusterMsg)], *payload; + unsigned char *payload; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; uint32_t channel_len, message_len; @@ -2537,7 +2538,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - payload = buf; + payload = (unsigned char*)buf; } else { payload = zmalloc(totlen); memcpy(payload,hdr,sizeof(*hdr)); @@ -2554,7 +2555,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { decrRefCount(channel); decrRefCount(message); - if (payload != buf) zfree(payload); + if (payload != (unsigned char*)buf) zfree(payload); } /* Send a FAIL message to all the nodes we are able to contact. @@ -2563,7 +2564,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { * we switch the node state to CLUSTER_NODE_FAIL and ask all the other * nodes to do the same ASAP. */ void clusterSendFail(char *nodename) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); @@ -2575,7 +2576,7 @@ void clusterSendFail(char *nodename) { * slots configuration. The node name, slots bitmap, and configEpoch info * are included. */ void clusterSendUpdate(clusterLink *link, clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; if (link == NULL) return; @@ -2583,7 +2584,7 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { memcpy(hdr->data.update.nodecfg.nodename,node->name,CLUSTER_NAMELEN); hdr->data.update.nodecfg.configEpoch = htonu64(node->configEpoch); memcpy(hdr->data.update.nodecfg.slots,node->slots,sizeof(node->slots)); - clusterSendMessage(link,buf,ntohl(hdr->totlen)); + clusterSendMessage(link,(unsigned char*)buf,ntohl(hdr->totlen)); } /* Send a MODULE message. @@ -2591,7 +2592,8 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, unsigned char *payload, uint32_t len) { - unsigned char buf[sizeof(clusterMsg)], *heapbuf; + unsigned char *heapbuf; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2606,7 +2608,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - heapbuf = buf; + heapbuf = (unsigned char*)buf; } else { heapbuf = zmalloc(totlen); memcpy(heapbuf,hdr,sizeof(*hdr)); @@ -2619,7 +2621,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, else clusterBroadcastMessage(heapbuf,totlen); - if (heapbuf != buf) zfree(heapbuf); + if (heapbuf != (unsigned char*)buf) zfree(heapbuf); } /* This function gets a cluster node ID string as target, the same way the nodes @@ -2663,7 +2665,7 @@ void clusterPropagatePublish(robj *channel, robj *message) { * Note that we send the failover request to everybody, master and slave nodes, * but only the masters are supposed to reply to our query. */ void clusterRequestFailoverAuth(void) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2679,7 +2681,7 @@ void clusterRequestFailoverAuth(void) { /* Send a FAILOVER_AUTH_ACK message to the specified node. */ void clusterSendFailoverAuth(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2687,12 +2689,12 @@ void clusterSendFailoverAuth(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Send a MFSTART message to the specified node. */ void clusterSendMFStart(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2700,7 +2702,7 @@ void clusterSendMFStart(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_MFSTART); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Vote for the node asking for our vote if there are the conditions. */ diff --git a/src/defrag.c b/src/defrag.c index 50f6b41f..e794c8e4 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -374,7 +374,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { if ((newele = activeDefragStringOb(ele, &defragged))) de->v.val = newele, defragged++; } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) { - void *newptr, *ptr = ln->value; + void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) ln->value = newptr, defragged++; } diff --git a/src/evict.c b/src/evict.c index 176f4c36..71260c04 100644 --- a/src/evict.c +++ b/src/evict.c @@ -444,6 +444,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev * Otehrwise if we are over the memory limit, but not enough memory * was freed to return back under the limit, the function returns C_ERR. */ int freeMemoryIfNeeded(void) { + int keys_freed = 0; /* By default replicas should ignore maxmemory * and just be masters exact copies. */ if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; @@ -467,7 +468,7 @@ int freeMemoryIfNeeded(void) { latencyStartMonitor(latency); while (mem_freed < mem_tofree) { - int j, k, i, keys_freed = 0; + int j, k, i; static unsigned int next_db = 0; sds bestkey = NULL; int bestdbid; @@ -598,9 +599,7 @@ int freeMemoryIfNeeded(void) { mem_freed = mem_tofree; } } - } - - if (!keys_freed) { + } else { latencyEndMonitor(latency); latencyAddSampleIfNeeded("eviction-cycle",latency); goto cant_free; /* nothing to free... */ diff --git a/src/rax.c b/src/rax.c index b3c263dc..be474b05 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1791,7 +1791,8 @@ int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key if (eq && key_len == iter->key_len) return 1; else if (lt) return iter->key_len < key_len; else if (gt) return iter->key_len > key_len; - } if (cmp > 0) { + return 0; + } else if (cmp > 0) { return gt ? 1 : 0; } else /* (cmp < 0) */ { return lt ? 1 : 0; diff --git a/src/server.c b/src/server.c index 593f98f3..e089865f 100644 --- a/src/server.c +++ b/src/server.c @@ -4278,7 +4278,7 @@ sds genRedisInfoString(char *section) { if (server.repl_state != REPL_STATE_CONNECTED) { info = sdscatprintf(info, "master_link_down_since_seconds:%jd\r\n", - (intmax_t)server.unixtime-server.repl_down_since); + (intmax_t)(server.unixtime-server.repl_down_since)); } info = sdscatprintf(info, "slave_priority:%d\r\n" From f019f28e7b9f05205c02261bb3af8261869d600a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 12:35:05 +0200 Subject: [PATCH 104/116] LOLWUT: version 6 initial concept. --- src/lolwut5.c | 4 +-- src/lolwut6.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/lolwut5.c b/src/lolwut5.c index 4e982814..0200fece 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -106,7 +106,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ * logical canvas. The actual returned string will require a terminal that is * width/2 large and height/4 tall in order to hold the whole image without * overflowing or scrolling, since each Barille character is 2x4. */ -sds lwRenderCanvas(lwCanvas *canvas) { +static sds renderCanvas(lwCanvas *canvas) { sds text = sdsempty(); for (int y = 0; y < canvas->height; y += 4) { for (int x = 0; x < canvas->width; x += 2) { @@ -166,7 +166,7 @@ void lolwut5Command(client *c) { /* Generate some computer art and reply. */ lwCanvas *canvas = lwDrawSchotter(cols,squares_per_row,squares_per_col); - sds rendered = lwRenderCanvas(canvas); + sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); diff --git a/src/lolwut6.c b/src/lolwut6.c index 68f09036..8cc8d6e0 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -36,6 +36,94 @@ #include "server.h" #include "lolwut.h" +/* Render the canvas using the four gray levels of the standard color + * terminal: they match very well to the grayscale display of the gameboy. */ +static sds renderCanvas(lwCanvas *canvas) { + sds text = sdsempty(); + for (int y = 0; y < canvas->height; y++) { + for (int x = 0; x < canvas->width; x++) { + int color = lwGetPixel(canvas,x,y); + char *ce; /* Color escape sequence. */ + + /* Note that we set both the foreground and background color. + * This way we are able to get a more consistent result among + * different terminals implementations. */ + switch(color) { + case 0: ce = "0;30;40m"; break; /* Black */ + case 1: ce = "0;90;100m"; break; /* Gray 1 */ + case 2: ce = "0;37;47m"; break; /* Gray 2 */ + case 3: ce = "0;97;107m"; break; /* White */ + } + text = sdscatprintf(text,"\033[%s \033[0m",ce); + } + if (y != canvas->height-1) text = sdscatlen(text,"\n",1); + } + return text; +} + +/* Draw a skyscraper on the canvas, according to the parameters in the + * 'skyscraper' structure. Window colors are random and are always selected + * to be different than the color of the skyscraper itsefl. */ +struct skyscraper { + int xoff; /* X offset. */ + int width; /* Pixels width. */ + int height; /* Pixels height. */ + int windows; /* Draw windows if true. */ + int color; /* Color of the skyscraper. */ +}; + +void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { + int starty = canvas->height-1; + int endy = starty - si->height + 1; + for (int y = starty; y >= endy; y--) { + for (int x = si->xoff; x < si->xoff+si->width; x++) { + /* The roof is four pixels less wide. */ + if (y == endy && (x <= si->xoff+1 || x >= si->xoff+si->width-2)) + continue; + int color = si->color; + /* Alter the color if this is a place where we want to + * draw a window. We check that we are in the inner part of the + * skyscraper, so that windows are far from the borders. */ + if (si->windows && + x > si->xoff+1 && + x < si->xoff+si->width-2 && + y > endy+1 && + y < starty-1) + { + /* Calculate the x,y position relative to the start of + * the window area. */ + int relx = x - (si->xoff+1); + int rely = y - (endy+1); + + /* Note that we want the windows to be two pixels wide + * but just one pixel tall, because terminal "pixels" + * (characters) are not square. */ + if (relx/2 % 2 && rely % 2) { + do { + color = rand() % 4; + } while (color == si->color); + /* Except we want adjacent pixels creating the same + * window to be the same color. */ + if (relx % 2) color = lwGetPixel(canvas,x-1,y); + } + } + lwDrawPixel(canvas,x,y,color); + } + } +} + +/* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */ +void generateSkyline(lwCanvas *canvas) { + struct skyscraper si = { + .xoff = 4, + .width = 13, + .height = 15, + .windows = 1, + .color = 1 + }; + generateSkyscraper(canvas, &si); +} + /* The LOLWUT 6 command: * * LOLWUT [columns] [rows] @@ -45,7 +133,7 @@ */ void lolwut6Command(client *c) { long cols = 80; - long rows = 40; + long rows = 20; /* Parse the optional arguments if any. */ if (c->argc > 1 && @@ -64,12 +152,14 @@ void lolwut6Command(client *c) { if (rows > 1000) rows = 1000; /* Generate the city skyline and reply. */ - sds rendered = sdsempty(); + lwCanvas *canvas = lwCreateCanvas(cols,rows); + generateSkyline(canvas); + sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, "\nDedicated to the 8 bit game developers of the past. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); sdsfree(rendered); - // lwFreeCanvas(canvas); + lwFreeCanvas(canvas); } From bea0384f5d616dd44b5202ccb2c5345f4a671d13 Mon Sep 17 00:00:00 2001 From: charsyam Date: Mon, 7 Oct 2019 23:48:11 +0900 Subject: [PATCH 105/116] fix type salves to slaves --- src/replication.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/replication.c b/src/replication.c index 76090a9e..f10a9167 100644 --- a/src/replication.c +++ b/src/replication.c @@ -585,7 +585,7 @@ int startBgsaveForReplication(int mincapa) { } /* If we failed to BGSAVE, remove the slaves waiting for a full - * resynchorinization from the list of salves, inform them with + * resynchorinization from the list of slaves, inform them with * an error about what happened, close the connection ASAP. */ if (retval == C_ERR) { serverLog(LL_WARNING,"BGSAVE for replication failed"); @@ -606,7 +606,7 @@ int startBgsaveForReplication(int mincapa) { } /* If the target is socket, rdbSaveToSlavesSockets() already setup - * the salves for a full resync. Otherwise for disk target do it now.*/ + * the slaves for a full resync. Otherwise for disk target do it now.*/ if (!socket_target) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { From c10889150ef10a30edb2ba0503b06154222458df Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 18:24:34 +0200 Subject: [PATCH 106/116] LOLWUT: version 6 initial output. May change a bit. --- src/lolwut.c | 4 ++-- src/lolwut.h | 2 +- src/lolwut5.c | 2 +- src/lolwut6.c | 38 +++++++++++++++++++++++++++++--------- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index 7e2ceca7..0e1552ba 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -90,12 +90,12 @@ void lolwutCommand(client *c) { * canvas implementation that can be reused. */ /* Allocate and return a new canvas of the specified size. */ -lwCanvas *lwCreateCanvas(int width, int height) { +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) { lwCanvas *canvas = zmalloc(sizeof(*canvas)); canvas->width = width; canvas->height = height; canvas->pixels = zmalloc(width*height); - memset(canvas->pixels,0,width*height); + memset(canvas->pixels,bgcolor,width*height); return canvas; } diff --git a/src/lolwut.h b/src/lolwut.h index c049ac90..38c0de42 100644 --- a/src/lolwut.h +++ b/src/lolwut.h @@ -41,7 +41,7 @@ typedef struct lwCanvas { } lwCanvas; /* Drawing functions implemented inside lolwut.c. */ -lwCanvas *lwCreateCanvas(int width, int height); +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor); void lwFreeCanvas(lwCanvas *canvas); void lwDrawPixel(lwCanvas *canvas, int x, int y, int color); int lwGetPixel(lwCanvas *canvas, int x, int y); diff --git a/src/lolwut5.c b/src/lolwut5.c index 0200fece..5a934880 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -74,7 +74,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ int padding = canvas_width > 4 ? 2 : 0; float square_side = (float)(canvas_width-padding*2) / squares_per_row; int canvas_height = square_side * squares_per_col + padding*2; - lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height); + lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height, 0); for (int y = 0; y < squares_per_col; y++) { for (int x = 0; x < squares_per_row; x++) { diff --git a/src/lolwut6.c b/src/lolwut6.c index 8cc8d6e0..44c93362 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -114,14 +114,34 @@ void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { /* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */ void generateSkyline(lwCanvas *canvas) { - struct skyscraper si = { - .xoff = 4, - .width = 13, - .height = 15, - .windows = 1, - .color = 1 - }; - generateSkyscraper(canvas, &si); + struct skyscraper si; + + /* First draw the background skyscraper without windows, using the + * two different grays. */ + si.color = 1; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 10 + rand()%9; + si.height = canvas->height/2 + rand()%canvas->height/2; + si.windows = 0; + si.color = si.color == 1 ? 2 : 1; + generateSkyscraper(canvas, &si); + offset += si.width/2; + } + + /* Now draw the foreground skyscraper with the windows. */ + si.color = 0; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 5 + rand()%14; + if (si.width % 4) si.width += (si.width % 3); + si.height = canvas->height/2 + rand()%canvas->height/2; + si.windows = 1; + generateSkyscraper(canvas, &si); + offset += si.width+1; + } } /* The LOLWUT 6 command: @@ -152,7 +172,7 @@ void lolwut6Command(client *c) { if (rows > 1000) rows = 1000; /* Generate the city skyline and reply. */ - lwCanvas *canvas = lwCreateCanvas(cols,rows); + lwCanvas *canvas = lwCreateCanvas(cols,rows,3); generateSkyline(canvas); sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, From 58cdc054e30e889299fba6e25b780377af3fca9d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 18:41:29 +0200 Subject: [PATCH 107/116] LOLWUT: version 6: improve rithm of the image. --- src/lolwut6.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/lolwut6.c b/src/lolwut6.c index 44c93362..045f06fe 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -62,8 +62,8 @@ static sds renderCanvas(lwCanvas *canvas) { } /* Draw a skyscraper on the canvas, according to the parameters in the - * 'skyscraper' structure. Window colors are random and are always selected - * to be different than the color of the skyscraper itsefl. */ + * 'skyscraper' structure. Window colors are random and are always one + * of the two grays. */ struct skyscraper { int xoff; /* X offset. */ int width; /* Pixels width. */ @@ -100,7 +100,7 @@ void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { * (characters) are not square. */ if (relx/2 % 2 && rely % 2) { do { - color = rand() % 4; + color = 1 + rand() % 2; } while (color == si->color); /* Except we want adjacent pixels creating the same * window to be the same color. */ @@ -117,17 +117,25 @@ void generateSkyline(lwCanvas *canvas) { struct skyscraper si; /* First draw the background skyscraper without windows, using the - * two different grays. */ - si.color = 1; - for (int offset = -10; offset < canvas->width;) { - offset += rand() % 8; - si.xoff = offset; - si.width = 10 + rand()%9; - si.height = canvas->height/2 + rand()%canvas->height/2; - si.windows = 0; - si.color = si.color == 1 ? 2 : 1; - generateSkyscraper(canvas, &si); - offset += si.width/2; + * two different grays. We use two passes to make sure that the lighter + * ones are always in the background. */ + for (int color = 2; color >= 1; color--) { + si.color = color; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 10 + rand()%9; + if (color == 2) + si.height = canvas->height/2 + rand()%canvas->height/2; + else + si.height = canvas->height/2 + rand()%canvas->height/3; + si.windows = 0; + generateSkyscraper(canvas, &si); + if (color == 2) + offset += si.width/2; + else + offset += si.width+1; + } } /* Now draw the foreground skyscraper with the windows. */ @@ -137,10 +145,10 @@ void generateSkyline(lwCanvas *canvas) { si.xoff = offset; si.width = 5 + rand()%14; if (si.width % 4) si.width += (si.width % 3); - si.height = canvas->height/2 + rand()%canvas->height/2; + si.height = canvas->height/3 + rand()%canvas->height/2; si.windows = 1; generateSkyscraper(canvas, &si); - offset += si.width+1; + offset += si.width+5; } } From c448f6fe55500af0d0f726f43892a7acaef4a1eb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 18:57:25 +0200 Subject: [PATCH 108/116] LOLWUT: version 6: give credits. --- src/lolwut6.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lolwut6.c b/src/lolwut6.c index 045f06fe..46f76cc2 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -31,6 +31,11 @@ * This file implements the LOLWUT command. The command should do something * fun and interesting, and should be replaced by a new implementation at * each new version of Redis. + * + * Thanks to the Shhh computer art collective for the help in tuning the + * output to have a better artistic effect. Also thanks to Michele Hiki Falcone + * for sending me the original gameboy image that this LOLWUT makes + * generative. */ #include "server.h" From c65347ab17d61a4118efdc4a3568bf71b088ab63 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 19:19:38 +0200 Subject: [PATCH 109/116] LOLWUT: version 6: change text message & credits. --- src/lolwut6.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lolwut6.c b/src/lolwut6.c index 46f76cc2..b76d8069 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -32,10 +32,11 @@ * fun and interesting, and should be replaced by a new implementation at * each new version of Redis. * + * Thanks to Michele Hiki Falcone for the original image that ispired + * the image, part of his game, Plaguemon. + * * Thanks to the Shhh computer art collective for the help in tuning the - * output to have a better artistic effect. Also thanks to Michele Hiki Falcone - * for sending me the original gameboy image that this LOLWUT makes - * generative. + * output to have a better artistic effect. */ #include "server.h" @@ -189,7 +190,8 @@ void lolwut6Command(client *c) { generateSkyline(canvas); sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, - "\nDedicated to the 8 bit game developers of the past. Redis ver. "); + "\nDedicated to the 8 bit game developers of past and present.\n" + "Original 8 bit image from Plaguemon by hikikomori. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); From 3ef6b79c801cd6bf9301a3407319f0a79ac882d6 Mon Sep 17 00:00:00 2001 From: Jamison Judge Date: Mon, 7 Oct 2019 11:01:01 -0700 Subject: [PATCH 110/116] stream.h: fix typo --- src/stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stream.h b/src/stream.h index 8ae90ce7..1163b352 100644 --- a/src/stream.h +++ b/src/stream.h @@ -88,7 +88,7 @@ typedef struct streamNACK { /* Stream propagation informations, passed to functions in order to propagate * XCLAIM commands to AOF and slaves. */ -typedef struct sreamPropInfo { +typedef struct streamPropInfo { robj *keyname; robj *groupname; } streamPropInfo; From 009862ab7eee394061eeaf1aa187fa339ca50be4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 8 Oct 2019 17:02:52 +0200 Subject: [PATCH 111/116] Geo: output 10 chars of geohash, not 11. This does not limit the actual precision, because the last digit bits were garbage, and the shift value became even negative in the last iteration. --- src/geo.c | 8 ++++---- tests/unit/geo.tcl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/geo.c b/src/geo.c index f47f4ee2..049335a4 100644 --- a/src/geo.c +++ b/src/geo.c @@ -734,14 +734,14 @@ void geohashCommand(client *c) { r[1].max = 90; geohashEncode(&r[0],&r[1],xy[0],xy[1],26,&hash); - char buf[12]; + char buf[11]; int i; - for (i = 0; i < 11; i++) { + for (i = 0; i < 10; i++) { int idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; buf[i] = geoalphabet[idx]; } - buf[11] = '\0'; - addReplyBulkCBuffer(c,buf,11); + buf[10] = '\0'; + addReplyBulkCBuffer(c,buf,10); } } } diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl index 49e421ee..76b9bda3 100644 --- a/tests/unit/geo.tcl +++ b/tests/unit/geo.tcl @@ -129,7 +129,7 @@ start_server {tags {"geo"}} { r del points r geoadd points -5.6 42.6 test lindex [r geohash points test] 0 - } {ezs42e44yx0} + } {ezs42e44yx} test {GEOPOS simple} { r del points From 1a292e06342963ab1f82adb70637461466264a89 Mon Sep 17 00:00:00 2001 From: omg-by <504094596@qq.com> Date: Thu, 10 Oct 2019 00:43:28 +0800 Subject: [PATCH 112/116] there should is AUTH && HELLO non authenticated state. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index e089865f..a9b6b650 100644 --- a/src/server.c +++ b/src/server.c @@ -3366,7 +3366,7 @@ int processCommand(client *c) { !c->authenticated; if (auth_required) { /* AUTH and HELLO are valid even in non authenticated state. */ - if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) { + if (c->cmd->proc != authCommand && c->cmd->proc != helloCommand) { flagTransaction(c); addReply(c,shared.noautherr); return C_OK; From 98600c9a11419991cdc3fa11acff9100d793f220 Mon Sep 17 00:00:00 2001 From: Daniel Dai <764122422@qq.com> Date: Wed, 9 Oct 2019 14:15:31 -0400 Subject: [PATCH 113/116] update typo --- tests/integration/replication.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 5d32555b..1c18582c 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -319,7 +319,7 @@ start_server {tags {"repl"}} { } } -test {slave fails full sync and diskless load swapdb recoveres it} { +test {slave fails full sync and diskless load swapdb recovers it} { start_server {tags {"repl"}} { set slave [srv 0 client] set slave_host [srv 0 host] From 2fae0192e86c4222d8c735fda8a1f35164e1ca92 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 10 Oct 2019 09:47:48 +0200 Subject: [PATCH 114/116] Fix usage of server.stream_node_max_* --- src/t_stream.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 9e7d3d12..ea9a620f 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -242,17 +242,17 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ * the current node is full. */ if (lp != NULL) { if (server.stream_node_max_bytes && - lp_bytes > server.stream_node_max_bytes) + lp_bytes >= server.stream_node_max_bytes) { lp = NULL; } else if (server.stream_node_max_entries) { int64_t count = lpGetInteger(lpFirst(lp)); - if (count > server.stream_node_max_entries) lp = NULL; + if (count >= server.stream_node_max_entries) lp = NULL; } } int flags = STREAM_ITEM_FLAG_NONE; - if (lp == NULL || lp_bytes > server.stream_node_max_bytes) { + if (lp == NULL || lp_bytes >= server.stream_node_max_bytes) { master_id = id; streamEncodeID(rax_key,&id); /* Create the listpack having the master entry ID and fields. */ From 747be463d2f825c1e0b620ef2b120a0695121d8a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Oct 2019 10:23:34 +0200 Subject: [PATCH 115/116] Cluster: fix memory leak of cached master. This is what happened: 1. Instance starts, is a slave in the cluster configuration, but actually server.masterhost is not set, so technically the instance is acting like a master. 2. loadDataFromDisk() calls replicationCacheMasterUsingMyself() even if the instance is a master, in the case it is logically a slave and the cluster is enabled. So now we have a cached master even if the instance is practically configured as a master (from the POV of server.masterhost value and so forth). 3. clusterCron() sees that the instance requires to replicate from its master, because logically it is a slave, so it calls replicationSetMaster() that will in turn call replicationCacheMasterUsingMyself(): before this commit, this call would overwrite the old cached master, creating a memory leak. --- src/replication.c | 5 ++++- src/server.c | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index 76090a9e..08121276 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2182,7 +2182,10 @@ void replicationSetMaster(char *ip, int port) { cancelReplicationHandshake(); /* Before destroying our master state, create a cached master using * our own parameters, to later PSYNC with the new master. */ - if (was_master) replicationCacheMasterUsingMyself(); + if (was_master) { + replicationDiscardCachedMaster(); + replicationCacheMasterUsingMyself(); + } server.repl_state = REPL_STATE_CONNECT; } diff --git a/src/server.c b/src/server.c index e089865f..9392ffb8 100644 --- a/src/server.c +++ b/src/server.c @@ -4719,12 +4719,14 @@ void loadDataFromDisk(void) { (float)(ustime()-start)/1000000); /* Restore the replication ID / offset from the RDB file. */ - if ((server.masterhost || (server.cluster_enabled && nodeIsSlave(server.cluster->myself)))&& + if ((server.masterhost || + (server.cluster_enabled && + nodeIsSlave(server.cluster->myself))) && rsi.repl_id_is_set && rsi.repl_offset != -1 && /* Note that older implementations may save a repl_stream_db - * of -1 inside the RDB file in a wrong way, see more information - * in function rdbPopulateSaveInfo. */ + * of -1 inside the RDB file in a wrong way, see more + * information in function rdbPopulateSaveInfo. */ rsi.repl_stream_db != -1) { memcpy(server.replid,rsi.repl_id,sizeof(server.replid)); From dd29d441364236992ce89230556526c707c3c960 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Oct 2019 15:22:42 +0200 Subject: [PATCH 116/116] Test: fix implementation-dependent test after code change. --- tests/unit/type/stream.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index ae6c2d7b..a7415ae8 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -355,12 +355,12 @@ start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries r XADD mystream * xitem v } r XTRIM mystream MAXLEN ~ 85 - assert {[r xlen mystream] == 89} + assert {[r xlen mystream] == 90} r config set stream-node-max-entries 1 r debug loadaof r XADD mystream * xitem v incr j - assert {[r xlen mystream] == 90} + assert {[r xlen mystream] == 91} } }