From f4f06efccc4064fa0f7f7f16098de746fa0e5b69 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 11:40:26 +0200 Subject: [PATCH 01/10] don't load value from VM for EXISTS --- TODO | 1 - redis.c | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/TODO b/TODO index bdbe7974..331c560a 100644 --- a/TODO +++ b/TODO @@ -15,7 +15,6 @@ Virtual Memory sub-TODO: * Check if the page selection algorithm is working well * Divide swappability of objects by refcount * Use multiple open FDs against the VM file, one for thread. -* EXISTS should avoid loading the object if possible without making the code too specialized. * vm-min-age option * Make sure objects loaded from the VM are specially encoded when possible. * Check what happens performance-wise if instead to create threads again and again the same threads are reused forever. Note: this requires a way to disable this clients in the child, but waiting for empty new jobs queue can be enough. diff --git a/redis.c b/redis.c index c22a1c8c..19f431cf 100644 --- a/redis.c +++ b/redis.c @@ -4343,7 +4343,12 @@ static void delCommand(redisClient *c) { } static void existsCommand(redisClient *c) { - addReply(c,lookupKeyRead(c->db,c->argv[1]) ? shared.cone : shared.czero); + expireIfNeeded(c->db,c->argv[1]); + if (dictFind(c->db->dict,c->argv[1])) { + addReply(c, shared.cone); + } else { + addReply(c, shared.czero); + } } static void selectCommand(redisClient *c) { From f3b52411db839f4716bdbd7ac9f90331c94f0ccd Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 11:55:12 +0200 Subject: [PATCH 02/10] make append only filename configurable --- redis.c | 3 +++ redis.conf | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/redis.c b/redis.c index 19f431cf..7e27e0fb 100644 --- a/redis.c +++ b/redis.c @@ -1886,6 +1886,9 @@ static void loadServerConfig(char *filename) { if ((server.appendonly = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { + zfree(server.appendfilename); + server.appendfilename = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"appendfsync") && argc == 2) { if (!strcasecmp(argv[1],"no")) { server.appendfsync = APPENDFSYNC_NO; diff --git a/redis.conf b/redis.conf index c946a80e..4d3689ed 100644 --- a/redis.conf +++ b/redis.conf @@ -159,13 +159,14 @@ dir ./ # Still if append only mode is enabled Redis will load the data from the # log file at startup ignoring the dump.rdb file. # -# The name of the append only file is "appendonly.aof" -# # IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append # log file in background when it gets too big. appendonly no +# The name of the append only file (default: "appendonly.aof") +# appendfilename appendonly.aof + # The fsync() call tells the Operating System to actually write data on disk # instead to wait for more data in the output buffer. Some OS will really flush # data on disk, some other OS will just try to do it ASAP. From 6f07874621392f7f0eb5f092f79f1afe8939b9c3 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 12:45:27 +0200 Subject: [PATCH 03/10] extract preloading of multiple keys according to the command prototype to a separate function --- redis.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/redis.c b/redis.c index 7e27e0fb..e342416f 100644 --- a/redis.c +++ b/redis.c @@ -9548,6 +9548,19 @@ static int waitForSwappedKey(redisClient *c, robj *key) { return 1; } +/* Preload keys for any command with first, last and step values for + * the command keys prototype, as defined in the command table. */ +static void waitForMultipleSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv) { + int j, last; + if (cmd->vm_firstkey == 0) return; + last = cmd->vm_lastkey; + if (last < 0) last = argc+last; + for (j = cmd->vm_firstkey; j <= last; j += cmd->vm_keystep) { + redisAssert(j < argc); + waitForSwappedKey(c,argv[j]); + } +} + /* Preload keys needed for the ZUNION and ZINTER commands. */ static void zunionInterBlockClientOnSwappedKeys(redisClient *c) { int i, num; @@ -9568,16 +9581,10 @@ static void zunionInterBlockClientOnSwappedKeys(redisClient *c) { * Return 1 if the client is marked as blocked, 0 if the client can * continue as the keys it is going to access appear to be in memory. */ static int blockClientOnSwappedKeys(struct redisCommand *cmd, redisClient *c) { - int j, last; - if (cmd->vm_preload_proc != NULL) { cmd->vm_preload_proc(c); } else { - if (cmd->vm_firstkey == 0) return 0; - last = cmd->vm_lastkey; - if (last < 0) last = c->argc+last; - for (j = cmd->vm_firstkey; j <= last; j += cmd->vm_keystep) - waitForSwappedKey(c,c->argv[j]); + waitForMultipleSwappedKeys(c,cmd,c->argc,c->argv); } /* If the client was blocked for at least one key, mark it as blocked. */ From ca1788b560a0e70047f1bcef354a3b6317b4360d Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 12:58:44 +0200 Subject: [PATCH 04/10] make prototype of custom function to preload keys from the vm match the prototype of waitForMultipleSwappedKeys --- redis.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/redis.c b/redis.c index e342416f..55a2ac21 100644 --- a/redis.c +++ b/redis.c @@ -450,6 +450,7 @@ typedef struct pubsubPattern { } pubsubPattern; typedef void redisCommandProc(redisClient *c); +typedef void redisVmPreloadProc(redisClient *c, struct redisCommand *cmd, int argc, robj **argv); struct redisCommand { char *name; redisCommandProc *proc; @@ -458,7 +459,7 @@ struct redisCommand { /* Use a function to determine which keys need to be loaded * in the background prior to executing this command. Takes precedence * over vm_firstkey and others, ignored when NULL */ - redisCommandProc *vm_preload_proc; + redisVmPreloadProc *vm_preload_proc; /* What keys should be loaded in background when calling this command? */ int vm_firstkey; /* The first argument that's a key (0 = no keys) */ int vm_lastkey; /* THe last argument that's a key */ @@ -608,7 +609,7 @@ static robj *vmReadObjectFromSwap(off_t page, int type); static void waitEmptyIOJobsQueue(void); static void vmReopenSwapFile(void); static int vmFreePage(off_t page); -static void zunionInterBlockClientOnSwappedKeys(redisClient *c); +static void zunionInterBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv); static int blockClientOnSwappedKeys(struct redisCommand *cmd, redisClient *c); static int dontWaitForSwappedKey(redisClient *c, robj *key); static void handleClientsBlockedOnSwappedKey(redisDb *db, robj *key); @@ -9562,11 +9563,14 @@ static void waitForMultipleSwappedKeys(redisClient *c, struct redisCommand *cmd, } /* Preload keys needed for the ZUNION and ZINTER commands. */ -static void zunionInterBlockClientOnSwappedKeys(redisClient *c) { +static void zunionInterBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv) { int i, num; - num = atoi(c->argv[2]->ptr); + REDIS_NOTUSED(cmd); + REDIS_NOTUSED(argc); + + num = atoi(argv[2]->ptr); for (i = 0; i < num; i++) { - waitForSwappedKey(c,c->argv[3+i]); + waitForSwappedKey(c,argv[3+i]); } } @@ -9582,7 +9586,7 @@ static void zunionInterBlockClientOnSwappedKeys(redisClient *c) { * continue as the keys it is going to access appear to be in memory. */ static int blockClientOnSwappedKeys(struct redisCommand *cmd, redisClient *c) { if (cmd->vm_preload_proc != NULL) { - cmd->vm_preload_proc(c); + cmd->vm_preload_proc(c,cmd,c->argc,c->argv); } else { waitForMultipleSwappedKeys(c,cmd,c->argc,c->argv); } From 739ba0d21119778438c36ea60fdb263271783468 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 12:59:34 +0200 Subject: [PATCH 05/10] add sanity check to zunionInterBlockClientOnSwappedKeys, as the number of keys used is provided as argument to the function --- redis.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/redis.c b/redis.c index 55a2ac21..46f97273 100644 --- a/redis.c +++ b/redis.c @@ -9562,13 +9562,15 @@ static void waitForMultipleSwappedKeys(redisClient *c, struct redisCommand *cmd, } } -/* Preload keys needed for the ZUNION and ZINTER commands. */ +/* Preload keys needed for the ZUNION and ZINTER commands. + * Note that the number of keys to preload is user-defined, so we need to + * apply a sanity check against argc. */ static void zunionInterBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv) { int i, num; REDIS_NOTUSED(cmd); - REDIS_NOTUSED(argc); num = atoi(argv[2]->ptr); + if (num > (argc-3)) return; for (i = 0; i < num; i++) { waitForSwappedKey(c,argv[3+i]); } From 3805e04f788a27938095b3d79d987210140283a9 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 14:17:10 +0200 Subject: [PATCH 06/10] added function that preloads all keys needed to execute a MULTI/EXEC block --- redis.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/redis.c b/redis.c index 46f97273..f81db41c 100644 --- a/redis.c +++ b/redis.c @@ -610,6 +610,7 @@ static void waitEmptyIOJobsQueue(void); static void vmReopenSwapFile(void); static int vmFreePage(off_t page); static void zunionInterBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv); +static void execBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv); static int blockClientOnSwappedKeys(struct redisCommand *cmd, redisClient *c); static int dontWaitForSwappedKey(redisClient *c, robj *key); static void handleClientsBlockedOnSwappedKey(redisDb *db, robj *key); @@ -826,7 +827,7 @@ static struct redisCommand cmdTable[] = { {"lastsave",lastsaveCommand,1,REDIS_CMD_INLINE,NULL,0,0,0}, {"type",typeCommand,2,REDIS_CMD_INLINE,NULL,1,1,1}, {"multi",multiCommand,1,REDIS_CMD_INLINE,NULL,0,0,0}, - {"exec",execCommand,1,REDIS_CMD_INLINE|REDIS_CMD_DENYOOM,NULL,0,0,0}, + {"exec",execCommand,1,REDIS_CMD_INLINE|REDIS_CMD_DENYOOM,execBlockClientOnSwappedKeys,0,0,0}, {"discard",discardCommand,1,REDIS_CMD_INLINE,NULL,0,0,0}, {"sync",syncCommand,1,REDIS_CMD_INLINE,NULL,0,0,0}, {"flushdb",flushdbCommand,1,REDIS_CMD_INLINE,NULL,0,0,0}, @@ -9576,6 +9577,32 @@ static void zunionInterBlockClientOnSwappedKeys(redisClient *c, struct redisComm } } +/* Preload keys needed to execute the entire MULTI/EXEC block. + * + * This function is called by blockClientOnSwappedKeys when EXEC is issued, + * and will block the client when any command requires a swapped out value. */ +static void execBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv) { + int i, margc; + struct redisCommand *mcmd; + robj **margv; + REDIS_NOTUSED(cmd); + REDIS_NOTUSED(argc); + REDIS_NOTUSED(argv); + + if (!(c->flags & REDIS_MULTI)) return; + for (i = 0; i < c->mstate.count; i++) { + mcmd = c->mstate.commands[i].cmd; + margc = c->mstate.commands[i].argc; + margv = c->mstate.commands[i].argv; + + if (mcmd->vm_preload_proc != NULL) { + mcmd->vm_preload_proc(c,mcmd,margc,margv); + } else { + waitForMultipleSwappedKeys(c,mcmd,margc,margv); + } + } +} + /* Is this client attempting to run a command against swapped keys? * If so, block it ASAP, load the keys in background, then resume it. * From 0a6f3f0f8a42731b36a78d58c5c11728faf97a43 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 14:33:34 +0200 Subject: [PATCH 07/10] swap arguments in blockClientOnSwappedKeys to be consistent --- redis.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/redis.c b/redis.c index f81db41c..ce34ba68 100644 --- a/redis.c +++ b/redis.c @@ -611,7 +611,7 @@ static void vmReopenSwapFile(void); static int vmFreePage(off_t page); static void zunionInterBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv); static void execBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd, int argc, robj **argv); -static int blockClientOnSwappedKeys(struct redisCommand *cmd, redisClient *c); +static int blockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd); static int dontWaitForSwappedKey(redisClient *c, robj *key); static void handleClientsBlockedOnSwappedKey(redisDb *db, robj *key); static void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask); @@ -2396,7 +2396,7 @@ static int processCommand(redisClient *c) { addReply(c,shared.queued); } else { if (server.vm_enabled && server.vm_max_threads > 0 && - blockClientOnSwappedKeys(cmd,c)) return 1; + blockClientOnSwappedKeys(c,cmd)) return 1; call(c,cmd); } @@ -9613,7 +9613,7 @@ static void execBlockClientOnSwappedKeys(redisClient *c, struct redisCommand *cm * * Return 1 if the client is marked as blocked, 0 if the client can * continue as the keys it is going to access appear to be in memory. */ -static int blockClientOnSwappedKeys(struct redisCommand *cmd, redisClient *c) { +static int blockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd) { if (cmd->vm_preload_proc != NULL) { cmd->vm_preload_proc(c,cmd,c->argc,c->argv); } else { From 3317c679711e2ae82c452e783d912e5276b5f6d7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 14:36:59 +0200 Subject: [PATCH 08/10] update TODO --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index 331c560a..5e1de7a5 100644 --- a/TODO +++ b/TODO @@ -19,7 +19,6 @@ Virtual Memory sub-TODO: * Make sure objects loaded from the VM are specially encoded when possible. * Check what happens performance-wise if instead to create threads again and again the same threads are reused forever. Note: this requires a way to disable this clients in the child, but waiting for empty new jobs queue can be enough. * Sets of integers are slow to load, for a number of reasons. Fix it. (use slow_sets.rdb file for debugging). (p.s. this was now partially fixed). -* On EXEC try to block the client until relevant keys are loaded. * Hashes (GET/SET/DEL/INCRBY/EXISTS/FIELDS/LEN/MSET/MGET). Special encoding for hashes with less than N elements. * Write documentation for APPEND From 3350558346a07600bb7529781638696e04c425cc Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 7 May 2010 16:48:43 +0200 Subject: [PATCH 09/10] change command names no longer used to zunion/zinter --- redis-cli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis-cli.c b/redis-cli.c index 5410f026..98e9bdf1 100644 --- a/redis-cli.c +++ b/redis-cli.c @@ -112,8 +112,8 @@ static struct redisCommand cmdTable[] = { {"zincrby",4,CMDFLAG_NONE}, {"zrem",3,CMDFLAG_NONE}, {"zremrangebyscore",4,CMDFLAG_NONE}, - {"zmerge",-3,CMDFLAG_NONE}, - {"zmergeweighed",-4,CMDFLAG_NONE}, + {"zunion",-4,CMDFLAG_NONE}, + {"zinter",-4,CMDFLAG_NONE}, {"zrange",-4,CMDFLAG_NONE}, {"zrank",3,CMDFLAG_NONE}, {"zrevrank",3,CMDFLAG_NONE}, From 9376e434f08003678e8042855bc5c35d406b8db2 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 13 May 2010 14:15:06 +0200 Subject: [PATCH 10/10] feed SETEX as SET and EXPIREAT to AOF --- redis.c | 73 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/redis.c b/redis.c index ce34ba68..f52dbb82 100644 --- a/redis.c +++ b/redis.c @@ -8068,9 +8068,41 @@ static void flushAppendOnlyFile(void) { } } +static sds catAppendOnlyGenericCommand(sds buf, int argc, robj **argv) { + int j; + buf = sdscatprintf(buf,"*%d\r\n",argc); + for (j = 0; j < argc; j++) { + robj *o = getDecodedObject(argv[j]); + buf = sdscatprintf(buf,"$%lu\r\n",(unsigned long)sdslen(o->ptr)); + buf = sdscatlen(buf,o->ptr,sdslen(o->ptr)); + buf = sdscatlen(buf,"\r\n",2); + decrRefCount(o); + } + return buf; +} + +static sds catAppendOnlyExpireAtCommand(sds buf, robj *key, robj *seconds) { + int argc = 3; + long when; + robj *argv[3]; + + /* Make sure we can use strtol */ + seconds = getDecodedObject(seconds); + when = time(NULL)+strtol(seconds->ptr,NULL,10); + decrRefCount(seconds); + + argv[0] = createStringObject("EXPIREAT",8); + argv[1] = key; + argv[2] = createObject(REDIS_STRING, + sdscatprintf(sdsempty(),"%ld",when)); + buf = catAppendOnlyGenericCommand(buf, argc, argv); + decrRefCount(argv[0]); + decrRefCount(argv[2]); + return buf; +} + static void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) { sds buf = sdsempty(); - int j; robj *tmpargv[3]; /* The DB this command was targetting is not the same as the last command @@ -8084,36 +8116,19 @@ static void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv server.appendseldb = dictid; } - /* "Fix" the argv vector if the command is EXPIRE. We want to translate - * EXPIREs into EXPIREATs calls */ if (cmd->proc == expireCommand) { - long when; - - tmpargv[0] = createStringObject("EXPIREAT",8); + /* Translate EXPIRE into EXPIREAT */ + buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]); + } else if (cmd->proc == setexCommand) { + /* Translate SETEX to SET and EXPIREAT */ + tmpargv[0] = createStringObject("SET",3); tmpargv[1] = argv[1]; - incrRefCount(argv[1]); - when = time(NULL)+strtol(argv[2]->ptr,NULL,10); - tmpargv[2] = createObject(REDIS_STRING, - sdscatprintf(sdsempty(),"%ld",when)); - argv = tmpargv; - } - - /* Append the actual command */ - buf = sdscatprintf(buf,"*%d\r\n",argc); - for (j = 0; j < argc; j++) { - robj *o = argv[j]; - - o = getDecodedObject(o); - buf = sdscatprintf(buf,"$%lu\r\n",(unsigned long)sdslen(o->ptr)); - buf = sdscatlen(buf,o->ptr,sdslen(o->ptr)); - buf = sdscatlen(buf,"\r\n",2); - decrRefCount(o); - } - - /* Free the objects from the modified argv for EXPIREAT */ - if (cmd->proc == expireCommand) { - for (j = 0; j < 3; j++) - decrRefCount(argv[j]); + tmpargv[2] = argv[3]; + buf = catAppendOnlyGenericCommand(buf,3,tmpargv); + decrRefCount(tmpargv[0]); + buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]); + } else { + buf = catAppendOnlyGenericCommand(buf,argc,argv); } /* Append to the AOF buffer. This will be flushed on disk just before