From ec511fa709f3c8fc74d26ca52114dcb9add754cc Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 9 Oct 2018 13:11:02 +0800 Subject: [PATCH 1/5] Streams: add a new command XTREAM XSTREAM CREATE -- Create a new empty stream. XSTREAM SETID -- Set the current stream ID. --- src/server.c | 1 + src/server.h | 1 + src/t_stream.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/src/server.c b/src/server.c index 78aee5db..7f9b80ff 100644 --- a/src/server.c +++ b/src/server.c @@ -314,6 +314,7 @@ struct redisCommand redisCommandTable[] = { {"xread",xreadCommand,-4,"rs",0,xreadGetKeys,1,1,1,0,0}, {"xreadgroup",xreadCommand,-7,"ws",0,xreadGetKeys,1,1,1,0,0}, {"xgroup",xgroupCommand,-2,"wm",0,NULL,2,2,1,0,0}, + {"xstream",xstreamCommand,-2,"wmFR",0,NULL,2,2,1,0,0}, {"xack",xackCommand,-4,"wF",0,NULL,1,1,1,0,0}, {"xpending",xpendingCommand,-3,"r",0,NULL,1,1,1,0,0}, {"xclaim",xclaimCommand,-6,"wF",0,NULL,1,1,1,0,0}, diff --git a/src/server.h b/src/server.h index 4c4c0ce5..4c7be2fe 100644 --- a/src/server.h +++ b/src/server.h @@ -2107,6 +2107,7 @@ void xrevrangeCommand(client *c); void xlenCommand(client *c); void xreadCommand(client *c); void xgroupCommand(client *c); +void xstreamCommand(client *c); void xackCommand(client *c); void xpendingCommand(client *c); void xclaimCommand(client *c); diff --git a/src/t_stream.c b/src/t_stream.c index 4387e08a..f94f0f60 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1746,6 +1746,71 @@ NULL } } +/* XSTREAM CREATE + * XSTREAM SETID */ +void xstreamCommand(client *c) { + const char *help[] = { +"CREATE -- Create a new empty stream.", +"SETID -- Set the current stream ID.", +"HELP -- Prints this help.", +NULL + }; + stream *s = NULL; + char *opt = c->argv[1]->ptr; /* Subcommand name. */ + + /* Dispatch the different subcommands. */ + if (!strcasecmp(opt,"CREATE") && c->argc == 4) { + robj *o = lookupKeyWrite(c->db,c->argv[2]); + if (o) { + addReplySds(c, + sdsnew("-BUSYSTREAM Stream already exists\r\n")); + return; + } else { + streamID id; + if (!strcmp(c->argv[3]->ptr,"*")) { + id.ms = mstime(); + id.seq = 0; + } else if (streamParseStrictIDOrReply(c,c->argv[3],&id,0) != C_OK) { + return; + } + + o = createStreamObject(); + s = o->ptr; + s->last_id = id; + dbAdd(c->db,c->argv[2],o); + + addReply(c,shared.ok); + server.dirty++; + notifyKeyspaceEvent(NOTIFY_STREAM,"xstream-create", + c->argv[2],c->db->id); + } + } else if (!strcasecmp(opt,"SETID") && c->argc == 4) { + robj *o = lookupKeyWriteOrReply(c,c->argv[2],shared.nokeyerr); + if (o == NULL || checkType(c,o,OBJ_STREAM)) return; + s = o->ptr; + streamID id; + if (!strcmp(c->argv[3]->ptr,"$")) { + id = s->last_id; + } else if (streamParseStrictIDOrReply(c,c->argv[3],&id,0) != C_OK) { + return; + } + if (streamCompareID(&id,&s->last_id) < 0) { + addReplyError(c,"The ID specified in XSTREAM SETID is smaller than the " + "target stream top item"); + return; + } + s->last_id = id; + addReply(c,shared.ok); + server.dirty++; + notifyKeyspaceEvent(NOTIFY_STREAM,"xstream-setid", + c->argv[2],c->db->id); + } else if (!strcasecmp(opt,"HELP")) { + addReplyHelp(c, help); + } else { + addReplySubcommandSyntaxError(c); + } +} + /* XACK ... * * Acknowledge a message as processed. In practical terms we just check the From 5f3adbee33d555d436b03d19f32ce903da36252b Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 9 Oct 2018 15:21:08 +0800 Subject: [PATCH 2/5] Streams: add tests for XSTREAM command --- tests/unit/type/stream.tcl | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 2b69a2e9..769ab571 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -363,3 +363,42 @@ start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries assert {[r xlen mystream] == 90} } } + +start_server {tags {"xstream command"}} { + test {XSTREAM can CREATE an empty stream} { + r XSTREAM CREATE mystream * + assert {[dict get [r xinfo stream mystream] length] == 0} + } + + test {XSTREAM cannot CREATE on an busy stream} { + catch {r XSTREAM CREATE mystream *} err + set _ $err + } {BUSY*} + + test {XSTREAM can CREATE an empty stream with specific ID} { + r del mystream + r XSTREAM CREATE mystream "100-100" + assert {[dict get [r xinfo stream mystream] length] == 0} + assert {[dict get [r xinfo stream mystream] last-generated-id] == "100-100"} + } + + test {XSTREAM can SETID with $} { + r XSTREAM SETID mystream $ + assert {[dict get [r xinfo stream mystream] last-generated-id] == "100-100"} + } + + test {XSTREAM can SETID with specific ID} { + r XSTREAM SETID mystream "200-0" + assert {[dict get [r xinfo stream mystream] last-generated-id] == "200-0"} + } + + test {XSTREAM cannot SETID with smaller ID} { + catch {r XSTREAM SETID mystream "0-0"} err + set _ $err + } {ERR*smaller*} + + test {XSTREAM cannot SETID on non-existent key} { + catch {r XSTREAM SETID stream $} err + set _ $err + } {ERR no such key} +} From b3e80d2f654a66358c53addffd34945363cce2bb Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 8 Oct 2018 21:23:38 +0800 Subject: [PATCH 3/5] Stream & AOF: rewrite stream in correct way --- src/aof.c | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/aof.c b/src/aof.c index f8f26bdf..3f914b77 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1121,23 +1121,39 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { streamID id; int64_t numfields; - /* Reconstruct the stream data using XADD commands. */ - while(streamIteratorGetID(&si,&id,&numfields)) { - /* Emit a two elements array for each item. The first is - * the ID, the second is an array of field-value pairs. */ + if (s->length) { + /* Reconstruct the stream data using XADD commands. */ + while(streamIteratorGetID(&si,&id,&numfields)) { + /* Emit a two elements array for each item. The first is + * the ID, the second is an array of field-value pairs. */ - /* Emit the XADD ...fields... command. */ - if (rioWriteBulkCount(r,'*',3+numfields*2) == 0) return 0; - if (rioWriteBulkString(r,"XADD",4) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; - if (rioWriteBulkStreamID(r,&id) == 0) return 0; - while(numfields--) { - unsigned char *field, *value; - int64_t field_len, value_len; - streamIteratorGetField(&si,&field,&value,&field_len,&value_len); - if (rioWriteBulkString(r,(char*)field,field_len) == 0) return 0; - if (rioWriteBulkString(r,(char*)value,value_len) == 0) return 0; + /* Emit the XADD ...fields... command. */ + if (rioWriteBulkCount(r,'*',3+numfields*2) == 0) return 0; + if (rioWriteBulkString(r,"XADD",4) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkStreamID(r,&id) == 0) return 0; + while(numfields--) { + unsigned char *field, *value; + int64_t field_len, value_len; + streamIteratorGetField(&si,&field,&value,&field_len,&value_len); + if (rioWriteBulkString(r,(char*)field,field_len) == 0) return 0; + if (rioWriteBulkString(r,(char*)value,value_len) == 0) return 0; + } } + /* Append XSTREAM SETID after XADD, make sure lastid is correct, + * in case of XDEL lastid. */ + if (rioWriteBulkCount(r,'*',4) == 0) return 0; + if (rioWriteBulkString(r,"XSTREAM",7) == 0) return 0; + if (rioWriteBulkString(r,"SETID",5) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; + } else { + /* Using XSTREAM CREATE if the stream is empty. */ + if (rioWriteBulkCount(r,'*',4) == 0) return 0; + if (rioWriteBulkString(r,"XSTREAM",7) == 0) return 0; + if (rioWriteBulkString(r,"CREATE",6) == 0) return 0; + if (rioWriteBulkObject(r,key) == 0) return 0; + if (rioWriteBulkStreamID(r,&s->last_id) == 0) return 0; } /* Create all the stream consumer groups. */ From 3094eb36263e602731840ededd301ff21e7981d4 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 9 Oct 2018 15:45:58 +0800 Subject: [PATCH 4/5] Streams: add tests for aof rewrite --- tests/unit/type/stream.tcl | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 769ab571..9acf11c7 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -402,3 +402,26 @@ start_server {tags {"xstream command"}} { set _ $err } {ERR no such key} } + +start_server {tags {"stream"} overrides {appendonly yes aof-use-rdb-preamble no}} { + test {Empty stream can be rewrite into AOF correctly} { + r XSTREAM CREATE mystream 0 + assert {[dict get [r xinfo stream mystream] length] == 0} + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + assert {[dict get [r xinfo stream mystream] length] == 0} + } + + test {Stream can be rewrite into AOF correctly after XDEL lastid} { + r XADD mystream 1-1 a b + r XADD mystream 2-2 a b + assert {[dict get [r xinfo stream mystream] length] == 2} + r XDEL mystream 2-2 + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + assert {[dict get [r xinfo stream mystream] length] == 1} + assert {[dict get [r xinfo stream mystream] last-generated-id] == "2-2"} + } +} From 480e2994369de43e3b13019ee9995906d68dd14e Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 9 Oct 2018 16:22:30 +0800 Subject: [PATCH 5/5] Streams: rewrite id in XSTREAM CREATE * --- src/t_stream.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index f94f0f60..b20cf125 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1779,6 +1779,10 @@ NULL s->last_id = id; dbAdd(c->db,c->argv[2],o); + robj *idarg = createObjectFromStreamID(&id); + rewriteClientCommandArgument(c,3,idarg); + decrRefCount(idarg); + addReply(c,shared.ok); server.dirty++; notifyKeyspaceEvent(NOTIFY_STREAM,"xstream-create",