Merge remote branch 'pietern/unstable-zset' into unstable

This commit is contained in:
antirez 2011-04-14 13:31:14 +02:00
commit 7c0e1b53c4
13 changed files with 2199 additions and 858 deletions

View File

@ -340,6 +340,12 @@ list-max-ziplist-value 64
# set in order to use this special memory saving encoding.
set-max-intset-entries 512
# Similarly to hashes and lists, sorted sets are also specially encoded in
# order to save a lot of space. This encoding is only used when the length and
# elements of a sorted set are below the following limits:
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
# order to help rehashing the main Redis hash table (the one mapping top-level
# keys to values). The hash table implementation redis uses (see dict.c)

View File

@ -429,21 +429,55 @@ int rewriteAppendOnlyFile(char *filename) {
}
} else if (o->type == REDIS_ZSET) {
/* Emit the ZADDs needed to rebuild the sorted set */
zset *zs = o->ptr;
dictIterator *di = dictGetIterator(zs->dict);
dictEntry *de;
char cmd[]="*4\r\n$4\r\nZADD\r\n";
while((de = dictNext(di)) != NULL) {
char cmd[]="*4\r\n$4\r\nZADD\r\n";
robj *eleobj = dictGetEntryKey(de);
double *score = dictGetEntryVal(de);
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *zl = o->ptr;
unsigned char *eptr, *sptr;
unsigned char *vstr;
unsigned int vlen;
long long vll;
double score;
if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
if (fwriteBulkObject(fp,&key) == 0) goto werr;
if (fwriteBulkDouble(fp,*score) == 0) goto werr;
if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
eptr = ziplistIndex(zl,0);
redisAssert(eptr != NULL);
sptr = ziplistNext(zl,eptr);
redisAssert(sptr != NULL);
while (eptr != NULL) {
redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
score = zzlGetScore(sptr);
if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
if (fwriteBulkObject(fp,&key) == 0) goto werr;
if (fwriteBulkDouble(fp,score) == 0) goto werr;
if (vstr != NULL) {
if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
goto werr;
} else {
if (fwriteBulkLongLong(fp,vll) == 0)
goto werr;
}
zzlNext(zl,&eptr,&sptr);
}
} else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
zset *zs = o->ptr;
dictIterator *di = dictGetIterator(zs->dict);
dictEntry *de;
while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetEntryKey(de);
double *score = dictGetEntryVal(de);
if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
if (fwriteBulkObject(fp,&key) == 0) goto werr;
if (fwriteBulkDouble(fp,*score) == 0) goto werr;
if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
}
dictReleaseIterator(di);
} else {
redisPanic("Unknown sorted set encoding");
}
dictReleaseIterator(di);
} else if (o->type == REDIS_HASH) {
char cmd[]="*4\r\n$4\r\nHSET\r\n";

View File

@ -261,6 +261,10 @@ void loadServerConfig(char *filename) {
server.list_max_ziplist_value = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) {
server.set_max_intset_entries = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) {
server.zset_max_ziplist_entries = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) {
server.zset_max_ziplist_value = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"rename-command") && argc == 3) {
struct redisCommand *cmd = lookupCommand(argv[1]);
int retval;
@ -450,6 +454,12 @@ void configSetCommand(redisClient *c) {
} else if (!strcasecmp(c->argv[2]->ptr,"set-max-intset-entries")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.set_max_intset_entries = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"zset-max-ziplist-entries")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.zset_max_ziplist_entries = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"zset-max-ziplist-value")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.zset_max_ziplist_value = ll;
} else {
addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s",
(char*)c->argv[2]->ptr);
@ -601,6 +611,16 @@ void configGetCommand(redisClient *c) {
addReplyBulkLongLong(c,server.set_max_intset_entries);
matches++;
}
if (stringmatch(pattern,"zset-max-ziplist-entries",0)) {
addReplyBulkCString(c,"zset-max-ziplist-entries");
addReplyBulkLongLong(c,server.zset_max_ziplist_entries);
matches++;
}
if (stringmatch(pattern,"zset-max-ziplist-value",0)) {
addReplyBulkCString(c,"zset-max-ziplist-value");
addReplyBulkLongLong(c,server.zset_max_ziplist_value);
matches++;
}
setDeferredMultiBulkLength(c,replylen,matches*2);
}

View File

@ -127,22 +127,57 @@ void computeDatasetDigest(unsigned char *final) {
}
setTypeReleaseIterator(si);
} else if (o->type == REDIS_ZSET) {
zset *zs = o->ptr;
dictIterator *di = dictGetIterator(zs->dict);
dictEntry *de;
unsigned char eledigest[20];
while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetEntryKey(de);
double *score = dictGetEntryVal(de);
unsigned char eledigest[20];
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *zl = o->ptr;
unsigned char *eptr, *sptr;
unsigned char *vstr;
unsigned int vlen;
long long vll;
double score;
snprintf(buf,sizeof(buf),"%.17g",*score);
memset(eledigest,0,20);
mixObjectDigest(eledigest,eleobj);
mixDigest(eledigest,buf,strlen(buf));
xorDigest(digest,eledigest,20);
eptr = ziplistIndex(zl,0);
redisAssert(eptr != NULL);
sptr = ziplistNext(zl,eptr);
redisAssert(sptr != NULL);
while (eptr != NULL) {
redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
score = zzlGetScore(sptr);
memset(eledigest,0,20);
if (vstr != NULL) {
mixDigest(eledigest,vstr,vlen);
} else {
ll2string(buf,sizeof(buf),vll);
mixDigest(eledigest,buf,strlen(buf));
}
snprintf(buf,sizeof(buf),"%.17g",score);
mixDigest(eledigest,buf,strlen(buf));
xorDigest(digest,eledigest,20);
zzlNext(zl,&eptr,&sptr);
}
} else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
zset *zs = o->ptr;
dictIterator *di = dictGetIterator(zs->dict);
dictEntry *de;
while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetEntryKey(de);
double *score = dictGetEntryVal(de);
snprintf(buf,sizeof(buf),"%.17g",*score);
memset(eledigest,0,20);
mixObjectDigest(eledigest,eleobj);
mixDigest(eledigest,buf,strlen(buf));
xorDigest(digest,eledigest,20);
}
dictReleaseIterator(di);
} else {
redisPanic("Unknown sorted set encoding");
}
dictReleaseIterator(di);
} else if (o->type == REDIS_HASH) {
hashTypeIterator *hi;
robj *obj;

View File

@ -102,6 +102,13 @@ robj *createZsetObject(void) {
return o;
}
robj *createZsetZiplistObject(void) {
unsigned char *zl = ziplistNew();
robj *o = createObject(REDIS_ZSET,zl);
o->encoding = REDIS_ENCODING_ZIPLIST;
return o;
}
void freeStringObject(robj *o) {
if (o->encoding == REDIS_ENCODING_RAW) {
sdsfree(o->ptr);
@ -135,11 +142,20 @@ void freeSetObject(robj *o) {
}
void freeZsetObject(robj *o) {
zset *zs = o->ptr;
dictRelease(zs->dict);
zslFree(zs->zsl);
zfree(zs);
zset *zs;
switch (o->encoding) {
case REDIS_ENCODING_SKIPLIST:
zs = o->ptr;
dictRelease(zs->dict);
zslFree(zs->zsl);
zfree(zs);
break;
case REDIS_ENCODING_ZIPLIST:
zfree(o->ptr);
break;
default:
redisPanic("Unknown sorted set encoding");
}
}
void freeHashObject(robj *o) {

View File

@ -302,24 +302,33 @@ int rdbSaveObject(FILE *fp, robj *o) {
redisPanic("Unknown set encoding");
}
} else if (o->type == REDIS_ZSET) {
/* Save a set value */
zset *zs = o->ptr;
dictIterator *di = dictGetIterator(zs->dict);
dictEntry *de;
/* Save a sorted set value */
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
size_t l = ziplistBlobLen((unsigned char*)o->ptr);
if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1;
nwritten += n;
while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetEntryKey(de);
double *score = dictGetEntryVal(de);
if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1;
if ((n = rdbSaveRawString(fp,o->ptr,l)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1;
} else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
zset *zs = o->ptr;
dictIterator *di = dictGetIterator(zs->dict);
dictEntry *de;
if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1;
nwritten += n;
while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetEntryKey(de);
double *score = dictGetEntryVal(de);
if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1;
nwritten += n;
if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1;
nwritten += n;
}
dictReleaseIterator(di);
} else {
redisPanic("Unknown sorted set encoding");
}
dictReleaseIterator(di);
} else if (o->type == REDIS_HASH) {
/* Save a hash value */
if (o->encoding == REDIS_ENCODING_ZIPMAP) {
@ -386,6 +395,8 @@ int rdbSaveKeyValuePair(FILE *fp, robj *key, robj *val,
vtype = REDIS_LIST_ZIPLIST;
else if (vtype == REDIS_SET && val->encoding == REDIS_ENCODING_INTSET)
vtype = REDIS_SET_INTSET;
else if (vtype == REDIS_ZSET && val->encoding == REDIS_ENCODING_ZIPLIST)
vtype = REDIS_ZSET_ZIPLIST;
/* Save type, key, value */
if (rdbSaveType(fp,vtype) == -1) return -1;
if (rdbSaveStringObject(fp,key) == -1) return -1;
@ -745,11 +756,13 @@ robj *rdbLoadObject(int type, FILE *fp) {
} else if (type == REDIS_ZSET) {
/* Read list/set value */
size_t zsetlen;
size_t maxelelen = 0;
zset *zs;
if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
o = createZsetObject();
zs = o->ptr;
/* Load every single element of the list/set */
while(zsetlen--) {
robj *ele;
@ -759,10 +772,21 @@ robj *rdbLoadObject(int type, FILE *fp) {
if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
ele = tryObjectEncoding(ele);
if (rdbLoadDoubleValue(fp,&score) == -1) return NULL;
/* Don't care about integer-encoded strings. */
if (ele->encoding == REDIS_ENCODING_RAW &&
sdslen(ele->ptr) > maxelelen)
maxelelen = sdslen(ele->ptr);
znode = zslInsert(zs->zsl,score,ele);
dictAdd(zs->dict,ele,&znode->score);
incrRefCount(ele); /* added to skiplist */
}
/* Convert *after* loading, since sorted sets are not stored ordered. */
if (zsetLength(o) <= server.zset_max_ziplist_entries &&
maxelelen <= server.zset_max_ziplist_value)
zsetConvert(o,REDIS_ENCODING_ZIPLIST);
} else if (type == REDIS_HASH) {
size_t hashlen;
@ -811,7 +835,8 @@ robj *rdbLoadObject(int type, FILE *fp) {
}
} else if (type == REDIS_HASH_ZIPMAP ||
type == REDIS_LIST_ZIPLIST ||
type == REDIS_SET_INTSET)
type == REDIS_SET_INTSET ||
type == REDIS_ZSET_ZIPLIST)
{
robj *aux = rdbLoadStringObject(fp);
@ -846,8 +871,14 @@ robj *rdbLoadObject(int type, FILE *fp) {
if (intsetLen(o->ptr) > server.set_max_intset_entries)
setTypeConvert(o,REDIS_ENCODING_HT);
break;
case REDIS_ZSET_ZIPLIST:
o->type = REDIS_ZSET;
o->encoding = REDIS_ENCODING_ZIPLIST;
if (zsetLength(o) > server.zset_max_ziplist_entries)
zsetConvert(o,REDIS_ENCODING_SKIPLIST);
break;
default:
redisPanic("Unknown enoding");
redisPanic("Unknown encoding");
break;
}
} else {

View File

@ -846,6 +846,8 @@ void initServerConfig() {
server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES;
server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE;
server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES;
server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES;
server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE;
server.shutdown_asap = 0;
server.cache_flush_delay = 0;
server.cluster_enabled = 0;

View File

@ -71,10 +71,12 @@
#define REDIS_ZSET 3
#define REDIS_HASH 4
#define REDIS_VMPOINTER 8
/* Object types only used for persistence in .rdb files */
#define REDIS_HASH_ZIPMAP 9
#define REDIS_LIST_ZIPLIST 10
#define REDIS_SET_INTSET 11
#define REDIS_ZSET_ZIPLIST 12
/* Objects encoding. Some kind of objects like Strings and Hashes can be
* internally represented in multiple ways. The 'encoding' field of the object
@ -198,6 +200,8 @@
#define REDIS_LIST_MAX_ZIPLIST_ENTRIES 512
#define REDIS_LIST_MAX_ZIPLIST_VALUE 64
#define REDIS_SET_MAX_INTSET_ENTRIES 512
#define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128
#define REDIS_ZSET_MAX_ZIPLIST_VALUE 64
/* Sets operations codes */
#define REDIS_OP_UNION 0
@ -589,6 +593,8 @@ struct redisServer {
size_t list_max_ziplist_entries;
size_t list_max_ziplist_value;
size_t set_max_intset_entries;
size_t zset_max_ziplist_entries;
size_t zset_max_ziplist_value;
time_t unixtime; /* Unix time sampled every second. */
/* Virtual memory I/O threads stuff */
/* An I/O thread process an element taken from the io_jobs queue and
@ -853,6 +859,7 @@ robj *createSetObject(void);
robj *createIntsetObject(void);
robj *createHashObject(void);
robj *createZsetObject(void);
robj *createZsetZiplistObject(void);
int getLongFromObjectOrReply(redisClient *c, robj *o, long *target, const char *msg);
int checkType(redisClient *c, robj *o, int type);
int getLongLongFromObjectOrReply(redisClient *c, robj *o, long long *target, const char *msg);
@ -916,6 +923,12 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal);
zskiplist *zslCreate(void);
void zslFree(zskiplist *zsl);
zskiplistNode *zslInsert(zskiplist *zsl, double score, robj *obj);
unsigned char *zzlInsert(unsigned char *zl, robj *ele, double score);
double zzlGetScore(unsigned char *sptr);
void zzlNext(unsigned char *zl, unsigned char **eptr, unsigned char **sptr);
void zzlPrev(unsigned char *zl, unsigned char **eptr, unsigned char **sptr);
unsigned int zsetLength(robj *zobj);
void zsetConvert(robj *zobj, int encoding);
/* Core functions */
void freeMemoryIfNeeded(void);
@ -1009,6 +1022,8 @@ int stringmatchlen(const char *pattern, int patternLen,
int stringmatch(const char *pattern, const char *string, int nocase);
long long memtoll(const char *p, int *err);
int ll2string(char *s, size_t len, long long value);
int string2ll(char *s, size_t len, long long *value);
int d2string(char *s, size_t len, double value);
int isStringRepresentableAsLong(sds s, long *longval);
int isStringRepresentableAsLongLong(sds s, long long *longval);
int isObjectRepresentableAsLongLong(robj *o, long long *llongval);

View File

@ -199,6 +199,9 @@ void sortCommand(redisClient *c) {
j++;
}
/* Destructively convert encoded sorted sets for SORT. */
if (sortval->type == REDIS_ZSET) zsetConvert(sortval, REDIS_ENCODING_SKIPLIST);
/* Load the sorting vector with all the objects to sort */
switch(sortval->type) {
case REDIS_LIST: vectorlen = listTypeLength(sortval); break;

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,7 @@
#include "redis.h"
#include <ctype.h>
#include <limits.h>
#include <math.h>
/* Glob-style pattern matching. */
int stringmatchlen(const char *pattern, int patternLen,
@ -200,6 +201,102 @@ int ll2string(char *s, size_t len, long long value) {
return l;
}
/* Convert a string into a long long. Returns 1 if the string could be parsed
* into a (non-overflowing) long long, 0 otherwise. The value will be set to
* the parsed value when appropriate. */
int string2ll(char *s, size_t slen, long long *value) {
char *p = s;
size_t plen = 0;
int negative = 0;
unsigned long long v;
if (plen == slen)
return 0;
if (p[0] == '-') {
negative = 1;
p++; plen++;
/* Abort on only a negative sign. */
if (plen == slen)
return 0;
}
/* First digit should be 1-9. */
if (p[0] >= '1' && p[0] <= '9') {
v = p[0]-'0';
p++; plen++;
} else {
return 0;
}
while (plen < slen && p[0] >= '0' && p[0] <= '9') {
if (v > (ULLONG_MAX / 10)) /* Overflow. */
return 0;
v *= 10;
if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */
return 0;
v += p[0]-'0';
p++; plen++;
}
/* Return if not all bytes were used. */
if (plen < slen)
return 0;
if (negative) {
if (v > (-(unsigned long long)LLONG_MIN)) /* Overflow. */
return 0;
if (value != NULL) *value = -v;
} else {
if (v > LLONG_MAX) /* Overflow. */
return 0;
if (value != NULL) *value = v;
}
return 1;
}
/* Convert a double to a string representation. Returns the number of bytes
* required. The representation should always be parsable by stdtod(3). */
int d2string(char *buf, size_t len, double value) {
if (isnan(value)) {
len = snprintf(buf,len,"nan");
} else if (isinf(value)) {
if (value < 0)
len = snprintf(buf,len,"-inf");
else
len = snprintf(buf,len,"inf");
} else if (value == 0) {
/* See: http://en.wikipedia.org/wiki/Signed_zero, "Comparisons". */
if (1.0/value < 0)
len = snprintf(buf,len,"-0");
else
len = snprintf(buf,len,"0");
} else {
#if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL)
/* Check if the float is in a safe range to be casted into a
* long long. We are assuming that long long is 64 bit here.
* Also we are assuming that there are no implementations around where
* double has precision < 52 bit.
*
* Under this assumptions we test if a double is inside an interval
* where casting to long long is safe. Then using two castings we
* make sure the decimal part is zero. If all this is true we use
* integer printing function that is much faster. */
double min = -4503599627370495; /* (2^52)-1 */
double max = 4503599627370496; /* -(2^52) */
if (val > min && val < max && value == ((double)((long long)value)))
len = ll2string(buf,len,(long long)value);
else
#endif
len = snprintf(buf,len,"%.17g",value);
}
return len;
}
/* Check if the sds string 's' can be represented by a long long
* (that is, is a number that fits into long without any other space or
* character before or after the digits, so that converting this number

View File

@ -385,8 +385,8 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
* The pointer "p" points to the first entry that does NOT need to be
* updated, i.e. consecutive fields MAY need an update. */
static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p) {
unsigned int curlen = ZIPLIST_BYTES(zl), rawlen, rawlensize;
unsigned int offset, noffset, extra;
size_t curlen = ZIPLIST_BYTES(zl), rawlen, rawlensize;
size_t offset, noffset, extra;
unsigned char *np;
zlentry cur, next;
@ -441,7 +441,8 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) {
unsigned int i, totlen, deleted = 0;
int offset, nextdiff = 0;
size_t offset;
int nextdiff = 0;
zlentry first, tail;
first = zipEntry(p);
@ -493,8 +494,9 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
/* Insert item at "p". */
static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0;
unsigned int offset, nextdiff = 0;
size_t curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0;
size_t offset;
int nextdiff = 0;
unsigned char encoding = 0;
long long value;
zlentry entry, tail;
@ -678,7 +680,7 @@ unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char
* Also update *p in place, to be able to iterate over the
* ziplist, while deleting entries. */
unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
unsigned int offset = *p-zl;
size_t offset = *p-zl;
zl = __ziplistDelete(zl,*p,1);
/* Store pointer to current element in p, because ziplistDelete will

File diff suppressed because it is too large Load Diff