Merge branch 'master' into networking-perf

Resolved conflict in src/db.c and changed adding an error to the reply
in blockingPopGenericCommand to use the new API.
This commit is contained in:
Pieter Noordhuis 2010-09-03 16:44:31 +02:00
commit 9e83ac06ef
17 changed files with 352 additions and 49 deletions

69
README
View File

@ -1 +1,68 @@
Check the 'doc' directory. doc/README.html is a good starting point :)
Where to find complete Redis documentation?
-------------------------------------------
This README is just a fast "quick start" document. You can find more detailed
documentation here:
1) http://code.google.com/p/redis
2) Check the 'doc' directory. doc/README.html is a good starting point :)
Building Redis
--------------
It is as simple as:
% make
Redis is just a single binary, but if you want to install it you can use
the "make install" target that will copy the binary in /usr/local/bin
for default.
You can run a 32 bit Redis binary using:
% make 32bit
After you build Redis is a good idea to test it, using:
% make test
Running Redis
-------------
To run Redis with the default configuration just type:
% cd src
% ./redis-server
If you want to provide your redis.conf, you have to run it using an additional
parameter (the path of the configuration file):
% cd src
% ./redis-server /path/to/redis.conf
Playing with Redis
------------------
You can use redis-cli to play with Redis. Start a redis-server instance,
then in another terminal try the following:
% cd src
% ./redis-cli
redis> ping
PONG
redis> set foo bar
OK
redis> get foo
"bar"
redis> incr mycounter
(integer) 1
redis> incr mycounter
(integer) 2
redis>
You can find the list of all the available commands here:
http://code.google.com/p/redis/wiki/CommandReference
Enjoy!

View File

@ -33,6 +33,7 @@ CHECKAOFPRGNAME = redis-check-aof
all: redis-server redis-benchmark redis-cli redis-check-dump redis-check-aof
# Deps (use make dep to generate this)
adlist.o: adlist.c adlist.h zmalloc.h
ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c
@ -40,25 +41,61 @@ ae_epoll.o: ae_epoll.c
ae_kqueue.o: ae_kqueue.c
ae_select.o: ae_select.c
anet.o: anet.c fmacros.h anet.h
aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h sha1.h
dict.o: dict.c fmacros.h dict.h zmalloc.h
intset.o: intset.c intset.h zmalloc.h
linenoise.o: linenoise.c fmacros.h
lzf_c.o: lzf_c.c lzfP.h
lzf_d.o: lzf_d.c lzfP.h
multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \
adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
pqsort.o: pqsort.c
pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h lzf.h
redis-benchmark.o: redis-benchmark.c fmacros.h ae.h anet.h sds.h adlist.h \
zmalloc.h
redis-check-aof.o: redis-check-aof.c fmacros.h config.h
redis-check-dump.o: redis-check-dump.c lzf.h
redis-cli.o: redis-cli.c fmacros.h anet.h sds.h adlist.h zmalloc.h \
linenoise.h
redis.o: redis.c fmacros.h config.h redis.h ae.h sds.h anet.h dict.h \
adlist.h zmalloc.h lzf.h pqsort.h zipmap.h ziplist.h sha1.h
redis-cli.o: redis-cli.c fmacros.h version.h anet.h sds.h adlist.h \
zmalloc.h linenoise.h
redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
release.o: release.c release.h
replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \
adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
sds.o: sds.c sds.h zmalloc.h
sha1.o: sha1.c sha1.h
sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h pqsort.h
t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \
adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
util.o: util.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
vm.o: vm.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \
zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h
ziplist.o: ziplist.c zmalloc.h ziplist.h
zipmap.o: zipmap.c zmalloc.h
intset.o: intset.c zmalloc.h
zmalloc.o: zmalloc.c config.h
redis-server: $(OBJ)

View File

@ -21,6 +21,16 @@
#define redis_stat stat
#endif
/* test for proc filesystem */
#ifdef __linux__
#define HAVE_PROCFS 1
#endif
/* test for task_info() */
#if defined(__APPLE__)
#define HAVE_TASKINFO 1
#endif
/* test for backtrace() */
#if defined(__APPLE__) || defined(__linux__)
#define HAVE_BACKTRACE 1

View File

@ -221,17 +221,17 @@ void keysCommand(redisClient *c) {
dictIterator *di;
dictEntry *de;
sds pattern = c->argv[1]->ptr;
int plen = sdslen(pattern);
int plen = sdslen(pattern), allkeys;
unsigned long numkeys = 0;
void *replylen = addDeferredMultiBulkLength(c);
di = dictGetIterator(c->db->dict);
allkeys = (pattern[0] == '*' && pattern[1] == '\0');
while((de = dictNext(di)) != NULL) {
sds key = dictGetEntryKey(de);
robj *keyobj;
if ((pattern[0] == '*' && pattern[1] == '\0') ||
stringmatchlen(pattern,plen,key,sdslen(key),0)) {
if (allkeys || stringmatchlen(pattern,plen,key,sdslen(key),0)) {
keyobj = createStringObject(key,sdslen(key));
if (expireIfNeeded(c->db,keyobj) == 0) {
addReplyBulk(c,keyobj);

View File

@ -65,8 +65,8 @@
/* data type to hold offset in file and size */
typedef struct {
void *data;
unsigned long size;
unsigned long offset;
size_t size;
size_t offset;
} pos;
static unsigned char level = 0;
@ -77,8 +77,8 @@ static pos positions[16];
/* Hold a stack of errors */
typedef struct {
char error[16][1024];
unsigned long offset[16];
unsigned int level;
size_t offset[16];
size_t level;
} errors_t;
static errors_t errors;
@ -112,7 +112,7 @@ int readBytes(void *target, long num) {
if (p.offset + num > p.size) {
return 0;
} else {
memcpy(target, (void*)((unsigned long)p.data + p.offset), num);
memcpy(target, (void*)((size_t)p.data + p.offset), num);
if (!peek) positions[level].offset += num;
}
return 1;
@ -494,15 +494,17 @@ void printCentered(int indent, int width, char* body) {
printf("%s %s %s\n", head, body, tail);
}
void printValid(int ops, int bytes) {
void printValid(uint64_t ops, uint64_t bytes) {
char body[80];
sprintf(body, "Processed %d valid opcodes (in %d bytes)", ops, bytes);
sprintf(body, "Processed %llu valid opcodes (in %llu bytes)",
(unsigned long long) ops, (unsigned long long) bytes);
printCentered(4, 80, body);
}
void printSkipped(int bytes, int offset) {
void printSkipped(uint64_t bytes, uint64_t offset) {
char body[80];
sprintf(body, "Skipped %d bytes (resuming at 0x%08x)", bytes, offset);
sprintf(body, "Skipped %llu bytes (resuming at 0x%08llx)",
(unsigned long long) bytes, (unsigned long long) offset);
printCentered(4, 80, body);
}
@ -541,7 +543,7 @@ void printErrorStack(entry *e) {
}
void process() {
int i, num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
processHeader();
@ -558,7 +560,9 @@ void process() {
num_valid_bytes = 0;
/* search for next valid entry */
unsigned long offset = positions[0].offset + 1;
uint64_t offset = positions[0].offset + 1;
int i = 0;
while (!entry.success && offset < positions[0].size) {
positions[1].offset = offset;
@ -606,9 +610,10 @@ void process() {
}
/* print summary on errors */
if (num_errors > 0) {
if (num_errors) {
printf("\n");
printf("Total unprocessable opcodes: %d\n", num_errors);
printf("Total unprocessable opcodes: %llu\n",
(unsigned long long) num_errors);
}
}
@ -620,7 +625,7 @@ int main(int argc, char **argv) {
}
int fd;
unsigned long size;
off_t size;
struct stat stat;
void *data;
@ -634,6 +639,10 @@ int main(int argc, char **argv) {
size = stat.st_size;
}
if (sizeof(size_t) == sizeof(int32_t) && size >= INT_MAX) {
ERROR("Cannot check dump files >2GB on a 32-bit platform\n");
}
data = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
if (data == MAP_FAILED) {
ERROR("Cannot mmap: %s\n", argv[1]);

View File

@ -96,7 +96,7 @@ static sds cliReadLine(int fd) {
ssize_t ret;
ret = read(fd,&c,1);
if (ret == -1) {
if (ret <= 0) {
sdsfree(line);
return NULL;
} else if ((ret == 0) || (c == '\n')) {
@ -251,12 +251,32 @@ static int selectDb(int fd) {
return 0;
}
static void showInteractiveHelp(void) {
printf(
"\n"
"Welcome to redis-cli " REDIS_VERSION "!\n"
"Just type any valid Redis command to see a pretty printed output.\n"
"\n"
"It is possible to quote strings, like in:\n"
" set \"my key\" \"some string \\xff\\n\"\n"
"\n"
"You can find a list of valid Redis commands at\n"
" http://code.google.com/p/redis/wiki/CommandReference\n"
"\n"
"Note: redis-cli supports line editing, use up/down arrows for history."
"\n\n");
}
static int cliSendCommand(int argc, char **argv, int repeat) {
char *command = argv[0];
int fd, j, retval = 0;
sds cmd;
config.raw_output = !strcasecmp(command,"info");
if (!strcasecmp(command,"help")) {
showInteractiveHelp();
return 0;
}
if (!strcasecmp(command,"shutdown")) config.shutdown = 1;
if (!strcasecmp(command,"monitor")) config.monitor_mode = 1;
if (!strcasecmp(command,"subscribe") ||
@ -282,7 +302,8 @@ static int cliSendCommand(int argc, char **argv, int repeat) {
while(repeat--) {
anetWrite(fd,cmd,sdslen(cmd));
while (config.monitor_mode) {
cliReadSingleLineReply(fd,0);
if (cliReadSingleLineReply(fd,0)) exit(1);
printf("\n");
}
if (config.pubsub_mode) {
@ -477,10 +498,16 @@ int main(int argc, char **argv) {
if (config.auth != NULL) {
char *authargv[2];
int dbnum = config.dbnum;
/* We need to save the real configured database number and set it to
* zero here, otherwise cliSendCommand() will try to perform the
* SELECT command before the authentication, and it will fail. */
config.dbnum = 0;
authargv[0] = "AUTH";
authargv[1] = config.auth;
cliSendCommand(2, convertToSds(2, authargv), 1);
config.dbnum = dbnum; /* restore the right DB number */
}
/* Start interactive mode when no command is provided */

View File

@ -1163,6 +1163,7 @@ sds genRedisInfoString(void) {
"blocked_clients:%d\r\n"
"used_memory:%zu\r\n"
"used_memory_human:%s\r\n"
"mem_fragmentation_ratio:%.2f\r\n"
"changes_since_last_save:%lld\r\n"
"bgsave_in_progress:%d\r\n"
"last_save_time:%ld\r\n"
@ -1189,6 +1190,7 @@ sds genRedisInfoString(void) {
server.blpop_blocked_clients,
zmalloc_used_memory(),
hmem,
zmalloc_get_fragmentation_ratio(),
server.dirty,
server.bgsavechildpid != -1,
server.lastsave,

View File

@ -782,9 +782,20 @@ int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) {
/* Blocking RPOP/LPOP */
void blockingPopGenericCommand(redisClient *c, int where) {
robj *o;
long long lltimeout;
time_t timeout;
int j;
/* Make sure timeout is an integer value */
if (getLongLongFromObjectOrReply(c,c->argv[c->argc-1],&lltimeout,
"timeout is not an integer") != REDIS_OK) return;
/* Make sure the timeout is not negative */
if (lltimeout < 0) {
addReplyError(c,"timeout is negative");
return;
}
for (j = 1; j < c->argc-1; j++) {
o = lookupKeyWrite(c->db,c->argv[j]);
if (o != NULL) {
@ -823,8 +834,16 @@ void blockingPopGenericCommand(redisClient *c, int where) {
}
}
}
/* If we are inside a MULTI/EXEC and the list is empty the only thing
* we can do is treating it as a timeout (even with timeout 0). */
if (c->flags & REDIS_MULTI) {
addReply(c,shared.nullmultibulk);
return;
}
/* If the list is empty or the key does not exists we must block */
timeout = strtol(c->argv[c->argc-1]->ptr,NULL,10);
timeout = lltimeout;
if (timeout > 0) timeout += time(NULL);
blockForKeys(c,c->argv+1,c->argc-2,timeout);
}

View File

@ -1 +1 @@
#define REDIS_VERSION "2.1.3"
#define REDIS_VERSION "2.1.4"

View File

@ -110,6 +110,11 @@ void vmInit(void) {
/* LZF requires a lot of stack */
pthread_attr_init(&server.io_threads_attr);
pthread_attr_getstacksize(&server.io_threads_attr, &stacksize);
/* Solaris may report a stacksize of 0, let's set it to 1 otherwise
* multiplying it by 2 in the while loop later will not really help ;) */
if (!stacksize) stacksize = 1;
while (stacksize < REDIS_THREAD_STACK_SIZE) stacksize *= 2;
pthread_attr_setstacksize(&server.io_threads_attr, stacksize);
/* Listen for events in the threaded I/O pipe */

View File

@ -32,6 +32,7 @@
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include "config.h"
#if defined(__sun)
@ -170,3 +171,69 @@ size_t zmalloc_used_memory(void) {
void zmalloc_enable_thread_safeness(void) {
zmalloc_thread_safe = 1;
}
/* Fragmentation = RSS / allocated-bytes */
#if defined(HAVE_PROCFS)
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
float zmalloc_get_fragmentation_ratio(void) {
size_t allocated = zmalloc_used_memory();
int page = sysconf(_SC_PAGESIZE);
size_t rss;
char buf[4096];
char filename[256];
int fd, count;
char *p, *x;
snprintf(filename,256,"/proc/%d/stat",getpid());
if ((fd = open(filename,O_RDONLY)) == -1) return 0;
if (read(fd,buf,4096) <= 0) {
close(fd);
return 0;
}
close(fd);
p = buf;
count = 23; /* RSS is the 24th field in /proc/<pid>/stat */
while(p && count--) {
p = strchr(p,' ');
if (p) p++;
}
if (!p) return 0;
x = strchr(p,' ');
if (!x) return 0;
*x = '\0';
rss = strtoll(p,NULL,10);
rss *= page;
return (float)rss/allocated;
}
#elif defined(HAVE_TASKINFO)
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/sysctl.h>
#include <mach/task.h>
#include <mach/mach_init.h>
float zmalloc_get_fragmentation_ratio(void) {
task_t task = MACH_PORT_NULL;
struct task_basic_info t_info;
mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
if (task_for_pid(current_task(), getpid(), &task) != KERN_SUCCESS)
return 0;
task_info(task, TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count);
return (float)t_info.resident_size/zmalloc_used_memory();
}
#else
float zmalloc_get_fragmentation_ratio(void) {
return 0;
}
#endif

View File

@ -38,5 +38,6 @@ void zfree(void *ptr);
char *zstrdup(const char *s);
size_t zmalloc_used_memory(void);
void zmalloc_enable_thread_safeness(void);
float zmalloc_get_fragmentation_ratio(void);
#endif /* _ZMALLOC_H */

View File

@ -83,7 +83,9 @@ proc ping_server {host port} {
}
close $fd
} e]} {
puts "Can't PING server at $host:$port... $e"
puts -nonewline "."
} else {
puts -nonewline "ok"
}
return $retval
}
@ -170,14 +172,33 @@ proc start_server {options {code undefined}} {
if {$::valgrind} {
exec valgrind src/redis-server $config_file > $stdout 2> $stderr &
after 2000
} else {
exec src/redis-server $config_file > $stdout 2> $stderr &
after 500
}
# check that the server actually started
if {$code ne "undefined" && ![ping_server $::host $::port]} {
# ugly but tries to be as fast as possible...
set retrynum 20
set serverisup 0
puts -nonewline "=== ($tags) Starting server ${::host}:${::port} "
after 10
if {$code ne "undefined"} {
while {[incr retrynum -1]} {
catch {
if {[ping_server $::host $::port]} {
set serverisup 1
}
}
if {$serverisup} break
after 50
}
} else {
set serverisup 1
}
puts {}
if {!$serverisup} {
error_and_quit $config_file [exec cat $stderr]
}

View File

@ -1,4 +1,4 @@
start_server {} {
start_server {tags {"other"}} {
test {SAVE - make sure there are all the types as values} {
# Wait for a background saving in progress to terminate
waitForBgsave r

View File

@ -1,4 +1,4 @@
start_server {} {
start_server {tags {"protocol"}} {
test {Handle an empty query well} {
set fd [r channel]
puts -nonewline $fd "\r\n"

View File

@ -139,6 +139,28 @@ start_server {
assert_equal 0 [r exists blist1]
}
test "$pop: with negative timeout" {
set rd [redis_deferring_client]
$rd $pop blist1 -1
assert_error "ERR*is negative*" {$rd read}
}
test "$pop: with non-integer timeout" {
set rd [redis_deferring_client]
$rd $pop blist1 1.1
assert_error "ERR*not an integer*" {$rd read}
}
test "$pop: with zero timeout should block indefinitely" {
# To test this, use a timeout of 0 and wait a second.
# The blocking pop should still be waiting for a push.
set rd [redis_deferring_client]
$rd $pop blist1 0
after 1000
r rpush blist1 foo
assert_equal {blist1 foo} [$rd read]
}
test "$pop: second argument is not a list" {
set rd [redis_deferring_client]
r del blist1 blist2
@ -172,6 +194,17 @@ start_server {
}
}
test {BLPOP inside a transaction} {
r del xlist
r lpush xlist foo
r lpush xlist bar
r multi
r blpop xlist 0
r blpop xlist 0
r blpop xlist 0
r exec
} {{xlist bar} {xlist foo} {}}
test {LPUSHX, RPUSHX - generic} {
r del xlist
assert_equal 0 [r lpushx xlist a]

View File

@ -106,14 +106,17 @@ start_server {
}
r sadd set5 0
# it is possible that a hashtable encoded only contains integers,
# because it is converted from an intset to a hashtable when a
# non-integer element is added and then removed.
# To make sure the sets are encoded as the type we are testing -- also
# when the VM is enabled and the values may be swapped in and out
# while the tests are running -- an extra element is added to every
# set that determines its encoding.
set large 200
if {$type eq "hashtable"} {
for {set i 1} {$i <= 5} {incr i} {
r sadd [format "set%d" $i] foo
r srem [format "set%d" $i] foo
}
set large foo
}
for {set i 1} {$i <= 5} {incr i} {
r sadd [format "set%d" $i] $large
}
test "Generated sets must be encoded as $type" {
@ -123,20 +126,20 @@ start_server {
}
test "SINTER with two sets - $type" {
assert_equal {195 196 197 198 199} [lsort [r sinter set1 set2]]
assert_equal [list 195 196 197 198 199 $large] [lsort [r sinter set1 set2]]
}
test "SINTERSTORE with two sets - $type" {
r sinterstore setres set1 set2
assert_encoding intset setres
assert_equal {195 196 197 198 199} [lsort [r smembers setres]]
assert_encoding $type setres
assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres]]
}
test "SINTERSTORE with two sets, after a DEBUG RELOAD - $type" {
r debug reload
r sinterstore setres set1 set2
assert_encoding intset setres
assert_equal {195 196 197 198 199} [lsort [r smembers setres]]
assert_encoding $type setres
assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres]]
}
test "SUNION with two sets - $type" {
@ -146,18 +149,18 @@ start_server {
test "SUNIONSTORE with two sets - $type" {
r sunionstore setres set1 set2
assert_encoding intset setres
assert_encoding $type setres
set expected [lsort -uniq "[r smembers set1] [r smembers set2]"]
assert_equal $expected [lsort [r smembers setres]]
}
test "SINTER against three sets - $type" {
assert_equal {195 199} [lsort [r sinter set1 set2 set3]]
assert_equal [list 195 199 $large] [lsort [r sinter set1 set2 set3]]
}
test "SINTERSTORE with three sets - $type" {
r sinterstore setres set1 set2 set3
assert_equal {195 199} [r smembers setres]
assert_equal [list 195 199 $large] [lsort [r smembers setres]]
}
test "SUNION with non existing keys - $type" {
@ -175,7 +178,9 @@ start_server {
test "SDIFFSTORE with three sets - $type" {
r sdiffstore setres set1 set4 set5
assert_encoding intset setres
# The type is determined by type of the first key to diff against.
# See the implementation for more information.
assert_encoding $type setres
assert_equal {1 2 3 4} [lsort [r smembers setres]]
}
}