diff --git a/TODO b/TODO index 48ea8ac6..580adda6 100644 --- a/TODO +++ b/TODO @@ -31,6 +31,7 @@ APPEND ONLY FILE OPTIMIZATIONS ============= +* Avoid COW due to incrementing the dict iterators counter. * SORT: Don't copy the list into a vector when BY argument is constant. * Write the hash table size of every db in the dump, so that Redis can resize the hash table just one time when loading a big DB. * Read-only mode for slaves. diff --git a/src/Makefile b/src/Makefile index 4d0f86db..acbc9b5f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -25,7 +25,7 @@ PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin INSTALL= cp -p -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o dscache.o pubsub.o multi.o debug.o sort.o intset.o syncio.o diskstore.o +OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o dscache.o pubsub.o multi.o debug.o sort.o intset.o syncio.o diskstore.o endian.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o diff --git a/src/config.h b/src/config.h index 40f22fa5..d98067c1 100644 --- a/src/config.h +++ b/src/config.h @@ -21,7 +21,7 @@ #define redis_malloc_size(p) malloc_size(p) #endif -/* define redis_fstat to fstat or fstat64() */ +/* Tefine redis_fstat to fstat or fstat64() */ #if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) #define redis_fstat fstat64 #define redis_stat stat64 @@ -30,22 +30,22 @@ #define redis_stat stat #endif -/* test for proc filesystem */ +/* Test for proc filesystem */ #ifdef __linux__ #define HAVE_PROCFS 1 #endif -/* test for task_info() */ +/* Test for task_info() */ #if defined(__APPLE__) #define HAVE_TASKINFO 1 #endif -/* test for backtrace() */ +/* Test for backtrace() */ #if defined(__APPLE__) || defined(__linux__) #define HAVE_BACKTRACE 1 #endif -/* test for polling API */ +/* Test for polling API */ #ifdef __linux__ #define HAVE_EPOLL 1 #endif @@ -54,11 +54,62 @@ #define HAVE_KQUEUE 1 #endif -/* define aof_fsync to fdatasync() in Linux and fsync() for all the rest */ +/* Define aof_fsync to fdatasync() in Linux and fsync() for all the rest */ #ifdef __linux__ #define aof_fsync fdatasync #else #define aof_fsync fsync #endif +/* Byte ordering detection */ +#include /* This will likely define BYTE_ORDER */ + +#ifndef BYTE_ORDER +#if (BSD >= 199103) +# include +#else +#if defined(linux) || defined(__linux__) +# include +#else +#define LITTLE_ENDIAN 1234 /* least-significant byte first (vax, pc) */ +#define BIG_ENDIAN 4321 /* most-significant byte first (IBM, net) */ +#define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ + +#if defined(vax) || defined(ns32000) || defined(sun386) || defined(__i386__) || \ + defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ + defined(__alpha__) || defined(__alpha) +#define BYTE_ORDER LITTLE_ENDIAN +#endif + +#if defined(sel) || defined(pyr) || defined(mc68000) || defined(sparc) || \ + defined(is68k) || defined(tahoe) || defined(ibm032) || defined(ibm370) || \ + defined(MIPSEB) || defined(_MIPSEB) || defined(_IBMR2) || defined(DGUX) ||\ + defined(apollo) || defined(__convex__) || defined(_CRAY) || \ + defined(__hppa) || defined(__hp9000) || \ + defined(__hp9000s300) || defined(__hp9000s700) || \ + defined (BIT_ZERO_ON_LEFT) || defined(m68k) || defined(__sparc) +#define BYTE_ORDER BIG_ENDIAN +#endif +#endif /* linux */ +#endif /* BSD */ +#endif /* BYTE_ORDER */ + +#if defined(__BYTE_ORDER) && !defined(BYTE_ORDER) +#if (__BYTE_ORDER == __LITTLE_ENDIAN) +#define BYTE_ORDER LITTLE_ENDIAN +#else +#define BYTE_ORDER BIG_ENDIAN +#endif +#endif + +#if !defined(BYTE_ORDER) || \ + (BYTE_ORDER != BIG_ENDIAN && BYTE_ORDER != LITTLE_ENDIAN) + /* you must determine what the correct bit order is for + * your compiler - the next line is an intentional error + * which will force your compiles to bomb until you fix + * the above macros. + */ +#error "Undefined or invalid BYTE_ORDER" +#endif + #endif diff --git a/src/endian.c b/src/endian.c new file mode 100644 index 00000000..aff2425a --- /dev/null +++ b/src/endian.c @@ -0,0 +1,63 @@ +/* Toggle the 16 bit unsigned integer pointed by *p from little endian to + * big endian */ +void memrev16(void *p) { + unsigned char *x = p, t; + + t = x[0]; + x[0] = x[1]; + x[1] = t; +} + +/* Toggle the 32 bit unsigned integer pointed by *p from little endian to + * big endian */ +void memrev32(void *p) { + unsigned char *x = p, t; + + t = x[0]; + x[0] = x[3]; + x[3] = t; + t = x[1]; + x[1] = x[2]; + x[2] = t; +} + +/* Toggle the 64 bit unsigned integer pointed by *p from little endian to + * big endian */ +void memrev64(void *p) { + unsigned char *x = p, t; + + t = x[0]; + x[0] = x[7]; + x[7] = t; + t = x[1]; + x[1] = x[6]; + x[6] = t; + t = x[2]; + x[2] = x[5]; + x[5] = t; + t = x[3]; + x[3] = x[4]; + x[4] = t; +} + +#ifdef TESTMAIN +#include + +int main(void) { + char buf[32]; + + sprintf(buf,"ciaoroma"); + memrev16(buf); + printf("%s\n", buf); + + sprintf(buf,"ciaoroma"); + memrev32(buf); + printf("%s\n", buf); + + sprintf(buf,"ciaoroma"); + memrev64(buf); + printf("%s\n", buf); + + return 0; +} +#endif diff --git a/src/endian.h b/src/endian.h new file mode 100644 index 00000000..bef82272 --- /dev/null +++ b/src/endian.h @@ -0,0 +1,20 @@ +#ifndef __ENDIAN_H +#define __ENDIAN_H + +void memrev16(void *p); +void memrev32(void *p); +void memrev64(void *p); + +/* variants of the function doing the actual convertion only if the target + * host is big endian */ +#if (BYTE_ORDER == LITTLE_ENDIAN) +#define memrev16ifbe(p) +#define memrev32ifbe(p) +#define memrev64ifbe(p) +#else +#define memrev16ifbe(p) memrev16(p) +#define memrev32ifbe(p) memrev32(p) +#define memrev64ifbe(p) memrev64(p) +#endif + +#endif diff --git a/src/intset.c b/src/intset.c index 13bd220e..4dd0141d 100644 --- a/src/intset.c +++ b/src/intset.c @@ -3,6 +3,7 @@ #include #include "intset.h" #include "zmalloc.h" +#include "endian.h" /* Note that these encodings are ordered, so: * INTSET_ENC_INT16 < INTSET_ENC_INT32 < INTSET_ENC_INT64. */ @@ -16,16 +17,29 @@ static uint8_t _intsetValueEncoding(int64_t v) { return INTSET_ENC_INT64; else if (v < INT16_MIN || v > INT16_MAX) return INTSET_ENC_INT32; - return INTSET_ENC_INT16; + else + return INTSET_ENC_INT16; } /* Return the value at pos, given an encoding. */ static int64_t _intsetGetEncoded(intset *is, int pos, uint8_t enc) { - if (enc == INTSET_ENC_INT64) - return ((int64_t*)is->contents)[pos]; - else if (enc == INTSET_ENC_INT32) - return ((int32_t*)is->contents)[pos]; - return ((int16_t*)is->contents)[pos]; + int64_t v64; + int32_t v32; + int16_t v16; + + if (enc == INTSET_ENC_INT64) { + memcpy(&v64,((int64_t*)is->contents)+pos,sizeof(v64)); + memrev64ifbe(&v64); + return v64; + } else if (enc == INTSET_ENC_INT32) { + memcpy(&v32,((int32_t*)is->contents)+pos,sizeof(v32)); + memrev32ifbe(&v32); + return v32; + } else { + memcpy(&v16,((int16_t*)is->contents)+pos,sizeof(v16)); + memrev16ifbe(&v16); + return v16; + } } /* Return the value at pos, using the configured encoding. */ @@ -35,12 +49,16 @@ static int64_t _intsetGet(intset *is, int pos) { /* Set the value at pos, using the configured encoding. */ static void _intsetSet(intset *is, int pos, int64_t value) { - if (is->encoding == INTSET_ENC_INT64) + if (is->encoding == INTSET_ENC_INT64) { ((int64_t*)is->contents)[pos] = value; - else if (is->encoding == INTSET_ENC_INT32) + memrev64ifbe(((int64_t*)is->contents)+pos); + } else if (is->encoding == INTSET_ENC_INT32) { ((int32_t*)is->contents)[pos] = value; - else + memrev32ifbe(((int32_t*)is->contents)+pos); + } else { ((int16_t*)is->contents)[pos] = value; + memrev16ifbe(((int16_t*)is->contents)+pos); + } } /* Create an empty intset. */ diff --git a/src/sha1.c b/src/sha1.c index 2c50433e..26a5565e 100644 --- a/src/sha1.c +++ b/src/sha1.c @@ -28,55 +28,7 @@ A million repetitions of "a" #include "solarisfixes.h" #endif #include "sha1.h" - -#ifndef BYTE_ORDER -#if (BSD >= 199103) -# include -#else -#if defined(linux) || defined(__linux__) -# include -#else -#define LITTLE_ENDIAN 1234 /* least-significant byte first (vax, pc) */ -#define BIG_ENDIAN 4321 /* most-significant byte first (IBM, net) */ -#define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ - -#if defined(vax) || defined(ns32000) || defined(sun386) || defined(__i386__) || \ - defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ - defined(__alpha__) || defined(__alpha) -#define BYTE_ORDER LITTLE_ENDIAN -#endif - -#if defined(sel) || defined(pyr) || defined(mc68000) || defined(sparc) || \ - defined(is68k) || defined(tahoe) || defined(ibm032) || defined(ibm370) || \ - defined(MIPSEB) || defined(_MIPSEB) || defined(_IBMR2) || defined(DGUX) ||\ - defined(apollo) || defined(__convex__) || defined(_CRAY) || \ - defined(__hppa) || defined(__hp9000) || \ - defined(__hp9000s300) || defined(__hp9000s700) || \ - defined (BIT_ZERO_ON_LEFT) || defined(m68k) || defined(__sparc) -#define BYTE_ORDER BIG_ENDIAN -#endif -#endif /* linux */ -#endif /* BSD */ -#endif /* BYTE_ORDER */ - -#if defined(__BYTE_ORDER) && !defined(BYTE_ORDER) -#if (__BYTE_ORDER == __LITTLE_ENDIAN) -#define BYTE_ORDER LITTLE_ENDIAN -#else -#define BYTE_ORDER BIG_ENDIAN -#endif -#endif - -#if !defined(BYTE_ORDER) || \ - (BYTE_ORDER != BIG_ENDIAN && BYTE_ORDER != LITTLE_ENDIAN && \ - BYTE_ORDER != PDP_ENDIAN) - /* you must determine what the correct bit order is for - * your compiler - the next line is an intentional error - * which will force your compiles to bomb until you fix - * the above macros. - */ -#error "Undefined or invalid BYTE_ORDER" -#endif +#include "config.h" #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) diff --git a/src/ziplist.c b/src/ziplist.c index 524f7238..55bb662b 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -68,6 +68,7 @@ #include #include "zmalloc.h" #include "ziplist.h" +#include "endian.h" int ll2string(char *s, size_t len, long long value); @@ -207,6 +208,7 @@ static unsigned int zipPrevDecodeLength(unsigned char *p, unsigned int *lensize) } else { if (lensize) *lensize = 1+sizeof(len); memcpy(&len,p+1,sizeof(len)); + memrev32ifbe(&len); } return len; } @@ -223,6 +225,7 @@ static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { } else { p[0] = ZIP_BIGLEN; memcpy(p+1,&len,sizeof(len)); + memrev32ifbe(p+1); return 1+sizeof(len); } } @@ -234,6 +237,7 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { if (p == NULL) return; p[0] = ZIP_BIGLEN; memcpy(p+1,&len,sizeof(len)); + memrev32ifbe(p+1); } /* Return the difference in number of bytes needed to store the new length @@ -287,12 +291,15 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi if (encoding == ZIP_INT_16B) { i16 = value; memcpy(p,&i16,sizeof(i16)); + memrev16ifbe(p); } else if (encoding == ZIP_INT_32B) { i32 = value; memcpy(p,&i32,sizeof(i32)); + memrev32ifbe(p); } else if (encoding == ZIP_INT_64B) { i64 = value; memcpy(p,&i64,sizeof(i64)); + memrev64ifbe(p); } else { assert(NULL); } @@ -305,12 +312,15 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { int64_t i64, ret = 0; if (encoding == ZIP_INT_16B) { memcpy(&i16,p,sizeof(i16)); + memrev16ifbe(&i16); ret = i16; } else if (encoding == ZIP_INT_32B) { memcpy(&i32,p,sizeof(i32)); + memrev16ifbe(&i32); ret = i32; } else if (encoding == ZIP_INT_64B) { memcpy(&i64,p,sizeof(i64)); + memrev16ifbe(&i64); ret = i64; } else { assert(NULL); diff --git a/src/zipmap.c b/src/zipmap.c index 693db7b9..9f0fc718 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -80,6 +80,7 @@ #include #include #include "zmalloc.h" +#include "endian.h" #define ZIPMAP_BIGLEN 254 #define ZIPMAP_END 255 @@ -108,6 +109,7 @@ static unsigned int zipmapDecodeLength(unsigned char *p) { if (len < ZIPMAP_BIGLEN) return len; memcpy(&len,p+1,sizeof(unsigned int)); + memrev32ifbe(&len); return len; } @@ -123,6 +125,7 @@ static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) { } else { p[0] = ZIPMAP_BIGLEN; memcpy(p+1,&len,sizeof(len)); + memrev32ifbe(p+1); return 1+sizeof(len); } }