From 3d04d29e7fd49902f494754ec8911792fedf1be8 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 12:53:34 +0200
Subject: [PATCH 01/66] extracted general methods to zip.c for reuse in other
 zip* structures

---
 zip.c    |  46 +++++++++++++++++++++++
 zipmap.c | 109 +++++++++++++++++--------------------------------------
 2 files changed, 80 insertions(+), 75 deletions(-)
 create mode 100644 zip.c

diff --git a/zip.c b/zip.c
new file mode 100644
index 00000000..cef7b1a3
--- /dev/null
+++ b/zip.c
@@ -0,0 +1,46 @@
+#define ZIP_BIGLEN 254
+#define ZIP_END 255
+
+/* The following macro returns the number of bytes needed to encode the length
+ * for the integer value _l, that is, 1 byte for lengths < ZIP_BIGLEN and
+ * 5 bytes for all the other lengths. */
+#define ZIP_LEN_BYTES(_l) (((_l) < ZIP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
+
+/* Decode the encoded length pointed by 'p' */
+static unsigned int zipDecodeLength(unsigned char *p) {
+    unsigned int len = *p;
+
+    if (len < ZIP_BIGLEN) return len;
+    memcpy(&len,p+1,sizeof(unsigned int));
+    return len;
+}
+
+/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
+ * the amount of bytes required to encode such a length. */
+static unsigned int zipEncodeLength(unsigned char *p, unsigned int len) {
+    if (p == NULL) {
+        return ZIP_LEN_BYTES(len);
+    } else {
+        if (len < ZIP_BIGLEN) {
+            p[0] = len;
+            return 1;
+        } else {
+            p[0] = ZIP_BIGLEN;
+            memcpy(p+1,&len,sizeof(len));
+            return 1+sizeof(len);
+        }
+    }
+}
+
+/* Return the total amount used by an entry (encoded length + payload). */
+static unsigned int zipRawEntryLength(unsigned char *p) {
+    unsigned int l = zipDecodeLength(p);
+    return zipEncodeLength(NULL,l) + l;
+}
+
+/* Resize the zip* structure. */
+static unsigned char *zipResize(unsigned char *z, unsigned int len) {
+    z = zrealloc(z,len);
+    z[len-1] = ZIP_END;
+    return z;
+}
diff --git a/zipmap.c b/zipmap.c
index 6fd6ca86..7223678a 100644
--- a/zipmap.c
+++ b/zipmap.c
@@ -80,54 +80,21 @@
 #include <string.h>
 #include <assert.h>
 #include "zmalloc.h"
-
-#define ZIPMAP_BIGLEN 254
-#define ZIPMAP_END 255
+#include "zip.c"
 
 /* The following defines the max value for the <free> field described in the
  * comments above, that is, the max number of trailing bytes in a value. */
 #define ZIPMAP_VALUE_MAX_FREE 4
 
-/* The following macro returns the number of bytes needed to encode the length
- * for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
- * 5 bytes for all the other lengths. */
-#define ZIPMAP_LEN_BYTES(_l) (((_l) < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
-
 /* Create a new empty zipmap. */
 unsigned char *zipmapNew(void) {
     unsigned char *zm = zmalloc(2);
 
     zm[0] = 0; /* Length */
-    zm[1] = ZIPMAP_END;
+    zm[1] = ZIP_END;
     return zm;
 }
 
-/* Decode the encoded length pointed by 'p' */
-static unsigned int zipmapDecodeLength(unsigned char *p) {
-    unsigned int len = *p;
-
-    if (len < ZIPMAP_BIGLEN) return len;
-    memcpy(&len,p+1,sizeof(unsigned int));
-    return len;
-}
-
-/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
- * the amount of bytes required to encode such a length. */
-static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
-    if (p == NULL) {
-        return ZIPMAP_LEN_BYTES(len);
-    } else {
-        if (len < ZIPMAP_BIGLEN) {
-            p[0] = len;
-            return 1;
-        } else {
-            p[0] = ZIPMAP_BIGLEN;
-            memcpy(p+1,&len,sizeof(len));
-            return 1+sizeof(len);
-        }
-    }
-}
-
 /* Search for a matching key, returning a pointer to the entry inside the
  * zipmap. Returns NULL if the key is not found.
  *
@@ -138,12 +105,12 @@ static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, uns
     unsigned char *p = zm+1, *k = NULL;
     unsigned int l,llen;
 
-    while(*p != ZIPMAP_END) {
+    while(*p != ZIP_END) {
         unsigned char free;
 
         /* Match or skip the key */
-        l = zipmapDecodeLength(p);
-        llen = zipmapEncodeLength(NULL,l);
+        l = zipDecodeLength(p);
+        llen = zipEncodeLength(NULL,l);
         if (k == NULL && l == klen && !memcmp(p+llen,key,l)) {
             /* Only return when the user doesn't care
              * for the total length of the zipmap. */
@@ -155,8 +122,8 @@ static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, uns
         }
         p += llen+l;
         /* Skip the value as well */
-        l = zipmapDecodeLength(p);
-        p += zipmapEncodeLength(NULL,l);
+        l = zipDecodeLength(p);
+        p += zipEncodeLength(NULL,l);
         free = p[0];
         p += l+1+free; /* +1 to skip the free byte */
     }
@@ -168,25 +135,23 @@ static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen)
     unsigned int l;
 
     l = klen+vlen+3;
-    if (klen >= ZIPMAP_BIGLEN) l += 4;
-    if (vlen >= ZIPMAP_BIGLEN) l += 4;
+    if (klen >= ZIP_BIGLEN) l += 4;
+    if (vlen >= ZIP_BIGLEN) l += 4;
     return l;
 }
 
 /* Return the total amount used by a key (encoded length + payload) */
 static unsigned int zipmapRawKeyLength(unsigned char *p) {
-    unsigned int l = zipmapDecodeLength(p);
-    
-    return zipmapEncodeLength(NULL,l) + l;
+    return zipRawEntryLength(p);
 }
 
 /* Return the total amount used by a value
  * (encoded length + single byte free count + payload) */
 static unsigned int zipmapRawValueLength(unsigned char *p) {
-    unsigned int l = zipmapDecodeLength(p);
+    unsigned int l = zipDecodeLength(p);
     unsigned int used;
     
-    used = zipmapEncodeLength(NULL,l);
+    used = zipEncodeLength(NULL,l);
     used += p[used] + 1 + l;
     return used;
 }
@@ -199,12 +164,6 @@ static unsigned int zipmapRawEntryLength(unsigned char *p) {
     return l + zipmapRawValueLength(p+l);
 }
 
-static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
-    zm = zrealloc(zm, len);
-    zm[len-1] = ZIPMAP_END;
-    return zm;
-}
-
 /* Set key to value, creating the key if it does not already exist.
  * If 'update' is not NULL, *update is set to 1 if the key was
  * already preset, otherwise to 0. */
@@ -219,12 +178,12 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
     p = zipmapLookupRaw(zm,key,klen,&zmlen);
     if (p == NULL) {
         /* Key not found: enlarge */
-        zm = zipmapResize(zm, zmlen+reqlen);
+        zm = zipResize(zm, zmlen+reqlen);
         p = zm+zmlen-1;
         zmlen = zmlen+reqlen;
 
         /* Increase zipmap length (this is an insert) */
-        if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
+        if (zm[0] < ZIP_BIGLEN) zm[0]++;
     } else {
         /* Key found. Is there enough space for the new value? */
         /* Compute the total length: */
@@ -235,7 +194,7 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
              * it can be resized. Then, move the tail backwards so this
              * pair fits at the current position. */
             offset = p-zm;
-            zm = zipmapResize(zm, zmlen-freelen+reqlen);
+            zm = zipResize(zm, zmlen-freelen+reqlen);
             p = zm+offset;
 
             /* The +1 in the number of bytes to be moved is caused by the
@@ -257,7 +216,7 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
         offset = p-zm;
         memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
         zmlen -= empty;
-        zm = zipmapResize(zm, zmlen);
+        zm = zipResize(zm, zmlen);
         p = zm+offset;
         vempty = 0;
     } else {
@@ -266,11 +225,11 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
 
     /* Just write the key + value and we are done. */
     /* Key: */
-    p += zipmapEncodeLength(p,klen);
+    p += zipEncodeLength(p,klen);
     memcpy(p,key,klen);
     p += klen;
     /* Value: */
-    p += zipmapEncodeLength(p,vlen);
+    p += zipEncodeLength(p,vlen);
     *p++ = vempty;
     memcpy(p,val,vlen);
     return zm;
@@ -284,10 +243,10 @@ unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int kle
     if (p) {
         freelen = zipmapRawEntryLength(p);
         memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
-        zm = zipmapResize(zm, zmlen-freelen);
+        zm = zipResize(zm, zmlen-freelen);
 
         /* Decrease zipmap length */
-        if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;
+        if (zm[0] < ZIP_BIGLEN) zm[0]--;
 
         if (deleted) *deleted = 1;
     } else {
@@ -313,17 +272,17 @@ unsigned char *zipmapRewind(unsigned char *zm) {
  * }
  */
 unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
-    if (zm[0] == ZIPMAP_END) return NULL;
+    if (zm[0] == ZIP_END) return NULL;
     if (key) {
         *key = zm;
-        *klen = zipmapDecodeLength(zm);
-        *key += ZIPMAP_LEN_BYTES(*klen);
+        *klen = zipDecodeLength(zm);
+        *key += ZIP_LEN_BYTES(*klen);
     }
     zm += zipmapRawKeyLength(zm);
     if (value) {
         *value = zm+1;
-        *vlen = zipmapDecodeLength(zm);
-        *value += ZIPMAP_LEN_BYTES(*vlen);
+        *vlen = zipDecodeLength(zm);
+        *value += ZIP_LEN_BYTES(*vlen);
     }
     zm += zipmapRawValueLength(zm);
     return zm;
@@ -336,8 +295,8 @@ int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned
 
     if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
     p += zipmapRawKeyLength(p);
-    *vlen = zipmapDecodeLength(p);
-    *value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
+    *vlen = zipDecodeLength(p);
+    *value = p + ZIP_LEN_BYTES(*vlen) + 1;
     return 1;
 }
 
@@ -349,14 +308,14 @@ int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
 /* Return the number of entries inside a zipmap */
 unsigned int zipmapLen(unsigned char *zm) {
     unsigned int len = 0;
-    if (zm[0] < ZIPMAP_BIGLEN) {
+    if (zm[0] < ZIP_BIGLEN) {
         len = zm[0];
     } else {
         unsigned char *p = zipmapRewind(zm);
         while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
 
         /* Re-store length if small enough */
-        if (len < ZIPMAP_BIGLEN) zm[0] = len;
+        if (len < ZIP_BIGLEN) zm[0] = len;
     }
     return len;
 }
@@ -366,21 +325,21 @@ void zipmapRepr(unsigned char *p) {
 
     printf("{status %u}",*p++);
     while(1) {
-        if (p[0] == ZIPMAP_END) {
+        if (p[0] == ZIP_END) {
             printf("{end}");
             break;
         } else {
             unsigned char e;
 
-            l = zipmapDecodeLength(p);
+            l = zipDecodeLength(p);
             printf("{key %u}",l);
-            p += zipmapEncodeLength(NULL,l);
+            p += zipEncodeLength(NULL,l);
             fwrite(p,l,1,stdout);
             p += l;
 
-            l = zipmapDecodeLength(p);
+            l = zipDecodeLength(p);
             printf("{value %u}",l);
-            p += zipmapEncodeLength(NULL,l);
+            p += zipEncodeLength(NULL,l);
             e = *p++;
             fwrite(p,l,1,stdout);
             p += l+e;

From 11ac6ff613387b4bafa9b405974653112228e96f Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 12:54:01 +0200
Subject: [PATCH 02/66] implementation for a ziplist with push and pop support

---
 ziplist.c | 150 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 ziplist.h |   2 +
 2 files changed, 152 insertions(+)
 create mode 100644 ziplist.c
 create mode 100644 ziplist.h

diff --git a/ziplist.c b/ziplist.c
new file mode 100644
index 00000000..a71fd888
--- /dev/null
+++ b/ziplist.c
@@ -0,0 +1,150 @@
+/* Memory layout of a ziplist, containing "foo", "bar", "quux":
+ * <zlbytes><zllen><len>"foo"<len>"bar"<len>"quux"
+ *
+ * <zlbytes> is an unsigned integer to hold the number of bytes that
+ * the ziplist occupies. This is stored to not have to traverse the ziplist
+ * to know the new length when pushing.
+ *
+ * <zllen> is the number of items in the ziplist. When this value is
+ * greater than 254, we need to traverse the entire list to know
+ * how many items it holds.
+ *
+ * <len> is the number of bytes occupied by a single entry. When this
+ * number is greater than 253, the length will occupy 5 bytes, where
+ * the extra bytes contain an unsigned integer to hold the length.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include "zmalloc.h"
+#include "sds.h"
+#include "ziplist.h"
+#include "zip.c"
+
+#define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
+#define ZIPLIST_LENGTH(zl) (*((zl)+sizeof(unsigned int)))
+#define ZIPLIST_HEADER_SIZE (sizeof(unsigned int)+1)
+
+/* Create a new empty ziplist. */
+unsigned char *ziplistNew(void) {
+    unsigned int bytes = ZIPLIST_HEADER_SIZE+1;
+    unsigned char *zl = zmalloc(bytes);
+    ZIPLIST_BYTES(zl) = bytes;
+    ZIPLIST_LENGTH(zl) = 0;
+    zl[bytes-1] = ZIP_END;
+    return zl;
+}
+
+static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
+    zl = zipResize(zl,len);
+    ZIPLIST_BYTES(zl) = len;
+    zl[len-1] = ZIP_END;
+    return zl;
+}
+
+static unsigned char *ziplistHead(unsigned char *zl) {
+    return zl+ZIPLIST_HEADER_SIZE;
+}
+
+static unsigned char *ziplistTail(unsigned char *zl) {
+    unsigned char *p, *q;
+    p = q = ziplistHead(zl);
+    while (*p != ZIP_END) {
+        q = p;
+        p += zipRawEntryLength(p);
+    }
+    return q;
+}
+
+unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where) {
+    unsigned int curlen = ZIPLIST_BYTES(zl);
+    unsigned int reqlen = zipEncodeLength(NULL,elen)+elen;
+    unsigned char *p;
+
+    /* Resize the ziplist */
+    zl = ziplistResize(zl,curlen+reqlen);
+
+    if (where == ZIPLIST_HEAD) {
+        p = zl+ZIPLIST_HEADER_SIZE;
+        if (*p != ZIP_END) {
+            /* Subtract one because of the ZIP_END bytes */
+            memmove(p+reqlen,p,curlen-ZIPLIST_HEADER_SIZE-1);
+        }
+    } else {
+        p = zl+curlen-1;
+    }
+
+    /* Increase length */
+    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)++;
+
+    /* Write the entry */
+    p += zipEncodeLength(p,elen);
+    memcpy(p,entry,elen);
+    return zl;
+}
+
+unsigned char *ziplistPop(unsigned char *zl, sds *value, int where) {
+    unsigned int curlen = ZIPLIST_BYTES(zl), len, rlen;
+    unsigned char *p;
+    *value = NULL;
+
+    /* Get pointer to element to remove */
+    p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : ziplistTail(zl);
+    if (*p == ZIP_END) return zl;
+    len = zipDecodeLength(p);
+    *value = sdsnewlen(p+zipEncodeLength(NULL,len),len);
+
+    /* Move list to front when popping from the head */
+    rlen = zipRawEntryLength(p);
+    if (where == ZIPLIST_HEAD) {
+        memmove(p,p+rlen,curlen-ZIPLIST_HEADER_SIZE-len);
+    }
+
+    /* Resize and update length */
+    zl = ziplistResize(zl,curlen-rlen);
+    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)--;
+    return zl;
+}
+
+void ziplistRepr(unsigned char *zl) {
+    unsigned char *p;
+    unsigned int l;
+
+    printf("{bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
+    p = ziplistHead(zl);
+    while(*p != ZIP_END) {
+        l = zipDecodeLength(p);
+        printf("{key %u}",l);
+        p += zipEncodeLength(NULL,l);
+        fwrite(p,l,1,stdout);
+        printf("\n");
+        p += l;
+    }
+    printf("{end}\n\n");
+}
+
+#ifdef ZIPLIST_TEST_MAIN
+int main(int argc, char **argv) {
+    unsigned char *zl;
+    sds s;
+
+    zl = ziplistNew();
+    zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
+    ziplistRepr(zl);
+    zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
+    ziplistRepr(zl);
+    zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
+    ziplistRepr(zl);
+
+    zl = ziplistPop(zl, &s, ZIPLIST_TAIL);
+    printf("Pop tail: %s (length %ld)\n", s, sdslen(s));
+    ziplistRepr(zl);
+
+    zl = ziplistPop(zl, &s, ZIPLIST_HEAD);
+    printf("Pop head: %s (length %ld)\n", s, sdslen(s));
+    ziplistRepr(zl);
+
+    return 0;
+}
+#endif
diff --git a/ziplist.h b/ziplist.h
new file mode 100644
index 00000000..bfc1854a
--- /dev/null
+++ b/ziplist.h
@@ -0,0 +1,2 @@
+#define ZIPLIST_HEAD 0
+#define ZIPLIST_TAIL 1

From 08253bf42b7f1fdc000dedfb2580bc9cb375449d Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 13:16:08 +0200
Subject: [PATCH 03/66] code to iterate over a ziplist

---
 ziplist.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 76 insertions(+), 7 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index a71fd888..112159ff 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -107,6 +107,30 @@ unsigned char *ziplistPop(unsigned char *zl, sds *value, int where) {
     return zl;
 }
 
+/* Returns an offset to use for iterating with ziplistNext. */
+unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
+    unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
+    unsigned int i = 0;
+    for (; i < index; i++) {
+        if (*p == ZIP_END) break;
+        p += zipRawEntryLength(p);
+    }
+    return p;
+}
+
+/* Store entry at current position in sds *value and return pointer
+ * to the next entry. */
+unsigned char *ziplistNext(unsigned char *p, sds *value) {
+    if (*p == ZIP_END) return NULL;
+    if (value) {
+        unsigned int len;
+        len = zipDecodeLength(p);
+        *value = sdsnewlen(p+zipEncodeLength(NULL,len),len);
+    }
+    p += zipRawEntryLength(p);
+    return p;
+}
+
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p;
     unsigned int l;
@@ -125,16 +149,20 @@ void ziplistRepr(unsigned char *zl) {
 }
 
 #ifdef ZIPLIST_TEST_MAIN
+
+unsigned char *createList() {
+    unsigned char *zl = ziplistNew();
+    zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
+    return zl;
+}
+
 int main(int argc, char **argv) {
-    unsigned char *zl;
+    unsigned char *zl, *p;
     sds s;
 
-    zl = ziplistNew();
-    zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
-    ziplistRepr(zl);
-    zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
-    ziplistRepr(zl);
-    zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
+    zl = createList();
     ziplistRepr(zl);
 
     zl = ziplistPop(zl, &s, ZIPLIST_TAIL);
@@ -145,6 +173,47 @@ int main(int argc, char **argv) {
     printf("Pop head: %s (length %ld)\n", s, sdslen(s));
     ziplistRepr(zl);
 
+    printf("Iterate list from 0 to end:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 0);
+        while ((p = ziplistNext(p, &s)) != NULL) {
+            printf("Entry: %s (length %ld)\n", s, sdslen(s));
+        }
+        printf("\n");
+    }
+
+    printf("Iterate list from 1 to end:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 1);
+        while ((p = ziplistNext(p, &s)) != NULL) {
+            printf("Entry: %s (length %ld)\n", s, sdslen(s));
+        }
+        printf("\n");
+    }
+
+    printf("Iterate list from 2 to end:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 2);
+        while ((p = ziplistNext(p, &s)) != NULL) {
+            printf("Entry: %s (length %ld)\n", s, sdslen(s));
+        }
+        printf("\n");
+    }
+
+    printf("Iterate starting out of range:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 3);
+        if (ziplistNext(p, &s) == NULL) {
+            printf("No entry\n");
+        } else {
+            printf("ERROR\n");
+        }
+    }
+
     return 0;
 }
 #endif

From 335d16bc0f8f18f0da929bf3be725b42287c1042 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 13:29:14 +0200
Subject: [PATCH 04/66] change iteration code to avoid allocating a new sds for
 each traversed entry

---
 ziplist.c | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 112159ff..fefdbb7e 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -120,12 +120,11 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
 
 /* Store entry at current position in sds *value and return pointer
  * to the next entry. */
-unsigned char *ziplistNext(unsigned char *p, sds *value) {
+unsigned char *ziplistNext(unsigned char *p, unsigned char **entry, unsigned int *elen) {
     if (*p == ZIP_END) return NULL;
-    if (value) {
-        unsigned int len;
-        len = zipDecodeLength(p);
-        *value = sdsnewlen(p+zipEncodeLength(NULL,len),len);
+    if (entry) {
+        *elen = zipDecodeLength(p);
+        *entry = p+ZIP_LEN_BYTES(*elen);
     }
     p += zipRawEntryLength(p);
     return p;
@@ -159,7 +158,8 @@ unsigned char *createList() {
 }
 
 int main(int argc, char **argv) {
-    unsigned char *zl, *p;
+    unsigned char *zl, *p, *entry;
+    unsigned int elen;
     sds s;
 
     zl = createList();
@@ -177,8 +177,10 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 0);
-        while ((p = ziplistNext(p, &s)) != NULL) {
-            printf("Entry: %s (length %ld)\n", s, sdslen(s));
+        while ((p = ziplistNext(p, &entry, &elen)) != NULL) {
+            printf("Entry: ");
+            fwrite(entry,elen,1,stdout);
+            printf(" (length %d)\n", elen);
         }
         printf("\n");
     }
@@ -187,8 +189,10 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 1);
-        while ((p = ziplistNext(p, &s)) != NULL) {
-            printf("Entry: %s (length %ld)\n", s, sdslen(s));
+        while ((p = ziplistNext(p, &entry, &elen)) != NULL) {
+            printf("Entry: ");
+            fwrite(entry,elen,1,stdout);
+            printf(" (length %d)\n", elen);
         }
         printf("\n");
     }
@@ -197,8 +201,10 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 2);
-        while ((p = ziplistNext(p, &s)) != NULL) {
-            printf("Entry: %s (length %ld)\n", s, sdslen(s));
+        while ((p = ziplistNext(p, &entry, &elen)) != NULL) {
+            printf("Entry: ");
+            fwrite(entry,elen,1,stdout);
+            printf(" (length %d)\n", elen);
         }
         printf("\n");
     }
@@ -207,7 +213,7 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 3);
-        if (ziplistNext(p, &s) == NULL) {
+        if (ziplistNext(p, &entry, &elen) == NULL) {
             printf("No entry\n");
         } else {
             printf("ERROR\n");

From 33c1269efc86a603c54c1045edc9402488a7224f Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 14:11:09 +0200
Subject: [PATCH 05/66] check if *value is non-NULL before setting it

---
 ziplist.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index fefdbb7e..a754e0fb 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -87,13 +87,13 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
 unsigned char *ziplistPop(unsigned char *zl, sds *value, int where) {
     unsigned int curlen = ZIPLIST_BYTES(zl), len, rlen;
     unsigned char *p;
-    *value = NULL;
+    if (value) *value = NULL;
 
     /* Get pointer to element to remove */
     p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : ziplistTail(zl);
     if (*p == ZIP_END) return zl;
     len = zipDecodeLength(p);
-    *value = sdsnewlen(p+zipEncodeLength(NULL,len),len);
+    if (value) *value = sdsnewlen(p+zipEncodeLength(NULL,len),len);
 
     /* Move list to front when popping from the head */
     rlen = zipRawEntryLength(p);

From 779deb60f92ab2980c41f8cb408315fbac2fa54b Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 14:11:29 +0200
Subject: [PATCH 06/66] code to delete an inner range from the ziplist

---
 ziplist.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/ziplist.c b/ziplist.c
index a754e0fb..5c822187 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -130,6 +130,27 @@ unsigned char *ziplistNext(unsigned char *p, unsigned char **entry, unsigned int
     return p;
 }
 
+/* Delete one or more entries from the ziplist. */
+unsigned char *ziplistDelete(unsigned char *zl, unsigned int index, unsigned int num) {
+    unsigned char *p, *first = ziplistIndex(zl, index);
+    unsigned int i, deleted = 0, totlen, newlen;
+    for (p = first, i = 0; *p != ZIP_END && i < num; i++) {
+        p += zipRawEntryLength(p);
+        deleted++;
+    }
+
+    totlen = p-first;
+    if (totlen > 0) {
+        /* Move current tail to the new tail when there *is* a tail */
+        if (*p != ZIP_END) memmove(first,p,ZIPLIST_BYTES(zl)-(p-zl)-1);
+
+        /* Resize and update length */
+        zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen);
+        if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl) -= deleted;
+    }
+    return zl;
+}
+
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p;
     unsigned int l;
@@ -218,6 +239,42 @@ int main(int argc, char **argv) {
         } else {
             printf("ERROR\n");
         }
+        printf("\n");
+    }
+
+    printf("Delete inclusive range 0,0:\n");
+    {
+        zl = createList();
+        zl = ziplistDelete(zl, 0, 1);
+        ziplistRepr(zl);
+    }
+
+    printf("Delete inclusive range 0,1:\n");
+    {
+        zl = createList();
+        zl = ziplistDelete(zl, 0, 2);
+        ziplistRepr(zl);
+    }
+
+    printf("Delete inclusive range 1,2:\n");
+    {
+        zl = createList();
+        zl = ziplistDelete(zl, 1, 2);
+        ziplistRepr(zl);
+    }
+
+    printf("Delete with start index out of range:\n");
+    {
+        zl = createList();
+        zl = ziplistDelete(zl, 5, 1);
+        ziplistRepr(zl);
+    }
+
+    printf("Delete with num overflow:\n");
+    {
+        zl = createList();
+        zl = ziplistDelete(zl, 1, 5);
+        ziplistRepr(zl);
     }
 
     return 0;

From ba5b4bde21de6fb0fbc73dc970eaff71050a26a6 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 15:44:47 +0200
Subject: [PATCH 07/66] rename ziplistDelete to ziplistDeleteRange

---
 ziplist.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 5c822187..c1278b65 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -130,8 +130,8 @@ unsigned char *ziplistNext(unsigned char *p, unsigned char **entry, unsigned int
     return p;
 }
 
-/* Delete one or more entries from the ziplist. */
-unsigned char *ziplistDelete(unsigned char *zl, unsigned int index, unsigned int num) {
+/* Delete a range of entries from the ziplist. */
+unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
     unsigned char *p, *first = ziplistIndex(zl, index);
     unsigned int i, deleted = 0, totlen, newlen;
     for (p = first, i = 0; *p != ZIP_END && i < num; i++) {
@@ -245,35 +245,35 @@ int main(int argc, char **argv) {
     printf("Delete inclusive range 0,0:\n");
     {
         zl = createList();
-        zl = ziplistDelete(zl, 0, 1);
+        zl = ziplistDeleteRange(zl, 0, 1);
         ziplistRepr(zl);
     }
 
     printf("Delete inclusive range 0,1:\n");
     {
         zl = createList();
-        zl = ziplistDelete(zl, 0, 2);
+        zl = ziplistDeleteRange(zl, 0, 2);
         ziplistRepr(zl);
     }
 
     printf("Delete inclusive range 1,2:\n");
     {
         zl = createList();
-        zl = ziplistDelete(zl, 1, 2);
+        zl = ziplistDeleteRange(zl, 1, 2);
         ziplistRepr(zl);
     }
 
     printf("Delete with start index out of range:\n");
     {
         zl = createList();
-        zl = ziplistDelete(zl, 5, 1);
+        zl = ziplistDeleteRange(zl, 5, 1);
         ziplistRepr(zl);
     }
 
     printf("Delete with num overflow:\n");
     {
         zl = createList();
-        zl = ziplistDelete(zl, 1, 5);
+        zl = ziplistDeleteRange(zl, 1, 5);
         ziplistRepr(zl);
     }
 

From 924727d905ec6943857f7a0832196f595884d461 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 15:45:30 +0200
Subject: [PATCH 08/66] allow pointer to be stored to current element when
 iterating over ziplist

---
 ziplist.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index c1278b65..0987bcca 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -120,12 +120,13 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
 
 /* Store entry at current position in sds *value and return pointer
  * to the next entry. */
-unsigned char *ziplistNext(unsigned char *p, unsigned char **entry, unsigned int *elen) {
+unsigned char *ziplistNext(unsigned char *p, unsigned char **q, unsigned char **entry, unsigned int *elen) {
     if (*p == ZIP_END) return NULL;
     if (entry) {
         *elen = zipDecodeLength(p);
         *entry = p+ZIP_LEN_BYTES(*elen);
     }
+    if (q != NULL) *q = p;
     p += zipRawEntryLength(p);
     return p;
 }
@@ -198,7 +199,7 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 0);
-        while ((p = ziplistNext(p, &entry, &elen)) != NULL) {
+        while ((p = ziplistNext(p, NULL, &entry, &elen)) != NULL) {
             printf("Entry: ");
             fwrite(entry,elen,1,stdout);
             printf(" (length %d)\n", elen);
@@ -210,7 +211,7 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 1);
-        while ((p = ziplistNext(p, &entry, &elen)) != NULL) {
+        while ((p = ziplistNext(p, NULL, &entry, &elen)) != NULL) {
             printf("Entry: ");
             fwrite(entry,elen,1,stdout);
             printf(" (length %d)\n", elen);
@@ -222,7 +223,7 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 2);
-        while ((p = ziplistNext(p, &entry, &elen)) != NULL) {
+        while ((p = ziplistNext(p, NULL, &entry, &elen)) != NULL) {
             printf("Entry: ");
             fwrite(entry,elen,1,stdout);
             printf(" (length %d)\n", elen);
@@ -234,7 +235,7 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 3);
-        if (ziplistNext(p, &entry, &elen) == NULL) {
+        if (ziplistNext(p, &entry, NULL, &elen) == NULL) {
             printf("No entry\n");
         } else {
             printf("ERROR\n");

From 0f10458c4f1b9a7397bf72e4f787041069cc9e8a Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 21 May 2010 15:46:00 +0200
Subject: [PATCH 09/66] allow entries to be deleted in place when iterating
 over a ziplist

---
 ziplist.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index 0987bcca..68579563 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -152,6 +152,27 @@ unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigne
     return zl;
 }
 
+/* Delete a single entry from the ziplist, pointed to by *p.
+ * Also update *p in place, to be able to iterate over the
+ * ziplist, while deleting entries. */
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
+    unsigned int offset = *p-zl, tail, len;
+    len = zipRawEntryLength(*p);
+    tail = ZIPLIST_BYTES(zl)-offset-len-1;
+
+    /* Move current tail to the new tail when there *is* a tail */
+    if (tail > 0) memmove(*p,*p+len,tail);
+
+    /* Resize and update length */
+    zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-len);
+    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)--;
+
+    /* Store new pointer to current element in p.
+     * This needs to be done because zl can change on realloc. */
+    *p = zl+offset;
+    return zl;
+}
+
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p;
     unsigned int l;
@@ -180,7 +201,7 @@ unsigned char *createList() {
 }
 
 int main(int argc, char **argv) {
-    unsigned char *zl, *p, *entry;
+    unsigned char *zl, *p, *q, *entry;
     unsigned int elen;
     sds s;
 
@@ -278,6 +299,26 @@ int main(int argc, char **argv) {
         ziplistRepr(zl);
     }
 
+    printf("Delete foo while iterating:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 0);
+        while ((p = ziplistNext(p, &q, &entry, &elen)) != NULL) {
+            if (strncmp("foo", entry, elen) == 0) {
+                printf("Delete foo\n");
+                zl = ziplistDelete(zl, &q);
+                p = q;
+            } else {
+                printf("Entry: ");
+                fwrite(entry,elen,1,stdout);
+                printf(" (length %d)\n", elen);
+            }
+        }
+        printf("\n");
+        ziplistRepr(zl);
+        printf("\n");
+    }
+
     return 0;
 }
 #endif

From f6eb17477aa6394adc699b5ec45705df9c584363 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 14:58:58 +0200
Subject: [PATCH 10/66] move length housekeeping to a macro

---
 ziplist.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 68579563..596217d5 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -25,6 +25,8 @@
 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
 #define ZIPLIST_LENGTH(zl) (*((zl)+sizeof(unsigned int)))
 #define ZIPLIST_HEADER_SIZE (sizeof(unsigned int)+1)
+#define ZIPLIST_INCR_LENGTH(zl,incr) { \
+    if (ZIPLIST_LENGTH(zl) < (ZIP_END-1)) ZIPLIST_LENGTH(zl)+=incr; }
 
 /* Create a new empty ziplist. */
 unsigned char *ziplistNew(void) {
@@ -75,12 +77,10 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
         p = zl+curlen-1;
     }
 
-    /* Increase length */
-    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)++;
-
     /* Write the entry */
     p += zipEncodeLength(p,elen);
     memcpy(p,entry,elen);
+    ZIPLIST_INCR_LENGTH(zl,1);
     return zl;
 }
 
@@ -103,7 +103,7 @@ unsigned char *ziplistPop(unsigned char *zl, sds *value, int where) {
 
     /* Resize and update length */
     zl = ziplistResize(zl,curlen-rlen);
-    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)--;
+    ZIPLIST_INCR_LENGTH(zl,-1);
     return zl;
 }
 
@@ -147,7 +147,7 @@ unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigne
 
         /* Resize and update length */
         zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen);
-        if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl) -= deleted;
+        ZIPLIST_INCR_LENGTH(zl,-deleted);
     }
     return zl;
 }
@@ -165,7 +165,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
 
     /* Resize and update length */
     zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-len);
-    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)--;
+    ZIPLIST_INCR_LENGTH(zl,-1);
 
     /* Store new pointer to current element in p.
      * This needs to be done because zl can change on realloc. */

From 29b14d5face8a434810794f524c8449581166adf Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 15:07:12 +0200
Subject: [PATCH 11/66] initial work for integer encoding in ziplists

---
 zip.c     | 154 ++++++++++++++++++++++++++++++++++++++++++++++--------
 ziplist.c |  98 ++++++++++++++++++++++++++--------
 2 files changed, 208 insertions(+), 44 deletions(-)

diff --git a/zip.c b/zip.c
index cef7b1a3..e8f01edb 100644
--- a/zip.c
+++ b/zip.c
@@ -1,41 +1,153 @@
 #define ZIP_BIGLEN 254
 #define ZIP_END 255
 
-/* The following macro returns the number of bytes needed to encode the length
- * for the integer value _l, that is, 1 byte for lengths < ZIP_BIGLEN and
- * 5 bytes for all the other lengths. */
-#define ZIP_LEN_BYTES(_l) (((_l) < ZIP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
+/* Entry encoding */
+#define ZIP_ENC_RAW     0
+#define ZIP_ENC_SHORT   1
+#define ZIP_ENC_INT     2
+#define ZIP_ENC_LLONG   3
+#define ZIP_ENCODING(p) ((p)[0] >> 6)
 
-/* Decode the encoded length pointed by 'p' */
-static unsigned int zipDecodeLength(unsigned char *p) {
-    unsigned int len = *p;
+/* Length encoding for raw entries */
+#define ZIP_LEN_INLINE  0
+#define ZIP_LEN_UINT16  1
+#define ZIP_LEN_UINT32  2
 
-    if (len < ZIP_BIGLEN) return len;
-    memcpy(&len,p+1,sizeof(unsigned int));
+static unsigned int zipEncodingSize(char encoding) {
+    if (encoding == ZIP_ENC_SHORT) {
+        return sizeof(short int);
+    } else if (encoding == ZIP_ENC_INT) {
+        return sizeof(int);
+    } else if (encoding == ZIP_ENC_LLONG) {
+        return sizeof(long long);
+    }
+    assert(NULL);
+}
+
+/* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
+ * provided, it is set to the number of bytes required to encode the length. */
+static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) {
+    unsigned char encoding = ZIP_ENCODING(p), lenenc;
+    unsigned int len;
+
+    if (encoding == ZIP_ENC_RAW) {
+        lenenc = (p[0] >> 4) & 0x3;
+        if (lenenc == ZIP_LEN_INLINE) {
+            len = p[0] & 0xf;
+            if (lensize) *lensize = 1;
+        } else if (lenenc == ZIP_LEN_UINT16) {
+            len = p[1] | (p[2] << 8);
+            if (lensize) *lensize = 3;
+        } else {
+            len = p[1] | (p[2] << 8) | (p[3] << 16) | (p[4] << 24);
+            if (lensize) *lensize = 5;
+        }
+    } else {
+        len = zipEncodingSize(encoding);
+        if (lensize) *lensize = 1;
+    }
     return len;
 }
 
 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
  * the amount of bytes required to encode such a length. */
-static unsigned int zipEncodeLength(unsigned char *p, unsigned int len) {
-    if (p == NULL) {
-        return ZIP_LEN_BYTES(len);
-    } else {
-        if (len < ZIP_BIGLEN) {
-            p[0] = len;
-            return 1;
+static unsigned int zipEncodeLength(unsigned char *p, char encoding, unsigned int rawlen) {
+    unsigned char len = 1, lenenc, buf[5];
+    if (encoding == ZIP_ENC_RAW) {
+        if (rawlen <= 0xf) {
+            if (!p) return len;
+            lenenc = ZIP_LEN_INLINE;
+            buf[0] = rawlen;
+        } else if (rawlen <= 0xffff) {
+            len += 2;
+            if (!p) return len;
+            lenenc = ZIP_LEN_UINT16;
+            buf[1] = (rawlen     ) & 0xff;
+            buf[2] = (rawlen >> 8) & 0xff;
         } else {
-            p[0] = ZIP_BIGLEN;
-            memcpy(p+1,&len,sizeof(len));
-            return 1+sizeof(len);
+            len += 4;
+            if (!p) return len;
+            lenenc = ZIP_LEN_UINT32;
+            buf[1] = (rawlen      ) & 0xff;
+            buf[2] = (rawlen >>  8) & 0xff;
+            buf[3] = (rawlen >> 16) & 0xff;
+            buf[4] = (rawlen >> 24) & 0xff;
         }
+        buf[0] = (lenenc << 4) | (buf[0] & 0xf);
     }
+    if (!p) return len;
+
+    /* Apparently we need to store the length in 'p' */
+    buf[0] = (encoding << 6) | (buf[0] & 0x3f);
+    memcpy(p,buf,len);
+    return len;
+}
+
+/* Check if string pointed to by 'entry' can be encoded as an integer.
+ * Stores the integer value in 'v' and its encoding in 'encoding'.
+ * Warning: this function requires a NULL-terminated string! */
+static int zipTryEncoding(unsigned char *entry, long long *v, char *encoding) {
+    long long value;
+    char *eptr;
+
+    if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
+        value = strtoll(entry,&eptr,10);
+        if (eptr[0] != '\0') return 0;
+        if (value >= SHRT_MIN && value <= SHRT_MAX) {
+            *encoding = ZIP_ENC_SHORT;
+        } else if (value >= INT_MIN && value <= INT_MAX) {
+            *encoding = ZIP_ENC_INT;
+        } else {
+            *encoding = ZIP_ENC_LLONG;
+        }
+        *v = value;
+        return 1;
+    }
+    return 0;
+}
+
+static void zipSaveInteger(unsigned char *p, long long value, char encoding) {
+    short int s;
+    int i;
+    long long l;
+    if (encoding == ZIP_ENC_SHORT) {
+        s = value;
+        memcpy(p,&s,sizeof(s));
+    } else if (encoding == ZIP_ENC_INT) {
+        i = value;
+        memcpy(p,&i,sizeof(i));
+    } else if (encoding == ZIP_ENC_LLONG) {
+        l = value;
+        memcpy(p,&l,sizeof(l));
+    } else {
+        assert(NULL);
+    }
+}
+
+static long long zipLoadInteger(unsigned char *p, char encoding) {
+    short int s;
+    int i;
+    long long l, ret;
+    if (encoding == ZIP_ENC_SHORT) {
+        memcpy(&s,p,sizeof(s));
+        ret = s;
+    } else if (encoding == ZIP_ENC_INT) {
+        memcpy(&i,p,sizeof(i));
+        ret = i;
+    } else if (encoding == ZIP_ENC_LLONG) {
+        memcpy(&l,p,sizeof(l));
+        ret = l;
+    } else {
+        assert(NULL);
+    }
+    return ret;
 }
 
 /* Return the total amount used by an entry (encoded length + payload). */
 static unsigned int zipRawEntryLength(unsigned char *p) {
-    unsigned int l = zipDecodeLength(p);
-    return zipEncodeLength(NULL,l) + l;
+    unsigned int lensize, len;
+    len = zipDecodeLength(p, &lensize);
+    return lensize + len;
 }
 
 /* Resize the zip* structure. */
diff --git a/ziplist.c b/ziplist.c
index 596217d5..a45ae339 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -15,8 +15,10 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <limits.h>
 #include "zmalloc.h"
 #include "sds.h"
 #include "ziplist.h"
@@ -60,13 +62,21 @@ static unsigned char *ziplistTail(unsigned char *zl) {
 }
 
 unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where) {
-    unsigned int curlen = ZIPLIST_BYTES(zl);
-    unsigned int reqlen = zipEncodeLength(NULL,elen)+elen;
+    unsigned int curlen = ZIPLIST_BYTES(zl), reqlen;
     unsigned char *p;
+    char encoding = ZIP_ENC_RAW;
+    long long value;
 
-    /* Resize the ziplist */
+    /* See if the entry can be encoded */
+    if (zipTryEncoding(entry,&value,&encoding)) {
+        reqlen = zipEncodingSize(encoding);
+    } else {
+        reqlen = elen;
+    }
+    reqlen += zipEncodeLength(NULL,encoding,elen);
+
+    /* Resize the ziplist and move if needed */
     zl = ziplistResize(zl,curlen+reqlen);
-
     if (where == ZIPLIST_HEAD) {
         p = zl+ZIPLIST_HEADER_SIZE;
         if (*p != ZIP_END) {
@@ -78,31 +88,44 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
     }
 
     /* Write the entry */
-    p += zipEncodeLength(p,elen);
-    memcpy(p,entry,elen);
+    p += zipEncodeLength(p,encoding,elen);
+    if (encoding != ZIP_ENC_RAW) {
+        zipSaveInteger(p,value,encoding);
+    } else {
+        memcpy(p,entry,elen);
+    }
     ZIPLIST_INCR_LENGTH(zl,1);
     return zl;
 }
 
-unsigned char *ziplistPop(unsigned char *zl, sds *value, int where) {
-    unsigned int curlen = ZIPLIST_BYTES(zl), len, rlen;
+unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
+    unsigned int curlen = ZIPLIST_BYTES(zl), rawlen;
+    unsigned int len, lensize;
     unsigned char *p;
-    if (value) *value = NULL;
+    long long value;
+    if (target) *target = NULL;
 
     /* Get pointer to element to remove */
     p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : ziplistTail(zl);
     if (*p == ZIP_END) return zl;
-    len = zipDecodeLength(p);
-    if (value) *value = sdsnewlen(p+zipEncodeLength(NULL,len),len);
+    len = zipDecodeLength(p,&lensize);
+    if (target) {
+        if (ZIP_ENCODING(p) == ZIP_ENC_RAW) {
+            *target = sdsnewlen(p+lensize,len);
+        } else {
+            value = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
+            *target = sdscatprintf(sdsempty(), "%lld", value);
+        }
+    }
 
     /* Move list to front when popping from the head */
-    rlen = zipRawEntryLength(p);
+    rawlen = lensize+len;
     if (where == ZIPLIST_HEAD) {
-        memmove(p,p+rlen,curlen-ZIPLIST_HEADER_SIZE-len);
+        memmove(p,p+rawlen,curlen-ZIPLIST_HEADER_SIZE-len);
     }
 
     /* Resize and update length */
-    zl = ziplistResize(zl,curlen-rlen);
+    zl = ziplistResize(zl,curlen-rawlen);
     ZIPLIST_INCR_LENGTH(zl,-1);
     return zl;
 }
@@ -121,10 +144,11 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
 /* Store entry at current position in sds *value and return pointer
  * to the next entry. */
 unsigned char *ziplistNext(unsigned char *p, unsigned char **q, unsigned char **entry, unsigned int *elen) {
+    unsigned int lensize;
     if (*p == ZIP_END) return NULL;
     if (entry) {
-        *elen = zipDecodeLength(p);
-        *entry = p+ZIP_LEN_BYTES(*elen);
+        *elen = zipDecodeLength(p,&lensize);
+        *entry = p+lensize;
     }
     if (q != NULL) *q = p;
     p += zipRawEntryLength(p);
@@ -174,16 +198,22 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
 }
 
 void ziplistRepr(unsigned char *zl) {
-    unsigned char *p;
-    unsigned int l;
+    unsigned char *p, encoding;
+    unsigned int l, lsize;
+    long long value;
 
-    printf("{bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
+    printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
     p = ziplistHead(zl);
     while(*p != ZIP_END) {
-        l = zipDecodeLength(p);
-        printf("{key %u}",l);
-        p += zipEncodeLength(NULL,l);
-        fwrite(p,l,1,stdout);
+        l = zipDecodeLength(p,&lsize);
+        printf("{header %u, payload %u} ",lsize,l);
+        encoding = ZIP_ENCODING(p);
+        p += lsize;
+        if (encoding == ZIP_ENC_RAW) {
+            fwrite(p,l,1,stdout);
+        } else {
+            printf("%lld", zipLoadInteger(p,encoding));
+        }
         printf("\n");
         p += l;
     }
@@ -200,11 +230,33 @@ unsigned char *createList() {
     return zl;
 }
 
+unsigned char *createIntList() {
+    unsigned char *zl = ziplistNew();
+    char buf[32];
+
+    sprintf(buf, "100");
+    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    sprintf(buf, "128000");
+    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    sprintf(buf, "-100");
+    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_HEAD);
+    sprintf(buf, "4294967296");
+    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_HEAD);
+    sprintf(buf, "non integer");
+    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    sprintf(buf, "much much longer non integer");
+    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    return zl;
+}
+
 int main(int argc, char **argv) {
     unsigned char *zl, *p, *q, *entry;
     unsigned int elen;
     sds s;
 
+    zl = createIntList();
+    ziplistRepr(zl);
+
     zl = createList();
     ziplistRepr(zl);
 

From af5f66fb98d2cb3c62a6aa66bf468559f5e0e04f Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 15:13:09 +0200
Subject: [PATCH 12/66] partial revert of c80df5 because ziplist functions are
 starting to divert too much from zipmap functions

---
 zipmap.c | 108 +++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 74 insertions(+), 34 deletions(-)

diff --git a/zipmap.c b/zipmap.c
index 7223678a..35faeabe 100644
--- a/zipmap.c
+++ b/zipmap.c
@@ -80,21 +80,54 @@
 #include <string.h>
 #include <assert.h>
 #include "zmalloc.h"
-#include "zip.c"
+
+#define ZIPMAP_BIGLEN 254
+#define ZIPMAP_END 255
 
 /* The following defines the max value for the <free> field described in the
  * comments above, that is, the max number of trailing bytes in a value. */
 #define ZIPMAP_VALUE_MAX_FREE 4
 
+/* The following macro returns the number of bytes needed to encode the length
+ * for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
+ * 5 bytes for all the other lengths. */
+#define ZIPMAP_LEN_BYTES(_l) (((_l) < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
+
 /* Create a new empty zipmap. */
 unsigned char *zipmapNew(void) {
     unsigned char *zm = zmalloc(2);
 
     zm[0] = 0; /* Length */
-    zm[1] = ZIP_END;
+    zm[1] = ZIPMAP_END;
     return zm;
 }
 
+/* Decode the encoded length pointed by 'p' */
+static unsigned int zipmapDecodeLength(unsigned char *p) {
+    unsigned int len = *p;
+
+    if (len < ZIPMAP_BIGLEN) return len;
+    memcpy(&len,p+1,sizeof(unsigned int));
+    return len;
+}
+
+/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
+ * the amount of bytes required to encode such a length. */
+static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
+    if (p == NULL) {
+        return ZIPMAP_LEN_BYTES(len);
+    } else {
+        if (len < ZIPMAP_BIGLEN) {
+            p[0] = len;
+            return 1;
+        } else {
+            p[0] = ZIPMAP_BIGLEN;
+            memcpy(p+1,&len,sizeof(len));
+            return 1+sizeof(len);
+        }
+    }
+}
+
 /* Search for a matching key, returning a pointer to the entry inside the
  * zipmap. Returns NULL if the key is not found.
  *
@@ -105,12 +138,12 @@ static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, uns
     unsigned char *p = zm+1, *k = NULL;
     unsigned int l,llen;
 
-    while(*p != ZIP_END) {
+    while(*p != ZIPMAP_END) {
         unsigned char free;
 
         /* Match or skip the key */
-        l = zipDecodeLength(p);
-        llen = zipEncodeLength(NULL,l);
+        l = zipmapDecodeLength(p);
+        llen = zipmapEncodeLength(NULL,l);
         if (k == NULL && l == klen && !memcmp(p+llen,key,l)) {
             /* Only return when the user doesn't care
              * for the total length of the zipmap. */
@@ -122,8 +155,8 @@ static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, uns
         }
         p += llen+l;
         /* Skip the value as well */
-        l = zipDecodeLength(p);
-        p += zipEncodeLength(NULL,l);
+        l = zipmapDecodeLength(p);
+        p += zipmapEncodeLength(NULL,l);
         free = p[0];
         p += l+1+free; /* +1 to skip the free byte */
     }
@@ -135,23 +168,24 @@ static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen)
     unsigned int l;
 
     l = klen+vlen+3;
-    if (klen >= ZIP_BIGLEN) l += 4;
-    if (vlen >= ZIP_BIGLEN) l += 4;
+    if (klen >= ZIPMAP_BIGLEN) l += 4;
+    if (vlen >= ZIPMAP_BIGLEN) l += 4;
     return l;
 }
 
 /* Return the total amount used by a key (encoded length + payload) */
 static unsigned int zipmapRawKeyLength(unsigned char *p) {
-    return zipRawEntryLength(p);
+    unsigned int l = zipmapDecodeLength(p);
+    return zipmapEncodeLength(NULL,l) + l;
 }
 
 /* Return the total amount used by a value
  * (encoded length + single byte free count + payload) */
 static unsigned int zipmapRawValueLength(unsigned char *p) {
-    unsigned int l = zipDecodeLength(p);
+    unsigned int l = zipmapDecodeLength(p);
     unsigned int used;
     
-    used = zipEncodeLength(NULL,l);
+    used = zipmapEncodeLength(NULL,l);
     used += p[used] + 1 + l;
     return used;
 }
@@ -164,6 +198,12 @@ static unsigned int zipmapRawEntryLength(unsigned char *p) {
     return l + zipmapRawValueLength(p+l);
 }
 
+static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
+    zm = zrealloc(zm, len);
+    zm[len-1] = ZIPMAP_END;
+    return zm;
+}
+
 /* Set key to value, creating the key if it does not already exist.
  * If 'update' is not NULL, *update is set to 1 if the key was
  * already preset, otherwise to 0. */
@@ -178,12 +218,12 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
     p = zipmapLookupRaw(zm,key,klen,&zmlen);
     if (p == NULL) {
         /* Key not found: enlarge */
-        zm = zipResize(zm, zmlen+reqlen);
+        zm = zipmapResize(zm, zmlen+reqlen);
         p = zm+zmlen-1;
         zmlen = zmlen+reqlen;
 
         /* Increase zipmap length (this is an insert) */
-        if (zm[0] < ZIP_BIGLEN) zm[0]++;
+        if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
     } else {
         /* Key found. Is there enough space for the new value? */
         /* Compute the total length: */
@@ -194,7 +234,7 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
              * it can be resized. Then, move the tail backwards so this
              * pair fits at the current position. */
             offset = p-zm;
-            zm = zipResize(zm, zmlen-freelen+reqlen);
+            zm = zipmapResize(zm, zmlen-freelen+reqlen);
             p = zm+offset;
 
             /* The +1 in the number of bytes to be moved is caused by the
@@ -216,7 +256,7 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
         offset = p-zm;
         memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
         zmlen -= empty;
-        zm = zipResize(zm, zmlen);
+        zm = zipmapResize(zm, zmlen);
         p = zm+offset;
         vempty = 0;
     } else {
@@ -225,11 +265,11 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
 
     /* Just write the key + value and we are done. */
     /* Key: */
-    p += zipEncodeLength(p,klen);
+    p += zipmapEncodeLength(p,klen);
     memcpy(p,key,klen);
     p += klen;
     /* Value: */
-    p += zipEncodeLength(p,vlen);
+    p += zipmapEncodeLength(p,vlen);
     *p++ = vempty;
     memcpy(p,val,vlen);
     return zm;
@@ -243,10 +283,10 @@ unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int kle
     if (p) {
         freelen = zipmapRawEntryLength(p);
         memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
-        zm = zipResize(zm, zmlen-freelen);
+        zm = zipmapResize(zm, zmlen-freelen);
 
         /* Decrease zipmap length */
-        if (zm[0] < ZIP_BIGLEN) zm[0]--;
+        if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;
 
         if (deleted) *deleted = 1;
     } else {
@@ -272,17 +312,17 @@ unsigned char *zipmapRewind(unsigned char *zm) {
  * }
  */
 unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
-    if (zm[0] == ZIP_END) return NULL;
+    if (zm[0] == ZIPMAP_END) return NULL;
     if (key) {
         *key = zm;
-        *klen = zipDecodeLength(zm);
-        *key += ZIP_LEN_BYTES(*klen);
+        *klen = zipmapDecodeLength(zm);
+        *key += ZIPMAP_LEN_BYTES(*klen);
     }
     zm += zipmapRawKeyLength(zm);
     if (value) {
         *value = zm+1;
-        *vlen = zipDecodeLength(zm);
-        *value += ZIP_LEN_BYTES(*vlen);
+        *vlen = zipmapDecodeLength(zm);
+        *value += ZIPMAP_LEN_BYTES(*vlen);
     }
     zm += zipmapRawValueLength(zm);
     return zm;
@@ -295,8 +335,8 @@ int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned
 
     if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
     p += zipmapRawKeyLength(p);
-    *vlen = zipDecodeLength(p);
-    *value = p + ZIP_LEN_BYTES(*vlen) + 1;
+    *vlen = zipmapDecodeLength(p);
+    *value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
     return 1;
 }
 
@@ -308,14 +348,14 @@ int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
 /* Return the number of entries inside a zipmap */
 unsigned int zipmapLen(unsigned char *zm) {
     unsigned int len = 0;
-    if (zm[0] < ZIP_BIGLEN) {
+    if (zm[0] < ZIPMAP_BIGLEN) {
         len = zm[0];
     } else {
         unsigned char *p = zipmapRewind(zm);
         while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
 
         /* Re-store length if small enough */
-        if (len < ZIP_BIGLEN) zm[0] = len;
+        if (len < ZIPMAP_BIGLEN) zm[0] = len;
     }
     return len;
 }
@@ -325,21 +365,21 @@ void zipmapRepr(unsigned char *p) {
 
     printf("{status %u}",*p++);
     while(1) {
-        if (p[0] == ZIP_END) {
+        if (p[0] == ZIPMAP_END) {
             printf("{end}");
             break;
         } else {
             unsigned char e;
 
-            l = zipDecodeLength(p);
+            l = zipmapDecodeLength(p);
             printf("{key %u}",l);
-            p += zipEncodeLength(NULL,l);
+            p += zipmapEncodeLength(NULL,l);
             fwrite(p,l,1,stdout);
             p += l;
 
-            l = zipDecodeLength(p);
+            l = zipmapDecodeLength(p);
             printf("{value %u}",l);
-            p += zipEncodeLength(NULL,l);
+            p += zipmapEncodeLength(NULL,l);
             e = *p++;
             fwrite(p,l,1,stdout);
             p += l+e;

From 37fff074a2f79a63c3fefbcf95fe72d83549b084 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 15:18:30 +0200
Subject: [PATCH 13/66] move code from zip.c to ziplist.c

---
 zip.c     | 158 -----------------------------------------------------
 ziplist.c | 159 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 157 insertions(+), 160 deletions(-)
 delete mode 100644 zip.c

diff --git a/zip.c b/zip.c
deleted file mode 100644
index e8f01edb..00000000
--- a/zip.c
+++ /dev/null
@@ -1,158 +0,0 @@
-#define ZIP_BIGLEN 254
-#define ZIP_END 255
-
-/* Entry encoding */
-#define ZIP_ENC_RAW     0
-#define ZIP_ENC_SHORT   1
-#define ZIP_ENC_INT     2
-#define ZIP_ENC_LLONG   3
-#define ZIP_ENCODING(p) ((p)[0] >> 6)
-
-/* Length encoding for raw entries */
-#define ZIP_LEN_INLINE  0
-#define ZIP_LEN_UINT16  1
-#define ZIP_LEN_UINT32  2
-
-static unsigned int zipEncodingSize(char encoding) {
-    if (encoding == ZIP_ENC_SHORT) {
-        return sizeof(short int);
-    } else if (encoding == ZIP_ENC_INT) {
-        return sizeof(int);
-    } else if (encoding == ZIP_ENC_LLONG) {
-        return sizeof(long long);
-    }
-    assert(NULL);
-}
-
-/* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
- * provided, it is set to the number of bytes required to encode the length. */
-static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) {
-    unsigned char encoding = ZIP_ENCODING(p), lenenc;
-    unsigned int len;
-
-    if (encoding == ZIP_ENC_RAW) {
-        lenenc = (p[0] >> 4) & 0x3;
-        if (lenenc == ZIP_LEN_INLINE) {
-            len = p[0] & 0xf;
-            if (lensize) *lensize = 1;
-        } else if (lenenc == ZIP_LEN_UINT16) {
-            len = p[1] | (p[2] << 8);
-            if (lensize) *lensize = 3;
-        } else {
-            len = p[1] | (p[2] << 8) | (p[3] << 16) | (p[4] << 24);
-            if (lensize) *lensize = 5;
-        }
-    } else {
-        len = zipEncodingSize(encoding);
-        if (lensize) *lensize = 1;
-    }
-    return len;
-}
-
-/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
- * the amount of bytes required to encode such a length. */
-static unsigned int zipEncodeLength(unsigned char *p, char encoding, unsigned int rawlen) {
-    unsigned char len = 1, lenenc, buf[5];
-    if (encoding == ZIP_ENC_RAW) {
-        if (rawlen <= 0xf) {
-            if (!p) return len;
-            lenenc = ZIP_LEN_INLINE;
-            buf[0] = rawlen;
-        } else if (rawlen <= 0xffff) {
-            len += 2;
-            if (!p) return len;
-            lenenc = ZIP_LEN_UINT16;
-            buf[1] = (rawlen     ) & 0xff;
-            buf[2] = (rawlen >> 8) & 0xff;
-        } else {
-            len += 4;
-            if (!p) return len;
-            lenenc = ZIP_LEN_UINT32;
-            buf[1] = (rawlen      ) & 0xff;
-            buf[2] = (rawlen >>  8) & 0xff;
-            buf[3] = (rawlen >> 16) & 0xff;
-            buf[4] = (rawlen >> 24) & 0xff;
-        }
-        buf[0] = (lenenc << 4) | (buf[0] & 0xf);
-    }
-    if (!p) return len;
-
-    /* Apparently we need to store the length in 'p' */
-    buf[0] = (encoding << 6) | (buf[0] & 0x3f);
-    memcpy(p,buf,len);
-    return len;
-}
-
-/* Check if string pointed to by 'entry' can be encoded as an integer.
- * Stores the integer value in 'v' and its encoding in 'encoding'.
- * Warning: this function requires a NULL-terminated string! */
-static int zipTryEncoding(unsigned char *entry, long long *v, char *encoding) {
-    long long value;
-    char *eptr;
-
-    if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
-        value = strtoll(entry,&eptr,10);
-        if (eptr[0] != '\0') return 0;
-        if (value >= SHRT_MIN && value <= SHRT_MAX) {
-            *encoding = ZIP_ENC_SHORT;
-        } else if (value >= INT_MIN && value <= INT_MAX) {
-            *encoding = ZIP_ENC_INT;
-        } else {
-            *encoding = ZIP_ENC_LLONG;
-        }
-        *v = value;
-        return 1;
-    }
-    return 0;
-}
-
-static void zipSaveInteger(unsigned char *p, long long value, char encoding) {
-    short int s;
-    int i;
-    long long l;
-    if (encoding == ZIP_ENC_SHORT) {
-        s = value;
-        memcpy(p,&s,sizeof(s));
-    } else if (encoding == ZIP_ENC_INT) {
-        i = value;
-        memcpy(p,&i,sizeof(i));
-    } else if (encoding == ZIP_ENC_LLONG) {
-        l = value;
-        memcpy(p,&l,sizeof(l));
-    } else {
-        assert(NULL);
-    }
-}
-
-static long long zipLoadInteger(unsigned char *p, char encoding) {
-    short int s;
-    int i;
-    long long l, ret;
-    if (encoding == ZIP_ENC_SHORT) {
-        memcpy(&s,p,sizeof(s));
-        ret = s;
-    } else if (encoding == ZIP_ENC_INT) {
-        memcpy(&i,p,sizeof(i));
-        ret = i;
-    } else if (encoding == ZIP_ENC_LLONG) {
-        memcpy(&l,p,sizeof(l));
-        ret = l;
-    } else {
-        assert(NULL);
-    }
-    return ret;
-}
-
-/* Return the total amount used by an entry (encoded length + payload). */
-static unsigned int zipRawEntryLength(unsigned char *p) {
-    unsigned int lensize, len;
-    len = zipDecodeLength(p, &lensize);
-    return lensize + len;
-}
-
-/* Resize the zip* structure. */
-static unsigned char *zipResize(unsigned char *z, unsigned int len) {
-    z = zrealloc(z,len);
-    z[len-1] = ZIP_END;
-    return z;
-}
diff --git a/ziplist.c b/ziplist.c
index a45ae339..c2e56bfb 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -22,14 +22,168 @@
 #include "zmalloc.h"
 #include "sds.h"
 #include "ziplist.h"
-#include "zip.c"
 
+#define ZIP_END 255
+
+/* Entry encoding */
+#define ZIP_ENC_RAW     0
+#define ZIP_ENC_SHORT   1
+#define ZIP_ENC_INT     2
+#define ZIP_ENC_LLONG   3
+#define ZIP_ENCODING(p) ((p)[0] >> 6)
+
+/* Length encoding for raw entries */
+#define ZIP_LEN_INLINE  0
+#define ZIP_LEN_UINT16  1
+#define ZIP_LEN_UINT32  2
+
+/* Utility macros */
 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
 #define ZIPLIST_LENGTH(zl) (*((zl)+sizeof(unsigned int)))
 #define ZIPLIST_HEADER_SIZE (sizeof(unsigned int)+1)
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
     if (ZIPLIST_LENGTH(zl) < (ZIP_END-1)) ZIPLIST_LENGTH(zl)+=incr; }
 
+/* Return bytes needed to store integer encoded by 'encoding' */
+static unsigned int zipEncodingSize(char encoding) {
+    if (encoding == ZIP_ENC_SHORT) {
+        return sizeof(short int);
+    } else if (encoding == ZIP_ENC_INT) {
+        return sizeof(int);
+    } else if (encoding == ZIP_ENC_LLONG) {
+        return sizeof(long long);
+    }
+    assert(NULL);
+}
+
+/* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
+ * provided, it is set to the number of bytes required to encode the length. */
+static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) {
+    unsigned char encoding = ZIP_ENCODING(p), lenenc;
+    unsigned int len;
+
+    if (encoding == ZIP_ENC_RAW) {
+        lenenc = (p[0] >> 4) & 0x3;
+        if (lenenc == ZIP_LEN_INLINE) {
+            len = p[0] & 0xf;
+            if (lensize) *lensize = 1;
+        } else if (lenenc == ZIP_LEN_UINT16) {
+            len = p[1] | (p[2] << 8);
+            if (lensize) *lensize = 3;
+        } else {
+            len = p[1] | (p[2] << 8) | (p[3] << 16) | (p[4] << 24);
+            if (lensize) *lensize = 5;
+        }
+    } else {
+        len = zipEncodingSize(encoding);
+        if (lensize) *lensize = 1;
+    }
+    return len;
+}
+
+/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
+ * the amount of bytes required to encode such a length. */
+static unsigned int zipEncodeLength(unsigned char *p, char encoding, unsigned int rawlen) {
+    unsigned char len = 1, lenenc, buf[5];
+    if (encoding == ZIP_ENC_RAW) {
+        if (rawlen <= 0xf) {
+            if (!p) return len;
+            lenenc = ZIP_LEN_INLINE;
+            buf[0] = rawlen;
+        } else if (rawlen <= 0xffff) {
+            len += 2;
+            if (!p) return len;
+            lenenc = ZIP_LEN_UINT16;
+            buf[1] = (rawlen     ) & 0xff;
+            buf[2] = (rawlen >> 8) & 0xff;
+        } else {
+            len += 4;
+            if (!p) return len;
+            lenenc = ZIP_LEN_UINT32;
+            buf[1] = (rawlen      ) & 0xff;
+            buf[2] = (rawlen >>  8) & 0xff;
+            buf[3] = (rawlen >> 16) & 0xff;
+            buf[4] = (rawlen >> 24) & 0xff;
+        }
+        buf[0] = (lenenc << 4) | (buf[0] & 0xf);
+    }
+    if (!p) return len;
+
+    /* Apparently we need to store the length in 'p' */
+    buf[0] = (encoding << 6) | (buf[0] & 0x3f);
+    memcpy(p,buf,len);
+    return len;
+}
+
+/* Check if string pointed to by 'entry' can be encoded as an integer.
+ * Stores the integer value in 'v' and its encoding in 'encoding'.
+ * Warning: this function requires a NULL-terminated string! */
+static int zipTryEncoding(unsigned char *entry, long long *v, char *encoding) {
+    long long value;
+    char *eptr;
+
+    if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
+        value = strtoll(entry,&eptr,10);
+        if (eptr[0] != '\0') return 0;
+        if (value >= SHRT_MIN && value <= SHRT_MAX) {
+            *encoding = ZIP_ENC_SHORT;
+        } else if (value >= INT_MIN && value <= INT_MAX) {
+            *encoding = ZIP_ENC_INT;
+        } else {
+            *encoding = ZIP_ENC_LLONG;
+        }
+        *v = value;
+        return 1;
+    }
+    return 0;
+}
+
+/* Store integer 'value' at 'p', encoded as 'encoding' */
+static void zipSaveInteger(unsigned char *p, long long value, char encoding) {
+    short int s;
+    int i;
+    long long l;
+    if (encoding == ZIP_ENC_SHORT) {
+        s = value;
+        memcpy(p,&s,sizeof(s));
+    } else if (encoding == ZIP_ENC_INT) {
+        i = value;
+        memcpy(p,&i,sizeof(i));
+    } else if (encoding == ZIP_ENC_LLONG) {
+        l = value;
+        memcpy(p,&l,sizeof(l));
+    } else {
+        assert(NULL);
+    }
+}
+
+/* Read integer encoded as 'encoding' from 'p' */
+static long long zipLoadInteger(unsigned char *p, char encoding) {
+    short int s;
+    int i;
+    long long l, ret;
+    if (encoding == ZIP_ENC_SHORT) {
+        memcpy(&s,p,sizeof(s));
+        ret = s;
+    } else if (encoding == ZIP_ENC_INT) {
+        memcpy(&i,p,sizeof(i));
+        ret = i;
+    } else if (encoding == ZIP_ENC_LLONG) {
+        memcpy(&l,p,sizeof(l));
+        ret = l;
+    } else {
+        assert(NULL);
+    }
+    return ret;
+}
+
+/* Return the total amount used by an entry (encoded length + payload). */
+static unsigned int zipRawEntryLength(unsigned char *p) {
+    unsigned int lensize, len;
+    len = zipDecodeLength(p, &lensize);
+    return lensize + len;
+}
+
 /* Create a new empty ziplist. */
 unsigned char *ziplistNew(void) {
     unsigned int bytes = ZIPLIST_HEADER_SIZE+1;
@@ -40,8 +194,9 @@ unsigned char *ziplistNew(void) {
     return zl;
 }
 
+/* Resize the ziplist. */
 static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
-    zl = zipResize(zl,len);
+    zl = zrealloc(zl,len);
     ZIPLIST_BYTES(zl) = len;
     zl[len-1] = ZIP_END;
     return zl;

From 75d8978e7a0acc5772aa5c6844a2d045d563a615 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 15:48:27 +0200
Subject: [PATCH 14/66] updated iteration code to work well with different
 encodings

---
 ziplist.c | 90 ++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 62 insertions(+), 28 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index c2e56bfb..fe2bfaf6 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -296,18 +296,31 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
     return p;
 }
 
-/* Store entry at current position in sds *value and return pointer
- * to the next entry. */
-unsigned char *ziplistNext(unsigned char *p, unsigned char **q, unsigned char **entry, unsigned int *elen) {
-    unsigned int lensize;
-    if (*p == ZIP_END) return NULL;
-    if (entry) {
-        *elen = zipDecodeLength(p,&lensize);
-        *entry = p+lensize;
+/* Return pointer to next entry in ziplist. */
+unsigned char *ziplistNext(unsigned char *p) {
+    return *p == ZIP_END ? p : p+zipRawEntryLength(p);
+}
+
+/* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
+ * on the encoding of the entry. 'e' is always set to NULL to be able
+ * to find out whether the string pointer or the integer value was set.
+ * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
+unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v) {
+    unsigned int len, lensize;
+    if (*p == ZIP_END) return 0;
+    if (e) *e = NULL;
+    len = zipDecodeLength(p,&lensize);
+    if (ZIP_ENCODING(p) == ZIP_ENC_RAW) {
+        if (e) {
+            *elen = len;
+            *e = p+lensize;
+        }
+    } else {
+        if (v) {
+            *v = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
+        }
     }
-    if (q != NULL) *q = p;
-    p += zipRawEntryLength(p);
-    return p;
+    return 1;
 }
 
 /* Delete a range of entries from the ziplist. */
@@ -382,6 +395,7 @@ unsigned char *createList() {
     zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
     zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
     zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
+    zl = ziplistPush(zl, (unsigned char*)"1024", 4, ZIPLIST_TAIL);
     return zl;
 }
 
@@ -407,6 +421,7 @@ unsigned char *createIntList() {
 int main(int argc, char **argv) {
     unsigned char *zl, *p, *q, *entry;
     unsigned int elen;
+    long long value;
     sds s;
 
     zl = createIntList();
@@ -427,10 +442,15 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 0);
-        while ((p = ziplistNext(p, NULL, &entry, &elen)) != NULL) {
+        while (ziplistGet(p, &entry, &elen, &value)) {
             printf("Entry: ");
-            fwrite(entry,elen,1,stdout);
-            printf(" (length %d)\n", elen);
+            if (entry) {
+                fwrite(entry,elen,1,stdout);
+            } else {
+                printf("%lld", value);
+            }
+            p = ziplistNext(p);
+            printf("\n");
         }
         printf("\n");
     }
@@ -439,10 +459,15 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 1);
-        while ((p = ziplistNext(p, NULL, &entry, &elen)) != NULL) {
+        while (ziplistGet(p, &entry, &elen, &value)) {
             printf("Entry: ");
-            fwrite(entry,elen,1,stdout);
-            printf(" (length %d)\n", elen);
+            if (entry) {
+                fwrite(entry,elen,1,stdout);
+            } else {
+                printf("%lld", value);
+            }
+            p = ziplistNext(p);
+            printf("\n");
         }
         printf("\n");
     }
@@ -451,10 +476,15 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 2);
-        while ((p = ziplistNext(p, NULL, &entry, &elen)) != NULL) {
+        while (ziplistGet(p, &entry, &elen, &value)) {
             printf("Entry: ");
-            fwrite(entry,elen,1,stdout);
-            printf(" (length %d)\n", elen);
+            if (entry) {
+                fwrite(entry,elen,1,stdout);
+            } else {
+                printf("%lld", value);
+            }
+            p = ziplistNext(p);
+            printf("\n");
         }
         printf("\n");
     }
@@ -462,8 +492,8 @@ int main(int argc, char **argv) {
     printf("Iterate starting out of range:\n");
     {
         zl = createList();
-        p = ziplistIndex(zl, 3);
-        if (ziplistNext(p, &entry, NULL, &elen) == NULL) {
+        p = ziplistIndex(zl, 4);
+        if (!ziplistGet(p, &entry, &elen, &value)) {
             printf("No entry\n");
         } else {
             printf("ERROR\n");
@@ -510,15 +540,19 @@ int main(int argc, char **argv) {
     {
         zl = createList();
         p = ziplistIndex(zl, 0);
-        while ((p = ziplistNext(p, &q, &entry, &elen)) != NULL) {
-            if (strncmp("foo", entry, elen) == 0) {
+        while (ziplistGet(p, &entry, &elen, &value)) {
+            if (entry && strncmp("foo", entry, elen) == 0) {
                 printf("Delete foo\n");
-                zl = ziplistDelete(zl, &q);
-                p = q;
+                zl = ziplistDelete(zl, &p);
             } else {
                 printf("Entry: ");
-                fwrite(entry,elen,1,stdout);
-                printf(" (length %d)\n", elen);
+                if (entry) {
+                    fwrite(entry,elen,1,stdout);
+                } else {
+                    printf("%lld", value);
+                }
+                p = ziplistNext(p);
+                printf("\n");
             }
         }
         printf("\n");

From c09c2c3b04f0df0e37c24ee10d29bdf49ae070ff Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 16:25:06 +0200
Subject: [PATCH 15/66] code to compare strings with entries in ziplist,
 regardless of their encoding

---
 ziplist.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index fe2bfaf6..55fcc80c 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -365,6 +365,28 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
     return zl;
 }
 
+/* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
+unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen) {
+    unsigned int zlen, lensize;
+    char encoding;
+    long long zval, eval;
+    if (*p == ZIP_END) return 0;
+
+    zlen = zipDecodeLength(p,&lensize);
+    if (zipTryEncoding(entry,&eval,&encoding)) {
+        /* Do integer compare */
+        zval = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
+        return zval == eval;
+    } else {
+        /* Raw compare */
+        if (zlen == elen) {
+            return memcmp(p+lensize,entry,elen) == 0;
+        } else {
+            return 0;
+        }
+    }
+}
+
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p, encoding;
     unsigned int l, lsize;
@@ -557,7 +579,31 @@ int main(int argc, char **argv) {
         }
         printf("\n");
         ziplistRepr(zl);
-        printf("\n");
+    }
+
+    printf("Compare strings with ziplist entries:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 0);
+        if (!ziplistCompare(p,"hello",5)) {
+            printf("ERROR\n");
+            return;
+        }
+        if (ziplistCompare(p,"hella",5)) {
+            printf("ERROR\n");
+            return;
+        }
+
+        p = ziplistIndex(zl, 3);
+        if (!ziplistCompare(p,"1024",4)) {
+            printf("ERROR\n");
+            return;
+        }
+        if (ziplistCompare(p,"1025",4)) {
+            printf("ERROR\n");
+            return;
+        }
+        printf("SUCCESS\n");
     }
 
     return 0;

From c7019f7019a7ded425e700f391263227e259bbcb Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 16:25:18 +0200
Subject: [PATCH 16/66] added header ziplist.h

---
 ziplist.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ziplist.h b/ziplist.h
index bfc1854a..4b2d1c93 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -1,2 +1,12 @@
 #define ZIPLIST_HEAD 0
 #define ZIPLIST_TAIL 1
+
+unsigned char *ziplistNew(void);
+unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where);
+unsigned char *ziplistPop(unsigned char *zl, sds *target, int where);
+unsigned char *ziplistIndex(unsigned char *zl, unsigned int index);
+unsigned char *ziplistNext(unsigned char *p);
+unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v);
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
+unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
+unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen);
\ No newline at end of file

From aa549962a4227f8732e3c9448ba8910160ec3ec4 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 19:35:26 +0200
Subject: [PATCH 17/66] re-introduce ZIP_BIGLEN for clarity

---
 ziplist.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index 55fcc80c..4907d07b 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -24,6 +24,7 @@
 #include "ziplist.h"
 
 #define ZIP_END 255
+#define ZIP_BIGLEN 254
 
 /* Entry encoding */
 #define ZIP_ENC_RAW     0
@@ -42,7 +43,7 @@
 #define ZIPLIST_LENGTH(zl) (*((zl)+sizeof(unsigned int)))
 #define ZIPLIST_HEADER_SIZE (sizeof(unsigned int)+1)
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
-    if (ZIPLIST_LENGTH(zl) < (ZIP_END-1)) ZIPLIST_LENGTH(zl)+=incr; }
+    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
 
 /* Return bytes needed to store integer encoded by 'encoding' */
 static unsigned int zipEncodingSize(char encoding) {

From 6205b46387ef47e0fcb9bd6c9f371c9eb5cdd7e5 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 19:35:40 +0200
Subject: [PATCH 18/66] add function to retrieve length of ziplist

---
 ziplist.c | 18 ++++++++++++++++++
 ziplist.h |  3 ++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index 4907d07b..3dd3c1e8 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -388,6 +388,24 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int
     }
 }
 
+/* Return length of ziplist. */
+unsigned int ziplistLen(unsigned char *zl) {
+    unsigned int len = 0;
+    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) {
+        len = ZIPLIST_LENGTH(zl);
+    } else {
+        unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
+        while (*p != ZIP_END) {
+            p += zipRawEntryLength(p);
+            len++;
+        }
+
+        /* Re-store length if small enough */
+        if (len < ZIP_BIGLEN) ZIPLIST_LENGTH(zl) = len;
+    }
+    return len;
+}
+
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p, encoding;
     unsigned int l, lsize;
diff --git a/ziplist.h b/ziplist.h
index 4b2d1c93..cea7f5de 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -9,4 +9,5 @@ unsigned char *ziplistNext(unsigned char *p);
 unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
-unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen);
\ No newline at end of file
+unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen);
+unsigned int ziplistLen(unsigned char *zl);
\ No newline at end of file

From c4aace9003e29fc9becce292683dd09bdce7785c Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 22 May 2010 21:49:36 +0200
Subject: [PATCH 19/66] fix compare function of ziplist to only load integer
 from ziplist when it is encoded as integer

---
 ziplist.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 3dd3c1e8..3dc534d3 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -374,18 +374,23 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int
     if (*p == ZIP_END) return 0;
 
     zlen = zipDecodeLength(p,&lensize);
-    if (zipTryEncoding(entry,&eval,&encoding)) {
-        /* Do integer compare */
-        zval = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
-        return zval == eval;
-    } else {
+    if (ZIP_ENCODING(p) == ZIP_ENC_RAW) {
         /* Raw compare */
         if (zlen == elen) {
             return memcmp(p+lensize,entry,elen) == 0;
         } else {
             return 0;
         }
+    } else {
+        if (zipTryEncoding(entry,&eval,&encoding)) {
+            /* Do integer compare */
+            zval = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
+            return zval == eval;
+        } else {
+            /* Ziplist entry is integer encoded, but given entry is not. */
+        }
     }
+    return 0;
 }
 
 /* Return length of ziplist. */

From 4812cf287347b035c113f8afdc964a007e7744a4 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 11:33:38 +0200
Subject: [PATCH 20/66] add function to retrieve ziplist size in bytes

---
 ziplist.c | 5 +++++
 ziplist.h | 3 ++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index 3dc534d3..4b4c568d 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -411,6 +411,11 @@ unsigned int ziplistLen(unsigned char *zl) {
     return len;
 }
 
+/* Return size in bytes of ziplist. */
+unsigned int ziplistSize(unsigned char *zl) {
+    return ZIPLIST_BYTES(zl);
+}
+
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p, encoding;
     unsigned int l, lsize;
diff --git a/ziplist.h b/ziplist.h
index cea7f5de..fecfe9b6 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -10,4 +10,5 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen,
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
 unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen);
-unsigned int ziplistLen(unsigned char *zl);
\ No newline at end of file
+unsigned int ziplistLen(unsigned char *zl);
+unsigned int ziplistSize(unsigned char *zl);

From fc2c0f7a6c9d515bff525c89b81df428277a17a0 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 11:35:48 +0200
Subject: [PATCH 21/66] fix some warnings

---
 ziplist.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 4b4c568d..0d739d3d 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -124,7 +124,7 @@ static int zipTryEncoding(unsigned char *entry, long long *v, char *encoding) {
     char *eptr;
 
     if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
-        value = strtoll(entry,&eptr,10);
+        value = strtoll((char*)entry,&eptr,10);
         if (eptr[0] != '\0') return 0;
         if (value >= SHRT_MIN && value <= SHRT_MAX) {
             *encoding = ZIP_ENC_SHORT;
@@ -327,7 +327,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen,
 /* Delete a range of entries from the ziplist. */
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
     unsigned char *p, *first = ziplistIndex(zl, index);
-    unsigned int i, deleted = 0, totlen, newlen;
+    unsigned int i, totlen, deleted = 0;
     for (p = first, i = 0; *p != ZIP_END && i < num; i++) {
         p += zipRawEntryLength(p);
         deleted++;
@@ -419,7 +419,6 @@ unsigned int ziplistSize(unsigned char *zl) {
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p, encoding;
     unsigned int l, lsize;
-    long long value;
 
     printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
     p = ziplistHead(zl);

From dcb9cf4e8213744e431f38722d04b841f78b316c Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 15:40:40 +0200
Subject: [PATCH 22/66] initial implementation for making the ziplist doubly
 linked

---
 ziplist.c | 159 ++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 119 insertions(+), 40 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 0d739d3d..adb7d054 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -23,6 +23,11 @@
 #include "sds.h"
 #include "ziplist.h"
 
+/* Important note: the ZIP_END value is used to depict the end of the
+ * ziplist structure. When a pointer contains an entry, the first couple
+ * of bytes contain the encoded length of the previous entry. This length
+ * is encoded as ZIP_ENC_RAW length, so the first two bits will contain 00
+ * and the byte will therefore never have a value of 255. */
 #define ZIP_END 255
 #define ZIP_BIGLEN 254
 
@@ -40,8 +45,9 @@
 
 /* Utility macros */
 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
-#define ZIPLIST_LENGTH(zl) (*((zl)+sizeof(unsigned int)))
-#define ZIPLIST_HEADER_SIZE (sizeof(unsigned int)+1)
+#define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
+#define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
+#define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
     if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
 
@@ -116,6 +122,14 @@ static unsigned int zipEncodeLength(unsigned char *p, char encoding, unsigned in
     return len;
 }
 
+/* Return the difference in number of bytes needed to store the new length
+ * "len" on the entry pointed to by "p". */
+static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
+    unsigned int prevlensize;
+    zipDecodeLength(p,&prevlensize);
+    return zipEncodeLength(NULL,ZIP_ENC_RAW,len)-prevlensize;
+}
+
 /* Check if string pointed to by 'entry' can be encoded as an integer.
  * Stores the integer value in 'v' and its encoding in 'encoding'.
  * Warning: this function requires a NULL-terminated string! */
@@ -180,9 +194,12 @@ static long long zipLoadInteger(unsigned char *p, char encoding) {
 
 /* Return the total amount used by an entry (encoded length + payload). */
 static unsigned int zipRawEntryLength(unsigned char *p) {
-    unsigned int lensize, len;
-    len = zipDecodeLength(p, &lensize);
-    return lensize + len;
+    unsigned int prevlensize, lensize, len;
+    /* Byte-size of encoded length of previous entry */
+    zipDecodeLength(p,&prevlensize);
+    /* Encoded length of this entry's payload */
+    len = zipDecodeLength(p+prevlensize, &lensize);
+    return prevlensize+lensize+len;
 }
 
 /* Create a new empty ziplist. */
@@ -190,6 +207,7 @@ unsigned char *ziplistNew(void) {
     unsigned int bytes = ZIPLIST_HEADER_SIZE+1;
     unsigned char *zl = zmalloc(bytes);
     ZIPLIST_BYTES(zl) = bytes;
+    ZIPLIST_TAIL_OFFSET(zl) = ZIPLIST_HEADER_SIZE;
     ZIPLIST_LENGTH(zl) = 0;
     zl[bytes-1] = ZIP_END;
     return zl;
@@ -218,17 +236,30 @@ static unsigned char *ziplistTail(unsigned char *zl) {
 }
 
 unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where) {
-    unsigned int curlen = ZIPLIST_BYTES(zl), reqlen;
-    unsigned char *p;
+    unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen;
+    unsigned char *p, *curtail;
     char encoding = ZIP_ENC_RAW;
     long long value;
 
+    /* We need to store the length of the current tail when the list
+     * is non-empty and we push at the tail. */
+    curtail = zl+ZIPLIST_TAIL_OFFSET(zl);
+    if (where == ZIPLIST_TAIL && curtail[0] != ZIP_END) {
+        prevlen = zipRawEntryLength(curtail);
+    } else {
+        prevlen = 0;
+    }
+
     /* See if the entry can be encoded */
     if (zipTryEncoding(entry,&value,&encoding)) {
         reqlen = zipEncodingSize(encoding);
     } else {
         reqlen = elen;
     }
+
+    /* We need space for both the length of the previous entry and
+     * the length of the payload. */
+    reqlen += zipEncodeLength(NULL,ZIP_ENC_RAW,prevlen);
     reqlen += zipEncodeLength(NULL,encoding,elen);
 
     /* Resize the ziplist and move if needed */
@@ -243,7 +274,17 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
         p = zl+curlen-1;
     }
 
+    /* Update tail offset if this is not the first element */
+    if (curtail[0] != ZIP_END) {
+        if (where == ZIPLIST_HEAD) {
+            ZIPLIST_TAIL_OFFSET(zl) += reqlen;
+        } else {
+            ZIPLIST_TAIL_OFFSET(zl) += prevlen;
+        }
+    }
+
     /* Write the entry */
+    p += zipEncodeLength(p,ZIP_ENC_RAW,prevlen);
     p += zipEncodeLength(p,encoding,elen);
     if (encoding != ZIP_ENC_RAW) {
         zipSaveInteger(p,value,encoding);
@@ -256,7 +297,9 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
 
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     unsigned int curlen = ZIPLIST_BYTES(zl), rawlen;
-    unsigned int len, lensize;
+    unsigned int prevrawlensize, prevrawlen, lensize, len, headerlen;
+    int nextdiff = 0;
+    unsigned char encoding;
     unsigned char *p;
     long long value;
     if (target) *target = NULL;
@@ -264,24 +307,46 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     /* Get pointer to element to remove */
     p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : ziplistTail(zl);
     if (*p == ZIP_END) return zl;
-    len = zipDecodeLength(p,&lensize);
+
+    prevrawlen = zipDecodeLength(p,&prevrawlensize);
+    len = zipDecodeLength(p+prevrawlensize,&lensize);
+    headerlen = prevrawlensize+lensize;
+    rawlen = headerlen+len;
     if (target) {
-        if (ZIP_ENCODING(p) == ZIP_ENC_RAW) {
-            *target = sdsnewlen(p+lensize,len);
+        encoding = ZIP_ENCODING(p+prevrawlensize);
+        if (encoding == ZIP_ENC_RAW) {
+            *target = sdsnewlen(p+headerlen,len);
         } else {
-            value = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
+            value = zipLoadInteger(p+headerlen,encoding);
             *target = sdscatprintf(sdsempty(), "%lld", value);
         }
     }
 
-    /* Move list to front when popping from the head */
-    rawlen = lensize+len;
     if (where == ZIPLIST_HEAD) {
-        memmove(p,p+rawlen,curlen-ZIPLIST_HEADER_SIZE-len);
+        /* The next entry will now be the head of the list */
+        if (p[rawlen] != ZIP_END) {
+            /* Tricky: storing the length of the previous entry in the next
+             * entry (this previous length is always 0 when popping from the
+             * head), might reduce the number of bytes needed.
+             *
+             * In this special case (new length is 0), we know that the
+             * byte difference to store is always <= 0, which means that
+             * we always have space to store it. */
+            nextdiff = zipPrevLenByteDiff(p+rawlen,0);
+            zipEncodeLength(p+rawlen-nextdiff,ZIP_ENC_RAW,0);
+        }
+        /* Move list to the front */
+        memmove(p,p+rawlen-nextdiff,curlen-ZIPLIST_HEADER_SIZE-rawlen+nextdiff);
+
+        /* Subtract the gained space from the tail offset */
+        ZIPLIST_TAIL_OFFSET(zl) -= rawlen+nextdiff;
+    } else {
+        /* Subtract the length of the previous element from the tail offset. */
+        ZIPLIST_TAIL_OFFSET(zl) -= prevrawlen;
     }
 
     /* Resize and update length */
-    zl = ziplistResize(zl,curlen-rawlen);
+    zl = ziplistResize(zl,curlen-rawlen+nextdiff);
     ZIPLIST_INCR_LENGTH(zl,-1);
     return zl;
 }
@@ -307,18 +372,21 @@ unsigned char *ziplistNext(unsigned char *p) {
  * to find out whether the string pointer or the integer value was set.
  * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
 unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v) {
-    unsigned int len, lensize;
+    unsigned int prevrawlensize, lensize, len, headerlen;
     if (*p == ZIP_END) return 0;
     if (e) *e = NULL;
-    len = zipDecodeLength(p,&lensize);
-    if (ZIP_ENCODING(p) == ZIP_ENC_RAW) {
+
+    zipDecodeLength(p,&prevrawlensize);
+    len = zipDecodeLength(p+prevrawlensize,&lensize);
+    headerlen = prevrawlensize+lensize;
+    if (ZIP_ENCODING(p+prevrawlensize) == ZIP_ENC_RAW) {
         if (e) {
             *elen = len;
-            *e = p+lensize;
+            *e = p+headerlen;
         }
     } else {
         if (v) {
-            *v = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
+            *v = zipLoadInteger(p+headerlen,ZIP_ENCODING(p+prevrawlensize));
         }
     }
     return 1;
@@ -368,24 +436,26 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
 
 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
 unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen) {
-    unsigned int zlen, lensize;
+    unsigned int prevrawlensize, lensize, len, headerlen;
     char encoding;
-    long long zval, eval;
+    long long val, eval;
     if (*p == ZIP_END) return 0;
 
-    zlen = zipDecodeLength(p,&lensize);
-    if (ZIP_ENCODING(p) == ZIP_ENC_RAW) {
+    zipDecodeLength(p,&prevrawlensize);
+    len = zipDecodeLength(p+prevrawlensize,&lensize);
+    headerlen = prevrawlensize+lensize;
+    if (ZIP_ENCODING(p+prevrawlensize) == ZIP_ENC_RAW) {
         /* Raw compare */
-        if (zlen == elen) {
-            return memcmp(p+lensize,entry,elen) == 0;
+        if (len == elen) {
+            return memcmp(p+headerlen,entry,elen) == 0;
         } else {
             return 0;
         }
     } else {
         if (zipTryEncoding(entry,&eval,&encoding)) {
             /* Do integer compare */
-            zval = zipLoadInteger(p+lensize,ZIP_ENCODING(p));
-            return zval == eval;
+            val = zipLoadInteger(p+headerlen,ZIP_ENCODING(p+prevrawlensize));
+            return val == eval;
         } else {
             /* Ziplist entry is integer encoded, but given entry is not. */
         }
@@ -418,22 +488,23 @@ unsigned int ziplistSize(unsigned char *zl) {
 
 void ziplistRepr(unsigned char *zl) {
     unsigned char *p, encoding;
-    unsigned int l, lsize;
+    unsigned int prevrawlensize, prevrawlen, lensize, len;
 
     printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
     p = ziplistHead(zl);
     while(*p != ZIP_END) {
-        l = zipDecodeLength(p,&lsize);
-        printf("{header %u, payload %u} ",lsize,l);
-        encoding = ZIP_ENCODING(p);
-        p += lsize;
+        prevrawlen = zipDecodeLength(p,&prevrawlensize);
+        len = zipDecodeLength(p+prevrawlensize,&lensize);
+        printf("{offset %ld, header %u, payload %u} ",p-zl,prevrawlensize+lensize,len);
+        encoding = ZIP_ENCODING(p+prevrawlensize);
+        p += prevrawlensize+lensize;
         if (encoding == ZIP_ENC_RAW) {
-            fwrite(p,l,1,stdout);
+            fwrite(p,len,1,stdout);
         } else {
             printf("%lld", zipLoadInteger(p,encoding));
         }
         printf("\n");
-        p += l;
+        p += len;
     }
     printf("{end}\n\n");
 }
@@ -488,6 +559,14 @@ int main(int argc, char **argv) {
     printf("Pop head: %s (length %ld)\n", s, sdslen(s));
     ziplistRepr(zl);
 
+    zl = ziplistPop(zl, &s, ZIPLIST_TAIL);
+    printf("Pop tail: %s (length %ld)\n", s, sdslen(s));
+    ziplistRepr(zl);
+
+    zl = ziplistPop(zl, &s, ZIPLIST_TAIL);
+    printf("Pop tail: %s (length %ld)\n", s, sdslen(s));
+    ziplistRepr(zl);
+
     printf("Iterate list from 0 to end:\n");
     {
         zl = createList();
@@ -614,21 +693,21 @@ int main(int argc, char **argv) {
         zl = createList();
         p = ziplistIndex(zl, 0);
         if (!ziplistCompare(p,"hello",5)) {
-            printf("ERROR\n");
+            printf("ERROR: not \"hello\"\n");
             return;
         }
         if (ziplistCompare(p,"hella",5)) {
-            printf("ERROR\n");
+            printf("ERROR: \"hella\"\n");
             return;
         }
 
         p = ziplistIndex(zl, 3);
         if (!ziplistCompare(p,"1024",4)) {
-            printf("ERROR\n");
+            printf("ERROR: not \"1024\"\n");
             return;
         }
         if (ziplistCompare(p,"1025",4)) {
-            printf("ERROR\n");
+            printf("ERROR: \"1025\"\n");
             return;
         }
         printf("SUCCESS\n");

From a5456b2cb69c5bde1ea5403c6459885b074a8575 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 16:20:01 +0200
Subject: [PATCH 23/66] use a struct to retrieve all details for an entry

---
 ziplist.c | 73 +++++++++++++++++++++++++++++++------------------------
 1 file changed, 41 insertions(+), 32 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index adb7d054..50b92514 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -51,6 +51,13 @@
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
     if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
 
+typedef struct zlentry {
+    unsigned int prevrawlensize, prevrawlen;
+    unsigned int lensize, len;
+    unsigned int headersize;
+    unsigned char encoding;
+} zlentry;
+
 /* Return bytes needed to store integer encoded by 'encoding' */
 static unsigned int zipEncodingSize(char encoding) {
     if (encoding == ZIP_ENC_SHORT) {
@@ -192,6 +199,16 @@ static long long zipLoadInteger(unsigned char *p, char encoding) {
     return ret;
 }
 
+/* Return a struct with all information about an entry. */
+static zlentry zipEntry(unsigned char *p) {
+    zlentry e;
+    e.prevrawlen = zipDecodeLength(p,&e.prevrawlensize);
+    e.len = zipDecodeLength(p+e.prevrawlensize,&e.lensize);
+    e.headersize = e.prevrawlensize+e.lensize;
+    e.encoding = ZIP_ENCODING(p+e.prevrawlensize);
+    return e;
+}
+
 /* Return the total amount used by an entry (encoded length + payload). */
 static unsigned int zipRawEntryLength(unsigned char *p) {
     unsigned int prevlensize, lensize, len;
@@ -297,9 +314,8 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
 
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     unsigned int curlen = ZIPLIST_BYTES(zl), rawlen;
-    unsigned int prevrawlensize, prevrawlen, lensize, len, headerlen;
+    zlentry entry;
     int nextdiff = 0;
-    unsigned char encoding;
     unsigned char *p;
     long long value;
     if (target) *target = NULL;
@@ -308,16 +324,13 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : ziplistTail(zl);
     if (*p == ZIP_END) return zl;
 
-    prevrawlen = zipDecodeLength(p,&prevrawlensize);
-    len = zipDecodeLength(p+prevrawlensize,&lensize);
-    headerlen = prevrawlensize+lensize;
-    rawlen = headerlen+len;
+    entry = zipEntry(p);
+    rawlen = entry.headersize+entry.len;
     if (target) {
-        encoding = ZIP_ENCODING(p+prevrawlensize);
-        if (encoding == ZIP_ENC_RAW) {
-            *target = sdsnewlen(p+headerlen,len);
+        if (entry.encoding == ZIP_ENC_RAW) {
+            *target = sdsnewlen(p+entry.headersize,entry.len);
         } else {
-            value = zipLoadInteger(p+headerlen,encoding);
+            value = zipLoadInteger(p+entry.headersize,entry.encoding);
             *target = sdscatprintf(sdsempty(), "%lld", value);
         }
     }
@@ -342,7 +355,7 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
         ZIPLIST_TAIL_OFFSET(zl) -= rawlen+nextdiff;
     } else {
         /* Subtract the length of the previous element from the tail offset. */
-        ZIPLIST_TAIL_OFFSET(zl) -= prevrawlen;
+        ZIPLIST_TAIL_OFFSET(zl) -= entry.prevrawlen;
     }
 
     /* Resize and update length */
@@ -372,21 +385,19 @@ unsigned char *ziplistNext(unsigned char *p) {
  * to find out whether the string pointer or the integer value was set.
  * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
 unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v) {
-    unsigned int prevrawlensize, lensize, len, headerlen;
+    zlentry entry;
     if (*p == ZIP_END) return 0;
     if (e) *e = NULL;
 
-    zipDecodeLength(p,&prevrawlensize);
-    len = zipDecodeLength(p+prevrawlensize,&lensize);
-    headerlen = prevrawlensize+lensize;
-    if (ZIP_ENCODING(p+prevrawlensize) == ZIP_ENC_RAW) {
+    entry = zipEntry(p);
+    if (entry.encoding == ZIP_ENC_RAW) {
         if (e) {
-            *elen = len;
-            *e = p+headerlen;
+            *elen = entry.len;
+            *e = p+entry.headersize;
         }
     } else {
         if (v) {
-            *v = zipLoadInteger(p+headerlen,ZIP_ENCODING(p+prevrawlensize));
+            *v = zipLoadInteger(p+entry.headersize,entry.encoding);
         }
     }
     return 1;
@@ -435,27 +446,25 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
 }
 
 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
-unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen) {
-    unsigned int prevrawlensize, lensize, len, headerlen;
-    char encoding;
-    long long val, eval;
+unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen) {
+    zlentry entry;
+    unsigned char sencoding;
+    long long val, sval;
     if (*p == ZIP_END) return 0;
 
-    zipDecodeLength(p,&prevrawlensize);
-    len = zipDecodeLength(p+prevrawlensize,&lensize);
-    headerlen = prevrawlensize+lensize;
-    if (ZIP_ENCODING(p+prevrawlensize) == ZIP_ENC_RAW) {
+    entry = zipEntry(p);
+    if (entry.encoding == ZIP_ENC_RAW) {
         /* Raw compare */
-        if (len == elen) {
-            return memcmp(p+headerlen,entry,elen) == 0;
+        if (entry.len == slen) {
+            return memcmp(p+entry.headersize,s,slen) == 0;
         } else {
             return 0;
         }
     } else {
-        if (zipTryEncoding(entry,&eval,&encoding)) {
+        if (zipTryEncoding(s,&sval,&sencoding)) {
             /* Do integer compare */
-            val = zipLoadInteger(p+headerlen,ZIP_ENCODING(p+prevrawlensize));
-            return val == eval;
+            val = zipLoadInteger(p+entry.headersize,entry.encoding);
+            return val == sval;
         } else {
             /* Ziplist entry is integer encoded, but given entry is not. */
         }

From d593c48869e1c815fd4b270a1820e71165dafba5 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 16:22:24 +0200
Subject: [PATCH 24/66] modify compare function to check if the encoding is
 equal before comparing

---
 ziplist.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 50b92514..08253f37 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -461,12 +461,12 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int sle
             return 0;
         }
     } else {
+        /* Try to compare encoded values */
         if (zipTryEncoding(s,&sval,&sencoding)) {
-            /* Do integer compare */
-            val = zipLoadInteger(p+entry.headersize,entry.encoding);
-            return val == sval;
-        } else {
-            /* Ziplist entry is integer encoded, but given entry is not. */
+            if (entry.encoding == sencoding) {
+                val = zipLoadInteger(p+entry.headersize,entry.encoding);
+                return val == sval;
+            }
         }
     }
     return 0;

From c8d9e7f4c0c237ca75e0e53c0c643ef0f09f81bb Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 16:25:05 +0200
Subject: [PATCH 25/66] change ziplistRepr to use the entry struct

---
 ziplist.c | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 08253f37..a0412421 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -496,24 +496,22 @@ unsigned int ziplistSize(unsigned char *zl) {
 }
 
 void ziplistRepr(unsigned char *zl) {
-    unsigned char *p, encoding;
-    unsigned int prevrawlensize, prevrawlen, lensize, len;
+    unsigned char *p;
+    zlentry entry;
 
     printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
     p = ziplistHead(zl);
     while(*p != ZIP_END) {
-        prevrawlen = zipDecodeLength(p,&prevrawlensize);
-        len = zipDecodeLength(p+prevrawlensize,&lensize);
-        printf("{offset %ld, header %u, payload %u} ",p-zl,prevrawlensize+lensize,len);
-        encoding = ZIP_ENCODING(p+prevrawlensize);
-        p += prevrawlensize+lensize;
-        if (encoding == ZIP_ENC_RAW) {
-            fwrite(p,len,1,stdout);
+        entry = zipEntry(p);
+        printf("{offset %ld, header %u, payload %u} ",p-zl,entry.headersize,entry.len);
+        p += entry.headersize;
+        if (entry.encoding == ZIP_ENC_RAW) {
+            fwrite(p,entry.len,1,stdout);
         } else {
-            printf("%lld", zipLoadInteger(p,encoding));
+            printf("%lld", zipLoadInteger(p,entry.encoding));
         }
         printf("\n");
-        p += len;
+        p += entry.len;
     }
     printf("{end}\n\n");
 }

From 03e52931dd4e7d3c9a87eaa0195fdb8b08ff74b2 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 16:26:55 +0200
Subject: [PATCH 26/66] rename argument names to s* to disambiguate from e*

---
 ziplist.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index a0412421..d3e3cbaa 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -384,20 +384,20 @@ unsigned char *ziplistNext(unsigned char *p) {
  * on the encoding of the entry. 'e' is always set to NULL to be able
  * to find out whether the string pointer or the integer value was set.
  * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
-unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v) {
+unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) {
     zlentry entry;
     if (*p == ZIP_END) return 0;
-    if (e) *e = NULL;
+    if (sstr) *sstr = NULL;
 
     entry = zipEntry(p);
     if (entry.encoding == ZIP_ENC_RAW) {
-        if (e) {
-            *elen = entry.len;
-            *e = p+entry.headersize;
+        if (sstr) {
+            *slen = entry.len;
+            *sstr = p+entry.headersize;
         }
     } else {
-        if (v) {
-            *v = zipLoadInteger(p+entry.headersize,entry.encoding);
+        if (sval) {
+            *sval = zipLoadInteger(p+entry.headersize,entry.encoding);
         }
     }
     return 1;
@@ -446,7 +446,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
 }
 
 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
-unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen) {
+unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
     zlentry entry;
     unsigned char sencoding;
     long long val, sval;
@@ -456,13 +456,13 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int sle
     if (entry.encoding == ZIP_ENC_RAW) {
         /* Raw compare */
         if (entry.len == slen) {
-            return memcmp(p+entry.headersize,s,slen) == 0;
+            return memcmp(p+entry.headersize,sstr,slen) == 0;
         } else {
             return 0;
         }
     } else {
         /* Try to compare encoded values */
-        if (zipTryEncoding(s,&sval,&sencoding)) {
+        if (zipTryEncoding(sstr,&sval,&sencoding)) {
             if (entry.encoding == sencoding) {
                 val = zipLoadInteger(p+entry.headersize,entry.encoding);
                 return val == sval;

From bb57b965c4252a48472d99a1b2e7a30d66d65796 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 16:37:47 +0200
Subject: [PATCH 27/66] use the entry struct in zipRawEntryLength

---
 ziplist.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index d3e3cbaa..16e9dbc1 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -209,14 +209,10 @@ static zlentry zipEntry(unsigned char *p) {
     return e;
 }
 
-/* Return the total amount used by an entry (encoded length + payload). */
+/* Return the total number of bytes used by the entry at "p". */
 static unsigned int zipRawEntryLength(unsigned char *p) {
-    unsigned int prevlensize, lensize, len;
-    /* Byte-size of encoded length of previous entry */
-    zipDecodeLength(p,&prevlensize);
-    /* Encoded length of this entry's payload */
-    len = zipDecodeLength(p+prevlensize, &lensize);
-    return prevlensize+lensize+len;
+    zlentry e = zipEntry(p);
+    return e.headersize + e.len;
 }
 
 /* Create a new empty ziplist. */

From 0c0d0564121eb758fa25e2edf43cb1cf42aadbbc Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 17:38:23 +0200
Subject: [PATCH 28/66] extract a generic delete function that can be used in
 pop and delete(range)

---
 ziplist.c | 103 ++++++++++++++++++++++++------------------------------
 1 file changed, 45 insertions(+), 58 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 16e9dbc1..57639c3a 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -56,6 +56,7 @@ typedef struct zlentry {
     unsigned int lensize, len;
     unsigned int headersize;
     unsigned char encoding;
+    unsigned char *p;
 } zlentry;
 
 /* Return bytes needed to store integer encoded by 'encoding' */
@@ -206,6 +207,7 @@ static zlentry zipEntry(unsigned char *p) {
     e.len = zipDecodeLength(p+e.prevrawlensize,&e.lensize);
     e.headersize = e.prevrawlensize+e.lensize;
     e.encoding = ZIP_ENCODING(p+e.prevrawlensize);
+    e.p = p;
     return e;
 }
 
@@ -246,6 +248,42 @@ static unsigned char *ziplistTail(unsigned char *zl) {
         p += zipRawEntryLength(p);
     }
     return q;
+
+/* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
+static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int num) {
+    unsigned int i, totlen, deleted = 0;
+    int nextdiff = 0;
+    zlentry first = zipEntry(p);
+    for (i = 0; p[0] != ZIP_END && i < num; i++) {
+        p += zipRawEntryLength(p);
+        deleted++;
+    }
+
+    totlen = p-first.p;
+    if (totlen > 0) {
+        if (p[0] != ZIP_END) {
+            /* Tricky: storing the prevlen in this entry might reduce or
+             * increase the number of bytes needed, compared to the current
+             * prevlen. Note that we can always store this length because
+             * it was previously stored by an entry that is being deleted. */
+            nextdiff = zipPrevLenByteDiff(p,first.prevrawlen);
+            zipEncodeLength(p-nextdiff,ZIP_ENC_RAW,first.prevrawlen);
+
+            /* Update offset for tail */
+            ZIPLIST_TAIL_OFFSET(zl) -= totlen+nextdiff;
+
+            /* Move tail to the front of the ziplist */
+            memmove(first.p,p-nextdiff,ZIPLIST_BYTES(zl)-(p-zl)-1+nextdiff);
+        } else {
+            /* The entire tail was deleted. No need to move memory. */
+            ZIPLIST_TAIL_OFFSET(zl) = (first.p-zl)-first.prevrawlen;
+        }
+
+        /* Resize and update length */
+        zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen+nextdiff);
+        ZIPLIST_INCR_LENGTH(zl,-deleted);
+    }
+    return zl;
 }
 
 unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where) {
@@ -309,9 +347,7 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
 }
 
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
-    unsigned int curlen = ZIPLIST_BYTES(zl), rawlen;
     zlentry entry;
-    int nextdiff = 0;
     unsigned char *p;
     long long value;
     if (target) *target = NULL;
@@ -321,7 +357,6 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     if (*p == ZIP_END) return zl;
 
     entry = zipEntry(p);
-    rawlen = entry.headersize+entry.len;
     if (target) {
         if (entry.encoding == ZIP_ENC_RAW) {
             *target = sdsnewlen(p+entry.headersize,entry.len);
@@ -331,32 +366,7 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
         }
     }
 
-    if (where == ZIPLIST_HEAD) {
-        /* The next entry will now be the head of the list */
-        if (p[rawlen] != ZIP_END) {
-            /* Tricky: storing the length of the previous entry in the next
-             * entry (this previous length is always 0 when popping from the
-             * head), might reduce the number of bytes needed.
-             *
-             * In this special case (new length is 0), we know that the
-             * byte difference to store is always <= 0, which means that
-             * we always have space to store it. */
-            nextdiff = zipPrevLenByteDiff(p+rawlen,0);
-            zipEncodeLength(p+rawlen-nextdiff,ZIP_ENC_RAW,0);
-        }
-        /* Move list to the front */
-        memmove(p,p+rawlen-nextdiff,curlen-ZIPLIST_HEADER_SIZE-rawlen+nextdiff);
-
-        /* Subtract the gained space from the tail offset */
-        ZIPLIST_TAIL_OFFSET(zl) -= rawlen+nextdiff;
-    } else {
-        /* Subtract the length of the previous element from the tail offset. */
-        ZIPLIST_TAIL_OFFSET(zl) -= entry.prevrawlen;
-    }
-
-    /* Resize and update length */
-    zl = ziplistResize(zl,curlen-rawlen+nextdiff);
-    ZIPLIST_INCR_LENGTH(zl,-1);
+    zl = __ziplistDelete(zl,p,1);
     return zl;
 }
 
@@ -401,42 +411,19 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl
 
 /* Delete a range of entries from the ziplist. */
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
-    unsigned char *p, *first = ziplistIndex(zl, index);
-    unsigned int i, totlen, deleted = 0;
-    for (p = first, i = 0; *p != ZIP_END && i < num; i++) {
-        p += zipRawEntryLength(p);
-        deleted++;
-    }
-
-    totlen = p-first;
-    if (totlen > 0) {
-        /* Move current tail to the new tail when there *is* a tail */
-        if (*p != ZIP_END) memmove(first,p,ZIPLIST_BYTES(zl)-(p-zl)-1);
-
-        /* Resize and update length */
-        zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-totlen);
-        ZIPLIST_INCR_LENGTH(zl,-deleted);
-    }
-    return zl;
+    unsigned char *p = ziplistIndex(zl,index);
+    return __ziplistDelete(zl,p,num);
 }
 
 /* Delete a single entry from the ziplist, pointed to by *p.
  * Also update *p in place, to be able to iterate over the
  * ziplist, while deleting entries. */
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
-    unsigned int offset = *p-zl, tail, len;
-    len = zipRawEntryLength(*p);
-    tail = ZIPLIST_BYTES(zl)-offset-len-1;
+    unsigned int offset = *p-zl;
+    zl = __ziplistDelete(zl,*p,1);
 
-    /* Move current tail to the new tail when there *is* a tail */
-    if (tail > 0) memmove(*p,*p+len,tail);
-
-    /* Resize and update length */
-    zl = ziplistResize(zl, ZIPLIST_BYTES(zl)-len);
-    ZIPLIST_INCR_LENGTH(zl,-1);
-
-    /* Store new pointer to current element in p.
-     * This needs to be done because zl can change on realloc. */
+    /* Store pointer to current element in p, because ziplistDelete will
+     * do a realloc which might result in a different "zl"-pointer. */
     *p = zl+offset;
     return zl;
 }

From 6435c76772bfc846c4f336d07f56668aef33a38b Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 18:52:49 +0200
Subject: [PATCH 29/66] function to insert an element at an arbitrary position
 in the list

---
 ziplist.c | 77 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 57639c3a..e54f3211 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -286,66 +286,79 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int n
     return zl;
 }
 
-unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where) {
-    unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen;
-    unsigned char *p, *curtail;
+/* Insert item at "p". */
+static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
+    unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0;
+    unsigned int offset, nextdiff = 0;
+    unsigned char *tail;
     char encoding = ZIP_ENC_RAW;
     long long value;
+    zlentry entry;
 
-    /* We need to store the length of the current tail when the list
-     * is non-empty and we push at the tail. */
-    curtail = zl+ZIPLIST_TAIL_OFFSET(zl);
-    if (where == ZIPLIST_TAIL && curtail[0] != ZIP_END) {
-        prevlen = zipRawEntryLength(curtail);
+    /* Find out prevlen for the entry that is inserted. */
+    if (p[0] != ZIP_END) {
+        entry = zipEntry(p);
+        prevlen = entry.prevrawlen;
     } else {
-        prevlen = 0;
+        tail = ziplistTail(zl);
+        if (tail[0] != ZIP_END) {
+            prevlen = zipRawEntryLength(tail);
+        }
     }
 
     /* See if the entry can be encoded */
-    if (zipTryEncoding(entry,&value,&encoding)) {
+    if (zipTryEncoding(s,&value,&encoding)) {
         reqlen = zipEncodingSize(encoding);
     } else {
-        reqlen = elen;
+        reqlen = slen;
     }
 
     /* We need space for both the length of the previous entry and
      * the length of the payload. */
     reqlen += zipEncodeLength(NULL,ZIP_ENC_RAW,prevlen);
-    reqlen += zipEncodeLength(NULL,encoding,elen);
+    reqlen += zipEncodeLength(NULL,encoding,slen);
 
-    /* Resize the ziplist and move if needed */
-    zl = ziplistResize(zl,curlen+reqlen);
-    if (where == ZIPLIST_HEAD) {
-        p = zl+ZIPLIST_HEADER_SIZE;
-        if (*p != ZIP_END) {
-            /* Subtract one because of the ZIP_END bytes */
-            memmove(p+reqlen,p,curlen-ZIPLIST_HEADER_SIZE-1);
-        }
+    /* When the insert position is not equal to the tail, we need to
+     * make sure that the next entry can hold this entry's length in
+     * its prevlen field. */
+    nextdiff = p[0] != ZIP_END ? zipPrevLenByteDiff(p,reqlen) : 0;
+
+    /* Store offset because a realloc may change the address of zl. */
+    offset = p-zl;
+    zl = ziplistResize(zl,curlen+reqlen+nextdiff);
+    p = zl+offset;
+
+    /* Apply memory move when necessary and update tail offset. */
+    if (p[0] != ZIP_END) {
+        /* Subtract one because of the ZIP_END bytes */
+        memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);
+        /* Encode this entry's raw length in the next entry. */
+        zipEncodeLength(p+reqlen,ZIP_ENC_RAW,reqlen);
+        /* Update offset for tail */
+        ZIPLIST_TAIL_OFFSET(zl) += reqlen+nextdiff;
     } else {
-        p = zl+curlen-1;
-    }
-
-    /* Update tail offset if this is not the first element */
-    if (curtail[0] != ZIP_END) {
-        if (where == ZIPLIST_HEAD) {
-            ZIPLIST_TAIL_OFFSET(zl) += reqlen;
-        } else {
-            ZIPLIST_TAIL_OFFSET(zl) += prevlen;
-        }
+        /* This element will be the new tail. */
+        ZIPLIST_TAIL_OFFSET(zl) = p-zl;
     }
 
     /* Write the entry */
     p += zipEncodeLength(p,ZIP_ENC_RAW,prevlen);
-    p += zipEncodeLength(p,encoding,elen);
+    p += zipEncodeLength(p,encoding,slen);
     if (encoding != ZIP_ENC_RAW) {
         zipSaveInteger(p,value,encoding);
     } else {
-        memcpy(p,entry,elen);
+        memcpy(p,s,slen);
     }
     ZIPLIST_INCR_LENGTH(zl,1);
     return zl;
 }
 
+unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
+    unsigned char *p;
+    p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : (zl+ZIPLIST_BYTES(zl)-1);
+    return __ziplistInsert(zl,p,s,slen);
+}
+
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     zlentry entry;
     unsigned char *p;

From 1ce81fa59b9f49cdd764ca35b312816396c93239 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 19:09:37 +0200
Subject: [PATCH 30/66] replace functions to get pointers to head and tail by
 macros

---
 ziplist.c | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index e54f3211..8ac84975 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -48,6 +48,9 @@
 #define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
 #define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
 #define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
+#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
+#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+ZIPLIST_TAIL_OFFSET(zl))
+#define ZIPLIST_ENTRY_END(zl) ((zl)+ZIPLIST_BYTES(zl)-1)
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
     if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
 
@@ -236,19 +239,6 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
     return zl;
 }
 
-static unsigned char *ziplistHead(unsigned char *zl) {
-    return zl+ZIPLIST_HEADER_SIZE;
-}
-
-static unsigned char *ziplistTail(unsigned char *zl) {
-    unsigned char *p, *q;
-    p = q = ziplistHead(zl);
-    while (*p != ZIP_END) {
-        q = p;
-        p += zipRawEntryLength(p);
-    }
-    return q;
-
 /* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
 static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int num) {
     unsigned int i, totlen, deleted = 0;
@@ -300,7 +290,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
         entry = zipEntry(p);
         prevlen = entry.prevrawlen;
     } else {
-        tail = ziplistTail(zl);
+        tail = ZIPLIST_ENTRY_TAIL(zl);
         if (tail[0] != ZIP_END) {
             prevlen = zipRawEntryLength(tail);
         }
@@ -355,7 +345,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
 
 unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
     unsigned char *p;
-    p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : (zl+ZIPLIST_BYTES(zl)-1);
+    p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);
     return __ziplistInsert(zl,p,s,slen);
 }
 
@@ -366,7 +356,7 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     if (target) *target = NULL;
 
     /* Get pointer to element to remove */
-    p = (where == ZIPLIST_HEAD) ? ziplistHead(zl) : ziplistTail(zl);
+    p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_TAIL(zl);
     if (*p == ZIP_END) return zl;
 
     entry = zipEntry(p);
@@ -496,7 +486,7 @@ void ziplistRepr(unsigned char *zl) {
     zlentry entry;
 
     printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl), ZIPLIST_LENGTH(zl));
-    p = ziplistHead(zl);
+    p = ZIPLIST_ENTRY_HEAD(zl);
     while(*p != ZIP_END) {
         entry = zipEntry(p);
         printf("{offset %ld, header %u, payload %u} ",p-zl,entry.headersize,entry.len);

From 7b1f85c0a24aecb2ed9a9d3cf9a4b61bd03f642b Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 19:26:35 +0200
Subject: [PATCH 31/66] use simpler encoding for the length of the previous
 entry

---
 ziplist.c | 43 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 7 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 8ac84975..b31e7425 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -133,12 +133,41 @@ static unsigned int zipEncodeLength(unsigned char *p, char encoding, unsigned in
     return len;
 }
 
+/* Decode the length of the previous element stored at "p". */
+static unsigned int zipPrevDecodeLength(unsigned char *p, unsigned int *lensize) {
+    unsigned int len = *p;
+    if (len < ZIP_BIGLEN) {
+        if (lensize) *lensize = 1;
+    } else {
+        if (lensize) *lensize = 1+sizeof(len);
+        memcpy(&len,p+1,sizeof(len));
+    }
+    return len;
+}
+
+/* Encode the length of the previous entry and write it to "p". Return the
+ * number of bytes needed to encode this length if "p" is NULL. */
+static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) {
+    if (p == NULL) {
+        return (len < ZIP_BIGLEN) ? 1 : sizeof(len)+1;
+    } else {
+        if (len < ZIP_BIGLEN) {
+            p[0] = len;
+            return 1;
+        } else {
+            p[0] = ZIP_BIGLEN;
+            memcpy(p+1,&len,sizeof(len));
+            return 1+sizeof(len);
+        }
+    }
+}
+
 /* Return the difference in number of bytes needed to store the new length
  * "len" on the entry pointed to by "p". */
 static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
     unsigned int prevlensize;
-    zipDecodeLength(p,&prevlensize);
-    return zipEncodeLength(NULL,ZIP_ENC_RAW,len)-prevlensize;
+    zipPrevDecodeLength(p,&prevlensize);
+    return zipPrevEncodeLength(NULL,len)-prevlensize;
 }
 
 /* Check if string pointed to by 'entry' can be encoded as an integer.
@@ -206,7 +235,7 @@ static long long zipLoadInteger(unsigned char *p, char encoding) {
 /* Return a struct with all information about an entry. */
 static zlentry zipEntry(unsigned char *p) {
     zlentry e;
-    e.prevrawlen = zipDecodeLength(p,&e.prevrawlensize);
+    e.prevrawlen = zipPrevDecodeLength(p,&e.prevrawlensize);
     e.len = zipDecodeLength(p+e.prevrawlensize,&e.lensize);
     e.headersize = e.prevrawlensize+e.lensize;
     e.encoding = ZIP_ENCODING(p+e.prevrawlensize);
@@ -257,7 +286,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int n
              * prevlen. Note that we can always store this length because
              * it was previously stored by an entry that is being deleted. */
             nextdiff = zipPrevLenByteDiff(p,first.prevrawlen);
-            zipEncodeLength(p-nextdiff,ZIP_ENC_RAW,first.prevrawlen);
+            zipPrevEncodeLength(p-nextdiff,first.prevrawlen);
 
             /* Update offset for tail */
             ZIPLIST_TAIL_OFFSET(zl) -= totlen+nextdiff;
@@ -305,7 +334,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
 
     /* We need space for both the length of the previous entry and
      * the length of the payload. */
-    reqlen += zipEncodeLength(NULL,ZIP_ENC_RAW,prevlen);
+    reqlen += zipPrevEncodeLength(NULL,prevlen);
     reqlen += zipEncodeLength(NULL,encoding,slen);
 
     /* When the insert position is not equal to the tail, we need to
@@ -323,7 +352,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
         /* Subtract one because of the ZIP_END bytes */
         memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);
         /* Encode this entry's raw length in the next entry. */
-        zipEncodeLength(p+reqlen,ZIP_ENC_RAW,reqlen);
+        zipPrevEncodeLength(p+reqlen,reqlen);
         /* Update offset for tail */
         ZIPLIST_TAIL_OFFSET(zl) += reqlen+nextdiff;
     } else {
@@ -332,7 +361,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
     }
 
     /* Write the entry */
-    p += zipEncodeLength(p,ZIP_ENC_RAW,prevlen);
+    p += zipPrevEncodeLength(p,prevlen);
     p += zipEncodeLength(p,encoding,slen);
     if (encoding != ZIP_ENC_RAW) {
         zipSaveInteger(p,value,encoding);

From a24ba809d1709e4cebf032994c03842611575e2c Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 19:34:14 +0200
Subject: [PATCH 32/66] fix compile warnings

---
 ziplist.c | 35 ++++++++++++++++++-----------------
 ziplist.h |  6 +++---
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index b31e7425..ec988bc8 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -173,12 +173,12 @@ static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
 /* Check if string pointed to by 'entry' can be encoded as an integer.
  * Stores the integer value in 'v' and its encoding in 'encoding'.
  * Warning: this function requires a NULL-terminated string! */
-static int zipTryEncoding(unsigned char *entry, long long *v, char *encoding) {
+static int zipTryEncoding(char *entry, long long *v, char *encoding) {
     long long value;
     char *eptr;
 
     if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
-        value = strtoll((char*)entry,&eptr,10);
+        value = strtoll(entry,&eptr,10);
         if (eptr[0] != '\0') return 0;
         if (value >= SHRT_MIN && value <= SHRT_MAX) {
             *encoding = ZIP_ENC_SHORT;
@@ -306,7 +306,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int n
 }
 
 /* Insert item at "p". */
-static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
+static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen) {
     unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0;
     unsigned int offset, nextdiff = 0;
     unsigned char *tail;
@@ -372,7 +372,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
     return zl;
 }
 
-unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
+unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int where) {
     unsigned char *p;
     p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);
     return __ziplistInsert(zl,p,s,slen);
@@ -422,7 +422,7 @@ unsigned char *ziplistNext(unsigned char *p) {
  * on the encoding of the entry. 'e' is always set to NULL to be able
  * to find out whether the string pointer or the integer value was set.
  * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
-unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) {
+unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval) {
     zlentry entry;
     if (*p == ZIP_END) return 0;
     if (sstr) *sstr = NULL;
@@ -431,7 +431,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl
     if (entry.encoding == ZIP_ENC_RAW) {
         if (sstr) {
             *slen = entry.len;
-            *sstr = p+entry.headersize;
+            *sstr = (char*)p+entry.headersize;
         }
     } else {
         if (sval) {
@@ -461,9 +461,9 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
 }
 
 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
-unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
+unsigned int ziplistCompare(unsigned char *p, char *sstr, unsigned int slen) {
     zlentry entry;
-    unsigned char sencoding;
+    char sencoding;
     long long val, sval;
     if (*p == ZIP_END) return 0;
 
@@ -535,10 +535,10 @@ void ziplistRepr(unsigned char *zl) {
 
 unsigned char *createList() {
     unsigned char *zl = ziplistNew();
-    zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
-    zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
-    zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
-    zl = ziplistPush(zl, (unsigned char*)"1024", 4, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, "foo", 3, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, "quux", 4, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, "hello", 5, ZIPLIST_HEAD);
+    zl = ziplistPush(zl, "1024", 4, ZIPLIST_TAIL);
     return zl;
 }
 
@@ -562,7 +562,8 @@ unsigned char *createIntList() {
 }
 
 int main(int argc, char **argv) {
-    unsigned char *zl, *p, *q, *entry;
+    unsigned char *zl, *p;
+    char *entry;
     unsigned int elen;
     long long value;
     sds s;
@@ -716,21 +717,21 @@ int main(int argc, char **argv) {
         p = ziplistIndex(zl, 0);
         if (!ziplistCompare(p,"hello",5)) {
             printf("ERROR: not \"hello\"\n");
-            return;
+            return 1;
         }
         if (ziplistCompare(p,"hella",5)) {
             printf("ERROR: \"hella\"\n");
-            return;
+            return 1;
         }
 
         p = ziplistIndex(zl, 3);
         if (!ziplistCompare(p,"1024",4)) {
             printf("ERROR: not \"1024\"\n");
-            return;
+            return 1;
         }
         if (ziplistCompare(p,"1025",4)) {
             printf("ERROR: \"1025\"\n");
-            return;
+            return 1;
         }
         printf("SUCCESS\n");
     }
diff --git a/ziplist.h b/ziplist.h
index fecfe9b6..aeeccdb8 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -2,13 +2,13 @@
 #define ZIPLIST_TAIL 1
 
 unsigned char *ziplistNew(void);
-unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int elen, int where);
+unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int where);
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where);
 unsigned char *ziplistIndex(unsigned char *zl, unsigned int index);
 unsigned char *ziplistNext(unsigned char *p);
-unsigned int ziplistGet(unsigned char *p, unsigned char **e, unsigned int *elen, long long *v);
+unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
-unsigned int ziplistCompare(unsigned char *p, unsigned char *entry, unsigned int elen);
+unsigned int ziplistCompare(unsigned char *p, char *entry, unsigned int elen);
 unsigned int ziplistLen(unsigned char *zl);
 unsigned int ziplistSize(unsigned char *zl);

From c03206fdf1dc0241d97204aba813c13b82f40d6e Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sat, 29 May 2010 22:17:16 +0200
Subject: [PATCH 33/66] ziplistIndex now accepts negative indices

---
 ziplist.c | 111 +++++++++++++++++++++++++++++++++++++++++++++++++-----
 ziplist.h |   2 +-
 2 files changed, 102 insertions(+), 11 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index ec988bc8..b29c630b 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -402,15 +402,29 @@ unsigned char *ziplistPop(unsigned char *zl, sds *target, int where) {
     return zl;
 }
 
-/* Returns an offset to use for iterating with ziplistNext. */
-unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
-    unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
-    unsigned int i = 0;
-    for (; i < index; i++) {
-        if (*p == ZIP_END) break;
-        p += zipRawEntryLength(p);
+/* Returns an offset to use for iterating with ziplistNext. When the given
+ * index is negative, the list is traversed back to front. When the list
+ * doesn't contain an element at the provided index, NULL is returned. */
+unsigned char *ziplistIndex(unsigned char *zl, int index) {
+    unsigned char *p;
+    zlentry entry;
+    if (index < 0) {
+        index = (-index)-1;
+        p = ZIPLIST_ENTRY_TAIL(zl);
+        if (p[0] != ZIP_END) {
+            entry = zipEntry(p);
+            while (entry.prevrawlen > 0 && index--) {
+                p -= entry.prevrawlen;
+                entry = zipEntry(p);
+            }
+        }
+    } else {
+        p = ZIPLIST_ENTRY_HEAD(zl);
+        while (p[0] != ZIP_END && index--) {
+            p += zipRawEntryLength(p);
+        }
     }
-    return p;
+    return p[0] == ZIP_END || index > 0 ? NULL : p;
 }
 
 /* Return pointer to next entry in ziplist. */
@@ -424,7 +438,7 @@ unsigned char *ziplistNext(unsigned char *p) {
  * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
 unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval) {
     zlentry entry;
-    if (*p == ZIP_END) return 0;
+    if (p == NULL || p[0] == ZIP_END) return 0;
     if (sstr) *sstr = NULL;
 
     entry = zipEntry(p);
@@ -444,7 +458,7 @@ unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long
 /* Delete a range of entries from the ziplist. */
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
     unsigned char *p = ziplistIndex(zl,index);
-    return __ziplistDelete(zl,p,num);
+    return p == NULL ? zl : __ziplistDelete(zl,p,num);
 }
 
 /* Delete a single entry from the ziplist, pointed to by *p.
@@ -590,6 +604,83 @@ int main(int argc, char **argv) {
     printf("Pop tail: %s (length %ld)\n", s, sdslen(s));
     ziplistRepr(zl);
 
+    printf("Get element at index 3:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 3);
+        if (!ziplistGet(p, &entry, &elen, &value)) {
+            printf("ERROR: Could not access index 3\n");
+            return 1;
+        }
+        if (entry) {
+            fwrite(entry,elen,1,stdout);
+            printf("\n");
+        } else {
+            printf("%lld\n", value);
+        }
+        printf("\n");
+    }
+
+    printf("Get element at index 4 (out of range):\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, 4);
+        if (p == NULL) {
+            printf("No entry\n");
+        } else {
+            printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl);
+            return 1;
+        }
+        printf("\n");
+    }
+
+    printf("Get element at index -1 (last element):\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, -1);
+        if (!ziplistGet(p, &entry, &elen, &value)) {
+            printf("ERROR: Could not access index -1\n");
+            return 1;
+        }
+        if (entry) {
+            fwrite(entry,elen,1,stdout);
+            printf("\n");
+        } else {
+            printf("%lld\n", value);
+        }
+        printf("\n");
+    }
+
+    printf("Get element at index -4 (first element):\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, -4);
+        if (!ziplistGet(p, &entry, &elen, &value)) {
+            printf("ERROR: Could not access index -4\n");
+            return 1;
+        }
+        if (entry) {
+            fwrite(entry,elen,1,stdout);
+            printf("\n");
+        } else {
+            printf("%lld\n", value);
+        }
+        printf("\n");
+    }
+
+    printf("Get element at index -5 (reverse out of range):\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, -5);
+        if (p == NULL) {
+            printf("No entry\n");
+        } else {
+            printf("ERROR: Out of range index should return NULL, returned offset: %ld\n", p-zl);
+            return 1;
+        }
+        printf("\n");
+    }
+
     printf("Iterate list from 0 to end:\n");
     {
         zl = createList();
diff --git a/ziplist.h b/ziplist.h
index aeeccdb8..29be0b40 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -4,7 +4,7 @@
 unsigned char *ziplistNew(void);
 unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int where);
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where);
-unsigned char *ziplistIndex(unsigned char *zl, unsigned int index);
+unsigned char *ziplistIndex(unsigned char *zl, int index);
 unsigned char *ziplistNext(unsigned char *p);
 unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);

From 177a0a0b0d66ca5d25cf3d06907292e8c02f1825 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 01:37:03 +0200
Subject: [PATCH 34/66] code style consistency fixes

---
 ziplist.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index b29c630b..48ec2a34 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -340,7 +340,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, char
     /* When the insert position is not equal to the tail, we need to
      * make sure that the next entry can hold this entry's length in
      * its prevlen field. */
-    nextdiff = p[0] != ZIP_END ? zipPrevLenByteDiff(p,reqlen) : 0;
+    nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0;
 
     /* Store offset because a realloc may change the address of zl. */
     offset = p-zl;
@@ -424,12 +424,12 @@ unsigned char *ziplistIndex(unsigned char *zl, int index) {
             p += zipRawEntryLength(p);
         }
     }
-    return p[0] == ZIP_END || index > 0 ? NULL : p;
+    return (p[0] == ZIP_END || index > 0) ? NULL : p;
 }
 
 /* Return pointer to next entry in ziplist. */
 unsigned char *ziplistNext(unsigned char *p) {
-    return *p == ZIP_END ? p : p+zipRawEntryLength(p);
+    return (p[0] == ZIP_END) ? NULL : p+zipRawEntryLength(p);
 }
 
 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
@@ -479,7 +479,7 @@ unsigned int ziplistCompare(unsigned char *p, char *sstr, unsigned int slen) {
     zlentry entry;
     char sencoding;
     long long val, sval;
-    if (*p == ZIP_END) return 0;
+    if (p[0] == ZIP_END) return 0;
 
     entry = zipEntry(p);
     if (entry.encoding == ZIP_ENC_RAW) {

From 033fb554bec6523beb638965dc8f4adb96887332 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 01:38:56 +0200
Subject: [PATCH 35/66] expose extra functionality from ziplist.c

---
 ziplist.c | 19 +++++++++++++++----
 ziplist.h |  2 ++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 48ec2a34..4bc4f75c 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -432,6 +432,12 @@ unsigned char *ziplistNext(unsigned char *p) {
     return (p[0] == ZIP_END) ? NULL : p+zipRawEntryLength(p);
 }
 
+/* Return pointer to previous entry in ziplist. */
+unsigned char *ziplistPrev(unsigned char *p) {
+    zlentry entry = zipEntry(p);
+    return (entry.prevrawlen == 0) ? NULL : p-entry.prevrawlen;
+}
+
 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
  * on the encoding of the entry. 'e' is always set to NULL to be able
  * to find out whether the string pointer or the integer value was set.
@@ -455,10 +461,9 @@ unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long
     return 1;
 }
 
-/* Delete a range of entries from the ziplist. */
-unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
-    unsigned char *p = ziplistIndex(zl,index);
-    return p == NULL ? zl : __ziplistDelete(zl,p,num);
+/* Insert an entry at "p". */
+unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen) {
+    return __ziplistInsert(zl,p,s,slen);
 }
 
 /* Delete a single entry from the ziplist, pointed to by *p.
@@ -474,6 +479,12 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
     return zl;
 }
 
+/* Delete a range of entries from the ziplist. */
+unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
+    unsigned char *p = ziplistIndex(zl,index);
+    return (p == NULL) ? zl : __ziplistDelete(zl,p,num);
+}
+
 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
 unsigned int ziplistCompare(unsigned char *p, char *sstr, unsigned int slen) {
     zlentry entry;
diff --git a/ziplist.h b/ziplist.h
index 29be0b40..8d37a8f5 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -6,8 +6,10 @@ unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int wh
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where);
 unsigned char *ziplistIndex(unsigned char *zl, int index);
 unsigned char *ziplistNext(unsigned char *p);
+unsigned char *ziplistPrev(unsigned char *p);
 unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
+unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
 unsigned int ziplistCompare(unsigned char *p, char *entry, unsigned int elen);
 unsigned int ziplistLen(unsigned char *zl);

From 0f3dfa87bc8870c1da6867b7e2d9d70f7e21fc93 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 01:39:41 +0200
Subject: [PATCH 36/66] change delete function to accept a direction argument,
 so "p" can be properly updated

---
 ziplist.c | 48 ++++++++++++++++++++++++++++++++++++++++++++----
 ziplist.h |  2 +-
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 4bc4f75c..71d965a8 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -469,13 +469,19 @@ unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsig
 /* Delete a single entry from the ziplist, pointed to by *p.
  * Also update *p in place, to be able to iterate over the
  * ziplist, while deleting entries. */
-unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p, int direction) {
     unsigned int offset = *p-zl;
     zl = __ziplistDelete(zl,*p,1);
 
     /* Store pointer to current element in p, because ziplistDelete will
-     * do a realloc which might result in a different "zl"-pointer. */
-    *p = zl+offset;
+     * do a realloc which might result in a different "zl"-pointer.
+     * When the delete direction is back to front, we might delete the last
+     * entry and end up with "p" pointing to ZIP_END, so check this. */
+    if (*(zl+offset) == ZIP_END && direction == ZIPLIST_HEAD) {
+        *p = ZIPLIST_ENTRY_TAIL(zl);
+    } else {
+        *p = zl+offset;
+    }
     return zl;
 }
 
@@ -755,6 +761,40 @@ int main(int argc, char **argv) {
         printf("\n");
     }
 
+    printf("Iterate from back to front:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, -1);
+        while (ziplistGet(p, &entry, &elen, &value)) {
+            printf("Entry: ");
+            if (entry) {
+                fwrite(entry,elen,1,stdout);
+            } else {
+                printf("%lld", value);
+            }
+            p = ziplistPrev(p);
+            printf("\n");
+        }
+        printf("\n");
+    }
+
+    printf("Iterate from back to front, deleting all items:\n");
+    {
+        zl = createList();
+        p = ziplistIndex(zl, -1);
+        while (ziplistGet(p, &entry, &elen, &value)) {
+            printf("Entry: ");
+            if (entry) {
+                fwrite(entry,elen,1,stdout);
+            } else {
+                printf("%lld", value);
+            }
+            zl = ziplistDelete(zl, &p, ZIPLIST_HEAD);
+            printf("\n");
+        }
+        printf("\n");
+    }
+
     printf("Delete inclusive range 0,0:\n");
     {
         zl = createList();
@@ -797,7 +837,7 @@ int main(int argc, char **argv) {
         while (ziplistGet(p, &entry, &elen, &value)) {
             if (entry && strncmp("foo", entry, elen) == 0) {
                 printf("Delete foo\n");
-                zl = ziplistDelete(zl, &p);
+                zl = ziplistDelete(zl, &p, ZIPLIST_TAIL);
             } else {
                 printf("Entry: ");
                 if (entry) {
diff --git a/ziplist.h b/ziplist.h
index 8d37a8f5..08035dd5 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -8,8 +8,8 @@ unsigned char *ziplistIndex(unsigned char *zl, int index);
 unsigned char *ziplistNext(unsigned char *p);
 unsigned char *ziplistPrev(unsigned char *p);
 unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
-unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen);
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p, int direction);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
 unsigned int ziplistCompare(unsigned char *p, char *entry, unsigned int elen);
 unsigned int ziplistLen(unsigned char *zl);

From c7d9d662a4c8ba1c17676055fd30a25ffa79304e Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 01:46:59 +0200
Subject: [PATCH 37/66] generic push function that supports the dual encoding

---
 redis.c | 66 +++++++++++++++++++++++++++++++++++----------------------
 1 file changed, 41 insertions(+), 25 deletions(-)

diff --git a/redis.c b/redis.c
index 92ae07b7..25068618 100644
--- a/redis.c
+++ b/redis.c
@@ -75,6 +75,7 @@
 #include "lzf.h"    /* LZF compression library */
 #include "pqsort.h" /* Partial qsort for SORT+LIMIT */
 #include "zipmap.h" /* Compact dictionary-alike data structure */
+#include "ziplist.h" /* Compact list data structure */
 #include "sha1.h"   /* SHA1 is used for DEBUG DIGEST */
 #include "release.h" /* Release and/or git repository information */
 
@@ -124,10 +125,12 @@
 /* Objects encoding. Some kind of objects like Strings and Hashes can be
  * internally represented in multiple ways. The 'encoding' field of the object
  * is set to one of this fields for this object. */
-#define REDIS_ENCODING_RAW 0    /* Raw representation */
-#define REDIS_ENCODING_INT 1    /* Encoded as integer */
-#define REDIS_ENCODING_ZIPMAP 2 /* Encoded as zipmap */
-#define REDIS_ENCODING_HT 3     /* Encoded as an hash table */
+#define REDIS_ENCODING_RAW 0     /* Raw representation */
+#define REDIS_ENCODING_INT 1     /* Encoded as integer */
+#define REDIS_ENCODING_HT 2      /* Encoded as hash table */
+#define REDIS_ENCODING_ZIPMAP 3  /* Encoded as zipmap */
+#define REDIS_ENCODING_LIST 4    /* Encoded as zipmap */
+#define REDIS_ENCODING_ZIPLIST 5 /* Encoded as ziplist */
 
 static char* strencoding[] = {
     "raw", "int", "zipmap", "hashtable"
@@ -3016,7 +3019,16 @@ static void freeStringObject(robj *o) {
 }
 
 static void freeListObject(robj *o) {
-    listRelease((list*) o->ptr);
+    switch (o->encoding) {
+    case REDIS_ENCODING_LIST:
+        listRelease((list*) o->ptr);
+        break;
+    case REDIS_ENCODING_ZIPLIST:
+        zfree(o->ptr);
+        break;
+    default:
+        redisPanic("Unknown list encoding type");
+    }
 }
 
 static void freeSetObject(robj *o) {
@@ -4760,26 +4772,36 @@ static void moveCommand(redisClient *c) {
 }
 
 /* =================================== Lists ================================ */
-static void pushGenericCommand(redisClient *c, int where) {
-    robj *lobj;
-    list *list;
+static void lPush(robj *subject, robj *value, int where) {
+    if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
+        int pos = (where == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
+        value = getDecodedObject(value);
+        subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),pos);
+        decrRefCount(value);
+    } else if (subject->encoding == REDIS_ENCODING_LIST) {
+        if (where == REDIS_HEAD) {
+            listAddNodeHead(subject->ptr,value);
+        } else {
+            listAddNodeTail(subject->ptr,value);
+        }
+        incrRefCount(value);
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+}
 
-    lobj = lookupKeyWrite(c->db,c->argv[1]);
+
+static void pushGenericCommand(redisClient *c, int where) {
+    robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
     if (lobj == NULL) {
         if (handleClientsWaitingListPush(c,c->argv[1],c->argv[2])) {
             addReply(c,shared.cone);
             return;
         }
-        lobj = createListObject();
-        list = lobj->ptr;
-        if (where == REDIS_HEAD) {
-            listAddNodeHead(list,c->argv[2]);
-        } else {
-            listAddNodeTail(list,c->argv[2]);
-        }
+        lobj = createObject(REDIS_LIST,ziplistNew());
+        lobj->encoding = REDIS_ENCODING_ZIPLIST;
         dictAdd(c->db->dict,c->argv[1],lobj);
         incrRefCount(c->argv[1]);
-        incrRefCount(c->argv[2]);
     } else {
         if (lobj->type != REDIS_LIST) {
             addReply(c,shared.wrongtypeerr);
@@ -4789,16 +4811,10 @@ static void pushGenericCommand(redisClient *c, int where) {
             addReply(c,shared.cone);
             return;
         }
-        list = lobj->ptr;
-        if (where == REDIS_HEAD) {
-            listAddNodeHead(list,c->argv[2]);
-        } else {
-            listAddNodeTail(list,c->argv[2]);
-        }
-        incrRefCount(c->argv[2]);
     }
+    lPush(lobj,c->argv[2],where);
+    addReplyLongLong(c,lLength(lobj));
     server.dirty++;
-    addReplyLongLong(c,listLength(list));
 }
 
 static void lpushCommand(redisClient *c) {

From d72562f7baee43d41a5a6b3019e45f14c59b49d7 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 01:50:32 +0200
Subject: [PATCH 38/66] generic pop and length function for ziplist encoding

---
 redis.c | 81 +++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 56 insertions(+), 25 deletions(-)

diff --git a/redis.c b/redis.c
index 25068618..e9fa6306 100644
--- a/redis.c
+++ b/redis.c
@@ -4790,6 +4790,52 @@ static void lPush(robj *subject, robj *value, int where) {
     }
 }
 
+static robj *lPop(robj *subject, int where) {
+    robj *value = NULL;
+    if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
+        unsigned char *p;
+        char *v;
+        unsigned int vlen;
+        long long vval;
+        int pos = (where == REDIS_HEAD) ? 0 : -1;
+        p = ziplistIndex(subject->ptr,pos);
+        if (ziplistGet(p,&v,&vlen,&vval)) {
+            if (v) {
+                value = createStringObject(v,vlen);
+            } else {
+                value = createStringObjectFromLongLong(vval);
+            }
+        }
+        subject->ptr = ziplistDelete(subject->ptr,&p,ZIPLIST_TAIL);
+    } else if (subject->encoding == REDIS_ENCODING_LIST) {
+        list *list = subject->ptr;
+        listNode *ln;
+        if (where == REDIS_HEAD) {
+            ln = listFirst(list);
+        } else {
+            ln = listLast(list);
+        }
+        if (ln != NULL) {
+            value = listNodeValue(ln);
+            incrRefCount(value);
+            listDelNode(list,ln);
+        }
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+    return value;
+}
+
+static unsigned long lLength(robj *subject) {
+    if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
+        return ziplistLen(subject->ptr);
+    } else if (subject->encoding == REDIS_ENCODING_LIST) {
+        return listLength((list*)subject->ptr);
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+}
+
 
 static void pushGenericCommand(redisClient *c, int where) {
     robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
@@ -4826,14 +4872,9 @@ static void rpushCommand(redisClient *c) {
 }
 
 static void llenCommand(redisClient *c) {
-    robj *o;
-    list *l;
-
-    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
-        checkType(c,o,REDIS_LIST)) return;
-
-    l = o->ptr;
-    addReplyUlong(c,listLength(l));
+    robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.czero);
+    if (o == NULL || checkType(c,o,REDIS_LIST)) return;
+    addReplyUlong(c,lLength(o));
 }
 
 static void lindexCommand(redisClient *c) {
@@ -4880,26 +4921,16 @@ static void lsetCommand(redisClient *c) {
 }
 
 static void popGenericCommand(redisClient *c, int where) {
-    robj *o;
-    list *list;
-    listNode *ln;
+    robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk);
+    if (o == NULL || checkType(c,o,REDIS_LIST)) return;
 
-    if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
-        checkType(c,o,REDIS_LIST)) return;
-    list = o->ptr;
-
-    if (where == REDIS_HEAD)
-        ln = listFirst(list);
-    else
-        ln = listLast(list);
-
-    if (ln == NULL) {
+    robj *value = lPop(o,where);
+    if (value == NULL) {
         addReply(c,shared.nullbulk);
     } else {
-        robj *ele = listNodeValue(ln);
-        addReplyBulk(c,ele);
-        listDelNode(list,ln);
-        if (listLength(list) == 0) deleteKey(c->db,c->argv[1]);
+        addReplyBulk(c,value);
+        decrRefCount(value);
+        if (lLength(o) == 0) deleteKey(c->db,c->argv[1]);
         server.dirty++;
     }
 }

From 697bd5673fe1591eeabcd9ac8581d1e5019cf0ec Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 02:11:59 +0200
Subject: [PATCH 39/66] inline support for dual encoding in the LINDEX and LSET
 commands

---
 redis.c | 82 +++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 53 insertions(+), 29 deletions(-)

diff --git a/redis.c b/redis.c
index e9fa6306..27e35309 100644
--- a/redis.c
+++ b/redis.c
@@ -4878,45 +4878,69 @@ static void llenCommand(redisClient *c) {
 }
 
 static void lindexCommand(redisClient *c) {
-    robj *o;
+    robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk);
+    if (o == NULL || checkType(c,o,REDIS_LIST)) return;
     int index = atoi(c->argv[2]->ptr);
-    list *list;
-    listNode *ln;
 
-    if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
-        checkType(c,o,REDIS_LIST)) return;
-    list = o->ptr;
-
-    ln = listIndex(list, index);
-    if (ln == NULL) {
-        addReply(c,shared.nullbulk);
+    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+        unsigned char *p;
+        char *v;
+        unsigned int vlen;
+        long long vval;
+        p = ziplistIndex(o->ptr,index);
+        if (ziplistGet(p,&v,&vlen,&vval)) {
+            if (v) {
+                addReplySds(c,sdsnewlen(v,vlen));
+            } else {
+                addReplyLongLong(c,vval);
+            }
+        } else {
+            addReply(c,shared.nullbulk);
+        }
+    } else if (o->encoding == REDIS_ENCODING_LIST) {
+        listNode *ln = listIndex(o->ptr,index);
+        if (ln != NULL) {
+            addReply(c,(robj*)listNodeValue(ln));
+        } else {
+            addReply(c,shared.nullbulk);
+        }
     } else {
-        robj *ele = listNodeValue(ln);
-        addReplyBulk(c,ele);
+        redisPanic("Unknown list encoding");
     }
 }
 
 static void lsetCommand(redisClient *c) {
-    robj *o;
+    robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr);
+    if (o == NULL || checkType(c,o,REDIS_LIST)) return;
     int index = atoi(c->argv[2]->ptr);
-    list *list;
-    listNode *ln;
+    robj *value = c->argv[3];
 
-    if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr)) == NULL ||
-        checkType(c,o,REDIS_LIST)) return;
-    list = o->ptr;
-
-    ln = listIndex(list, index);
-    if (ln == NULL) {
-        addReply(c,shared.outofrangeerr);
+    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+        unsigned char *p, *zl = o->ptr;
+        p = ziplistIndex(zl,index);
+        if (p == NULL) {
+            addReply(c,shared.outofrangeerr);
+        } else {
+            o->ptr = ziplistDelete(o->ptr,&p,ZIPLIST_TAIL);
+            value = getDecodedObject(value);
+            o->ptr = ziplistInsert(o->ptr,p,value->ptr,sdslen(value->ptr));
+            decrRefCount(value);
+            addReply(c,shared.ok);
+            server.dirty++;
+        }
+    } else if (o->encoding == REDIS_ENCODING_LIST) {
+        listNode *ln = listIndex(o->ptr,index);
+        if (ln == NULL) {
+            addReply(c,shared.outofrangeerr);
+        } else {
+            decrRefCount((robj*)listNodeValue(ln));
+            listNodeValue(ln) = value;
+            incrRefCount(value);
+            addReply(c,shared.ok);
+            server.dirty++;
+        }
     } else {
-        robj *ele = listNodeValue(ln);
-
-        decrRefCount(ele);
-        listNodeValue(ln) = c->argv[3];
-        incrRefCount(c->argv[3]);
-        addReply(c,shared.ok);
-        server.dirty++;
+        redisPanic("Unknown list encoding");
     }
 }
 

From a6dd455b09cb67e18799c0b2ed720d213608e13d Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 02:15:34 +0200
Subject: [PATCH 40/66] update LRANGE to use basic iteration code to support
 dual encoding

---
 redis.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 79 insertions(+), 10 deletions(-)

diff --git a/redis.c b/redis.c
index 27e35309..699fc9ae 100644
--- a/redis.c
+++ b/redis.c
@@ -4836,6 +4836,77 @@ static unsigned long lLength(robj *subject) {
     }
 }
 
+/* Structure to hold set iteration abstraction. */
+typedef struct {
+    robj *subject;
+    unsigned char encoding;
+    unsigned char *zi;
+    listNode *ln;
+} lIterator;
+
+/* Initialize an iterator at the specified index. */
+static lIterator *lInitIterator(robj *subject, int index) {
+    lIterator *li = zmalloc(sizeof(lIterator));
+    li->subject = subject;
+    li->encoding = subject->encoding;
+    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
+        li->zi = ziplistIndex(subject->ptr,index);
+    } else if (li->encoding == REDIS_ENCODING_LIST) {
+        li->ln = listIndex(subject->ptr,index);
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+    return li;
+}
+
+/* Clean up the iterator. */
+static void lReleaseIterator(lIterator *li) {
+    zfree(li);
+}
+
+/* Return entry or NULL at the current position of the iterator. */
+static robj *lGet(lIterator *li) {
+    robj *value = NULL;
+    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
+        char *v;
+        unsigned int vlen;
+        long long vval;
+        redisAssert(li->zi != NULL);
+        if (ziplistGet(li->zi,&v,&vlen,&vval)) {
+            if (v) {
+                value = createStringObject(v,vlen);
+            } else {
+                value = createStringObjectFromLongLong(vval);
+            }
+        }
+    } else if (li->encoding == REDIS_ENCODING_LIST) {
+        redisAssert(li->ln != NULL);
+        value = listNodeValue(li->ln);
+        incrRefCount(value);
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+    return value;
+}
+
+/* Move to the next or previous entry in the list. */
+static void lMove(lIterator *li, int where) {
+    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
+        redisAssert(li->zi != NULL);
+        if (where == REDIS_HEAD)
+            li->zi = ziplistPrev(li->zi);
+        else
+            li->zi = ziplistNext(li->zi);
+    } else if (li->encoding == REDIS_ENCODING_LIST) {
+        redisAssert(li->ln != NULL);
+        if (where == REDIS_HEAD)
+            li->ln = li->ln->prev;
+        else
+            li->ln = li->ln->next;
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+}
 
 static void pushGenericCommand(redisClient *c, int where) {
     robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
@@ -4968,19 +5039,15 @@ static void rpopCommand(redisClient *c) {
 }
 
 static void lrangeCommand(redisClient *c) {
-    robj *o;
+    robj *o, *value;
     int start = atoi(c->argv[2]->ptr);
     int end = atoi(c->argv[3]->ptr);
     int llen;
     int rangelen, j;
-    list *list;
-    listNode *ln;
-    robj *ele;
 
     if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL
          || checkType(c,o,REDIS_LIST)) return;
-    list = o->ptr;
-    llen = listLength(list);
+    llen = lLength(o);
 
     /* convert negative indexes */
     if (start < 0) start = llen+start;
@@ -4998,13 +5065,15 @@ static void lrangeCommand(redisClient *c) {
     rangelen = (end-start)+1;
 
     /* Return the result in form of a multi-bulk reply */
-    ln = listIndex(list, start);
     addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",rangelen));
+    lIterator *li = lInitIterator(o,start);
     for (j = 0; j < rangelen; j++) {
-        ele = listNodeValue(ln);
-        addReplyBulk(c,ele);
-        ln = ln->next;
+        value = lGet(li);
+        redisAssert(value != NULL);
+        addReplyBulk(c,value);
+        lMove(li,REDIS_TAIL);
     }
+    lReleaseIterator(li);
 }
 
 static void ltrimCommand(redisClient *c) {

From 9ae6b0be4ac05c919b943ed29ffa20e5560855a2 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 02:16:12 +0200
Subject: [PATCH 41/66] support dual encoding in LTRIM

---
 redis.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/redis.c b/redis.c
index 699fc9ae..4b8da89e 100644
--- a/redis.c
+++ b/redis.c
@@ -5087,8 +5087,7 @@ static void ltrimCommand(redisClient *c) {
 
     if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.ok)) == NULL ||
         checkType(c,o,REDIS_LIST)) return;
-    list = o->ptr;
-    llen = listLength(list);
+    llen = lLength(o);
 
     /* convert negative indexes */
     if (start < 0) start = llen+start;
@@ -5108,15 +5107,23 @@ static void ltrimCommand(redisClient *c) {
     }
 
     /* Remove list elements to perform the trim */
-    for (j = 0; j < ltrim; j++) {
-        ln = listFirst(list);
-        listDelNode(list,ln);
+    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+        o->ptr = ziplistDeleteRange(o->ptr,0,ltrim);
+        o->ptr = ziplistDeleteRange(o->ptr,-rtrim,rtrim);
+    } else if (o->encoding == REDIS_ENCODING_LIST) {
+        list = o->ptr;
+        for (j = 0; j < ltrim; j++) {
+            ln = listFirst(list);
+            listDelNode(list,ln);
+        }
+        for (j = 0; j < rtrim; j++) {
+            ln = listLast(list);
+            listDelNode(list,ln);
+        }
+    } else {
+        redisPanic("Unknown list encoding");
     }
-    for (j = 0; j < rtrim; j++) {
-        ln = listLast(list);
-        listDelNode(list,ln);
-    }
-    if (listLength(list) == 0) deleteKey(c->db,c->argv[1]);
+    if (lLength(o) == 0) deleteKey(c->db,c->argv[1]);
     server.dirty++;
     addReply(c,shared.ok);
 }

From d2ee16abec401c7c6575cdbd666dd8a139b4926c Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 02:17:36 +0200
Subject: [PATCH 42/66] update LREM to support dual encoding via extra
 iteration primitives

---
 redis.c | 85 +++++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 68 insertions(+), 17 deletions(-)

diff --git a/redis.c b/redis.c
index 4b8da89e..064f4cad 100644
--- a/redis.c
+++ b/redis.c
@@ -4864,6 +4864,17 @@ static void lReleaseIterator(lIterator *li) {
     zfree(li);
 }
 
+/* Whether the entry pointed at is a valid entry. */
+static int lIsEntry(lIterator *li) {
+    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
+        return li->zi != NULL;
+    } else if (li->encoding == REDIS_ENCODING_LIST) {
+        return li->ln != NULL;
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+}
+
 /* Return entry or NULL at the current position of the iterator. */
 static robj *lGet(lIterator *li) {
     robj *value = NULL;
@@ -4889,6 +4900,36 @@ static robj *lGet(lIterator *li) {
     return value;
 }
 
+/* Delete the element pointed to. */
+static void lDelete(lIterator *li, int direction) {
+    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
+        direction = (direction == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
+        li->subject->ptr = ziplistDelete(li->subject->ptr,&li->zi,direction);
+    } else if (li->encoding == REDIS_ENCODING_LIST) {
+        listNode *next;
+        if (direction == REDIS_HEAD)
+            next = li->ln->prev;
+        else
+            next = li->ln->next;
+        listDelNode(li->subject->ptr,li->ln);
+        li->ln = next;
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+}
+
+/* Compare the given object with the entry at the current position. */
+static int lEqualTo(lIterator *li, robj *o) {
+    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
+        redisAssert(o->encoding == REDIS_ENCODING_RAW);
+        return ziplistCompare(li->zi,o->ptr,sdslen(o->ptr));
+    } else if (li->encoding == REDIS_ENCODING_LIST) {
+        return equalStringObjects(o,listNodeValue(li->ln));
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+}
+
 /* Move to the next or previous entry in the list. */
 static void lMove(lIterator *li, int where) {
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
@@ -5129,35 +5170,45 @@ static void ltrimCommand(redisClient *c) {
 }
 
 static void lremCommand(redisClient *c) {
-    robj *o;
-    list *list;
-    listNode *ln, *next;
+    robj *subject, *obj = c->argv[3];
     int toremove = atoi(c->argv[2]->ptr);
     int removed = 0;
-    int fromtail = 0;
+    int direction;
 
-    if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
-        checkType(c,o,REDIS_LIST)) return;
-    list = o->ptr;
+    subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
+    if (subject == NULL || checkType(c,subject,REDIS_LIST)) return;
 
+    /* Make sure obj is raw when we're dealing with a ziplist */
+    if (subject->encoding == REDIS_ENCODING_ZIPLIST)
+        obj = getDecodedObject(obj);
+
+    lIterator *li;
     if (toremove < 0) {
         toremove = -toremove;
-        fromtail = 1;
+        direction = REDIS_HEAD;
+        li = lInitIterator(subject,-1);
+    } else {
+        direction = REDIS_TAIL;
+        li = lInitIterator(subject,0);
     }
-    ln = fromtail ? list->tail : list->head;
-    while (ln) {
-        robj *ele = listNodeValue(ln);
 
-        next = fromtail ? ln->prev : ln->next;
-        if (equalStringObjects(ele,c->argv[3])) {
-            listDelNode(list,ln);
+    while (toremove && lIsEntry(li)) {
+        if (lEqualTo(li,obj)) {
+            lDelete(li,direction);
             server.dirty++;
+            toremove--;
             removed++;
-            if (toremove && removed == toremove) break;
+        } else {
+            lMove(li,direction);
         }
-        ln = next;
     }
-    if (listLength(list) == 0) deleteKey(c->db,c->argv[1]);
+    lReleaseIterator(li);
+
+    /* Clean up raw encoded object */
+    if (subject->encoding == REDIS_ENCODING_ZIPLIST)
+        decrRefCount(obj);
+
+    if (lLength(subject) == 0) deleteKey(c->db,c->argv[1]);
     addReplySds(c,sdscatprintf(sdsempty(),":%d\r\n",removed));
 }
 

From dbaa41c618346be2ab58dfbab0a35e3ff1948ed1 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 03:06:12 +0200
Subject: [PATCH 43/66] the tail offset must be an integer pointer to hold a
 32-bit offset

---
 ziplist.c | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index 71d965a8..1354b4ea 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -45,7 +45,7 @@
 
 /* Utility macros */
 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
-#define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
+#define ZIPLIST_TAIL_OFFSET(zl) (*((unsigned int*)((zl)+sizeof(unsigned int))))
 #define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
 #define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
 #define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
@@ -853,6 +853,27 @@ int main(int argc, char **argv) {
         ziplistRepr(zl);
     }
 
+    printf("Create long list and check indices:\n");
+    {
+        zl = ziplistNew();
+        char buf[32];
+        int i,len;
+        for (i = 0; i < 1000; i++) {
+            len = sprintf(buf,"%d",i);
+            zl = ziplistPush(zl,buf,len,ZIPLIST_TAIL);
+        }
+        for (i = 0; i < 1000; i++) {
+            p = ziplistIndex(zl,i);
+            assert(ziplistGet(p,NULL,NULL,&value));
+            assert(i == value);
+
+            p = ziplistIndex(zl,-i-1);
+            assert(ziplistGet(p,NULL,NULL,&value));
+            assert(999-i == value);
+        }
+        printf("SUCCESS\n\n");
+    }
+
     printf("Compare strings with ziplist entries:\n");
     {
         zl = createList();

From bd8db0ada840e2c52ab37470256676a05c6fe855 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 03:08:24 +0200
Subject: [PATCH 44/66] fixed LINDEX to always return bulk response

---
 redis.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/redis.c b/redis.c
index 064f4cad..1ae32797 100644
--- a/redis.c
+++ b/redis.c
@@ -4993,6 +4993,7 @@ static void lindexCommand(redisClient *c) {
     robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk);
     if (o == NULL || checkType(c,o,REDIS_LIST)) return;
     int index = atoi(c->argv[2]->ptr);
+    robj *value = NULL;
 
     if (o->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p;
@@ -5002,17 +5003,20 @@ static void lindexCommand(redisClient *c) {
         p = ziplistIndex(o->ptr,index);
         if (ziplistGet(p,&v,&vlen,&vval)) {
             if (v) {
-                addReplySds(c,sdsnewlen(v,vlen));
+                value = createStringObject(v,vlen);
             } else {
-                addReplyLongLong(c,vval);
+                value = createStringObjectFromLongLong(vval);
             }
+            addReplyBulk(c,value);
+            decrRefCount(value);
         } else {
             addReply(c,shared.nullbulk);
         }
     } else if (o->encoding == REDIS_ENCODING_LIST) {
         listNode *ln = listIndex(o->ptr,index);
         if (ln != NULL) {
-            addReply(c,(robj*)listNodeValue(ln));
+            value = listNodeValue(ln);
+            addReplyBulk(c,value);
         } else {
             addReply(c,shared.nullbulk);
         }

From 3fbf9001ce88f2b3d154b099bcf2fd3783f24940 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Sun, 30 May 2010 03:14:38 +0200
Subject: [PATCH 45/66] fix LREM to remove *all* occurances when a zero
 argument is given

---
 redis.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/redis.c b/redis.c
index 1ae32797..2a488283 100644
--- a/redis.c
+++ b/redis.c
@@ -5196,12 +5196,12 @@ static void lremCommand(redisClient *c) {
         li = lInitIterator(subject,0);
     }
 
-    while (toremove && lIsEntry(li)) {
+    while (lIsEntry(li)) {
         if (lEqualTo(li,obj)) {
             lDelete(li,direction);
             server.dirty++;
-            toremove--;
             removed++;
+            if (toremove && removed == toremove) break;
         } else {
             lMove(li,direction);
         }

From 8632fb30401ccc6824b84499602fba7052cc3346 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 20:17:21 +0200
Subject: [PATCH 46/66] ziplistPrev should return the tail when the argument is
 ZIP_END

---
 ziplist.c | 32 +++++++++++++++++++++++---------
 ziplist.h |  4 ++--
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 1354b4ea..2aeffa41 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -428,14 +428,27 @@ unsigned char *ziplistIndex(unsigned char *zl, int index) {
 }
 
 /* Return pointer to next entry in ziplist. */
-unsigned char *ziplistNext(unsigned char *p) {
+unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
+    ((void) zl);
     return (p[0] == ZIP_END) ? NULL : p+zipRawEntryLength(p);
 }
 
 /* Return pointer to previous entry in ziplist. */
-unsigned char *ziplistPrev(unsigned char *p) {
-    zlentry entry = zipEntry(p);
-    return (entry.prevrawlen == 0) ? NULL : p-entry.prevrawlen;
+unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
+    zlentry entry;
+
+    /* Iterating backwards from ZIP_END should return the tail. When "p" is
+     * equal to the first element of the list, we're already at the head,
+     * and should return NULL. */
+    if (p[0] == ZIP_END) {
+        p = ZIPLIST_ENTRY_TAIL(zl);
+        return (p[0] == ZIP_END) ? NULL : p;
+    } else if (p == ZIPLIST_ENTRY_HEAD(zl)) {
+        return NULL;
+    } else {
+        entry = zipEntry(p);
+        return p-entry.prevrawlen;
+    }
 }
 
 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
@@ -709,7 +722,7 @@ int main(int argc, char **argv) {
             } else {
                 printf("%lld", value);
             }
-            p = ziplistNext(p);
+            p = ziplistNext(zl,p);
             printf("\n");
         }
         printf("\n");
@@ -726,7 +739,7 @@ int main(int argc, char **argv) {
             } else {
                 printf("%lld", value);
             }
-            p = ziplistNext(p);
+            p = ziplistNext(zl,p);
             printf("\n");
         }
         printf("\n");
@@ -743,7 +756,7 @@ int main(int argc, char **argv) {
             } else {
                 printf("%lld", value);
             }
-            p = ziplistNext(p);
+            p = ziplistNext(zl,p);
             printf("\n");
         }
         printf("\n");
@@ -772,7 +785,7 @@ int main(int argc, char **argv) {
             } else {
                 printf("%lld", value);
             }
-            p = ziplistPrev(p);
+            p = ziplistPrev(zl,p);
             printf("\n");
         }
         printf("\n");
@@ -789,7 +802,8 @@ int main(int argc, char **argv) {
             } else {
                 printf("%lld", value);
             }
-            zl = ziplistDelete(zl, &p, ZIPLIST_HEAD);
+            zl = ziplistDelete(zl,&p);
+            p = ziplistPrev(zl,p);
             printf("\n");
         }
         printf("\n");
diff --git a/ziplist.h b/ziplist.h
index 08035dd5..15153fea 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -5,8 +5,8 @@ unsigned char *ziplistNew(void);
 unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int where);
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where);
 unsigned char *ziplistIndex(unsigned char *zl, int index);
-unsigned char *ziplistNext(unsigned char *p);
-unsigned char *ziplistPrev(unsigned char *p);
+unsigned char *ziplistNext(unsigned char *zl, unsigned char *p);
+unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p);
 unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
 unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p, int direction);

From 6a8e35ad9252f1ea1ff0a449af52aef1f71815f8 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 20:18:23 +0200
Subject: [PATCH 47/66] ziplistDelete no longer needs a direction now
 ziplistPrev is fixed

---
 ziplist.c | 8 ++------
 ziplist.h | 2 +-
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 2aeffa41..dd94f14e 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -482,7 +482,7 @@ unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsig
 /* Delete a single entry from the ziplist, pointed to by *p.
  * Also update *p in place, to be able to iterate over the
  * ziplist, while deleting entries. */
-unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p, int direction) {
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
     unsigned int offset = *p-zl;
     zl = __ziplistDelete(zl,*p,1);
 
@@ -490,11 +490,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p, int direction
      * do a realloc which might result in a different "zl"-pointer.
      * When the delete direction is back to front, we might delete the last
      * entry and end up with "p" pointing to ZIP_END, so check this. */
-    if (*(zl+offset) == ZIP_END && direction == ZIPLIST_HEAD) {
-        *p = ZIPLIST_ENTRY_TAIL(zl);
-    } else {
-        *p = zl+offset;
-    }
+    *p = zl+offset;
     return zl;
 }
 
diff --git a/ziplist.h b/ziplist.h
index 15153fea..e8522182 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -9,7 +9,7 @@ unsigned char *ziplistNext(unsigned char *zl, unsigned char *p);
 unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p);
 unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
 unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen);
-unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p, int direction);
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
 unsigned int ziplistCompare(unsigned char *p, char *entry, unsigned int elen);
 unsigned int ziplistLen(unsigned char *zl);

From be02a7c0d653021336ae23092046d442ac017f11 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 20:25:31 +0200
Subject: [PATCH 48/66] update list iteration semantic to work as expected
 (i.e. "while(lNext(..))")

---
 redis.c | 130 +++++++++++++++++++++++++++++++-------------------------
 1 file changed, 72 insertions(+), 58 deletions(-)

diff --git a/redis.c b/redis.c
index 2a488283..d230a195 100644
--- a/redis.c
+++ b/redis.c
@@ -4806,7 +4806,7 @@ static robj *lPop(robj *subject, int where) {
                 value = createStringObjectFromLongLong(vval);
             }
         }
-        subject->ptr = ziplistDelete(subject->ptr,&p,ZIPLIST_TAIL);
+        subject->ptr = ziplistDelete(subject->ptr,&p);
     } else if (subject->encoding == REDIS_ENCODING_LIST) {
         list *list = subject->ptr;
         listNode *ln;
@@ -4840,15 +4840,24 @@ static unsigned long lLength(robj *subject) {
 typedef struct {
     robj *subject;
     unsigned char encoding;
+    unsigned char direction; /* Iteration direction */
     unsigned char *zi;
     listNode *ln;
 } lIterator;
 
+/* Structure for an entry while iterating over a list. */
+typedef struct {
+    lIterator *li;
+    unsigned char *zi;  /* Entry in ziplist */
+    listNode *ln;       /* Entry in linked list */
+} lEntry;
+
 /* Initialize an iterator at the specified index. */
-static lIterator *lInitIterator(robj *subject, int index) {
+static lIterator *lInitIterator(robj *subject, int index, unsigned char direction) {
     lIterator *li = zmalloc(sizeof(lIterator));
     li->subject = subject;
     li->encoding = subject->encoding;
+    li->direction = direction;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         li->zi = ziplistIndex(subject->ptr,index);
     } else if (li->encoding == REDIS_ENCODING_LIST) {
@@ -4864,26 +4873,45 @@ static void lReleaseIterator(lIterator *li) {
     zfree(li);
 }
 
-/* Whether the entry pointed at is a valid entry. */
-static int lIsEntry(lIterator *li) {
+/* Stores pointer to current the entry in the provided entry structure
+ * and advances the position of the iterator. Returns 1 when the current
+ * entry is in fact an entry, 0 otherwise. */
+static int lNext(lIterator *li, lEntry *entry) {
+    entry->li = li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
-        return li->zi != NULL;
+        entry->zi = li->zi;
+        if (entry->zi != NULL) {
+            if (li->direction == REDIS_TAIL)
+                li->zi = ziplistNext(li->subject->ptr,li->zi);
+            else
+                li->zi = ziplistPrev(li->subject->ptr,li->zi);
+            return 1;
+        }
     } else if (li->encoding == REDIS_ENCODING_LIST) {
-        return li->ln != NULL;
+        entry->ln = li->ln;
+        if (entry->ln != NULL) {
+            if (li->direction == REDIS_TAIL)
+                li->ln = li->ln->next;
+            else
+                li->ln = li->ln->prev;
+            return 1;
+        }
     } else {
         redisPanic("Unknown list encoding");
     }
+    return 0;
 }
 
 /* Return entry or NULL at the current position of the iterator. */
-static robj *lGet(lIterator *li) {
+static robj *lGet(lEntry *entry) {
+    lIterator *li = entry->li;
     robj *value = NULL;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         char *v;
         unsigned int vlen;
         long long vval;
-        redisAssert(li->zi != NULL);
-        if (ziplistGet(li->zi,&v,&vlen,&vval)) {
+        redisAssert(entry->zi != NULL);
+        if (ziplistGet(entry->zi,&v,&vlen,&vval)) {
             if (v) {
                 value = createStringObject(v,vlen);
             } else {
@@ -4891,8 +4919,8 @@ static robj *lGet(lIterator *li) {
             }
         }
     } else if (li->encoding == REDIS_ENCODING_LIST) {
-        redisAssert(li->ln != NULL);
-        value = listNodeValue(li->ln);
+        redisAssert(entry->ln != NULL);
+        value = listNodeValue(entry->ln);
         incrRefCount(value);
     } else {
         redisPanic("Unknown list encoding");
@@ -4900,50 +4928,39 @@ static robj *lGet(lIterator *li) {
     return value;
 }
 
-/* Delete the element pointed to. */
-static void lDelete(lIterator *li, int direction) {
-    if (li->encoding == REDIS_ENCODING_ZIPLIST) {
-        direction = (direction == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
-        li->subject->ptr = ziplistDelete(li->subject->ptr,&li->zi,direction);
-    } else if (li->encoding == REDIS_ENCODING_LIST) {
-        listNode *next;
-        if (direction == REDIS_HEAD)
-            next = li->ln->prev;
-        else
-            next = li->ln->next;
-        listDelNode(li->subject->ptr,li->ln);
-        li->ln = next;
-    } else {
-        redisPanic("Unknown list encoding");
-    }
-}
-
 /* Compare the given object with the entry at the current position. */
-static int lEqualTo(lIterator *li, robj *o) {
+static int lEqual(lEntry *entry, robj *o) {
+    lIterator *li = entry->li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         redisAssert(o->encoding == REDIS_ENCODING_RAW);
-        return ziplistCompare(li->zi,o->ptr,sdslen(o->ptr));
+        return ziplistCompare(entry->zi,o->ptr,sdslen(o->ptr));
     } else if (li->encoding == REDIS_ENCODING_LIST) {
-        return equalStringObjects(o,listNodeValue(li->ln));
+        return equalStringObjects(o,listNodeValue(entry->ln));
     } else {
         redisPanic("Unknown list encoding");
     }
 }
 
-/* Move to the next or previous entry in the list. */
-static void lMove(lIterator *li, int where) {
+/* Delete the element pointed to. */
+static void lDelete(lEntry *entry) {
+    lIterator *li = entry->li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
-        redisAssert(li->zi != NULL);
-        if (where == REDIS_HEAD)
-            li->zi = ziplistPrev(li->zi);
+        unsigned char *p = entry->zi;
+        li->subject->ptr = ziplistDelete(li->subject->ptr,&p);
+
+        /* Update position of the iterator depending on the direction */
+        if (li->direction == REDIS_TAIL)
+            li->zi = p;
         else
-            li->zi = ziplistNext(li->zi);
-    } else if (li->encoding == REDIS_ENCODING_LIST) {
-        redisAssert(li->ln != NULL);
-        if (where == REDIS_HEAD)
-            li->ln = li->ln->prev;
+            li->zi = ziplistPrev(li->subject->ptr,p);
+    } else if (entry->li->encoding == REDIS_ENCODING_LIST) {
+        listNode *next;
+        if (li->direction == REDIS_TAIL)
+            next = entry->ln->next;
         else
-            li->ln = li->ln->next;
+            next = entry->ln->prev;
+        listDelNode(li->subject->ptr,entry->ln);
+        li->ln = next;
     } else {
         redisPanic("Unknown list encoding");
     }
@@ -5037,7 +5054,7 @@ static void lsetCommand(redisClient *c) {
         if (p == NULL) {
             addReply(c,shared.outofrangeerr);
         } else {
-            o->ptr = ziplistDelete(o->ptr,&p,ZIPLIST_TAIL);
+            o->ptr = ziplistDelete(o->ptr,&p);
             value = getDecodedObject(value);
             o->ptr = ziplistInsert(o->ptr,p,value->ptr,sdslen(value->ptr));
             decrRefCount(value);
@@ -5089,6 +5106,7 @@ static void lrangeCommand(redisClient *c) {
     int end = atoi(c->argv[3]->ptr);
     int llen;
     int rangelen, j;
+    lEntry entry;
 
     if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL
          || checkType(c,o,REDIS_LIST)) return;
@@ -5111,12 +5129,12 @@ static void lrangeCommand(redisClient *c) {
 
     /* Return the result in form of a multi-bulk reply */
     addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",rangelen));
-    lIterator *li = lInitIterator(o,start);
+    lIterator *li = lInitIterator(o,start,REDIS_TAIL);
     for (j = 0; j < rangelen; j++) {
-        value = lGet(li);
-        redisAssert(value != NULL);
+        redisAssert(lNext(li,&entry));
+        value = lGet(&entry);
         addReplyBulk(c,value);
-        lMove(li,REDIS_TAIL);
+        decrRefCount(value);
     }
     lReleaseIterator(li);
 }
@@ -5177,7 +5195,7 @@ static void lremCommand(redisClient *c) {
     robj *subject, *obj = c->argv[3];
     int toremove = atoi(c->argv[2]->ptr);
     int removed = 0;
-    int direction;
+    lEntry entry;
 
     subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
     if (subject == NULL || checkType(c,subject,REDIS_LIST)) return;
@@ -5189,21 +5207,17 @@ static void lremCommand(redisClient *c) {
     lIterator *li;
     if (toremove < 0) {
         toremove = -toremove;
-        direction = REDIS_HEAD;
-        li = lInitIterator(subject,-1);
+        li = lInitIterator(subject,-1,REDIS_HEAD);
     } else {
-        direction = REDIS_TAIL;
-        li = lInitIterator(subject,0);
+        li = lInitIterator(subject,0,REDIS_TAIL);
     }
 
-    while (lIsEntry(li)) {
-        if (lEqualTo(li,obj)) {
-            lDelete(li,direction);
+    while (lNext(li,&entry)) {
+        if (lEqual(&entry,obj)) {
+            lDelete(&entry);
             server.dirty++;
             removed++;
             if (toremove && removed == toremove) break;
-        } else {
-            lMove(li,direction);
         }
     }
     lReleaseIterator(li);

From 0f62e1775d37d34a4a465abfc81becaf02ecae64 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 20:47:39 +0200
Subject: [PATCH 49/66] update RPOPLPUSH to support dual encoding

---
 redis.c | 44 ++++++++++++++++++--------------------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/redis.c b/redis.c
index d230a195..dc92d76f 100644
--- a/redis.c
+++ b/redis.c
@@ -4805,8 +4805,9 @@ static robj *lPop(robj *subject, int where) {
             } else {
                 value = createStringObjectFromLongLong(vval);
             }
+            /* We only need to delete an element when it exists */
+            subject->ptr = ziplistDelete(subject->ptr,&p);
         }
-        subject->ptr = ziplistDelete(subject->ptr,&p);
     } else if (subject->encoding == REDIS_ENCODING_LIST) {
         list *list = subject->ptr;
         listNode *ln;
@@ -5246,47 +5247,38 @@ static void lremCommand(redisClient *c) {
  * as well. This command was originally proposed by Ezra Zygmuntowicz.
  */
 static void rpoplpushcommand(redisClient *c) {
-    robj *sobj;
-    list *srclist;
-    listNode *ln;
-
+    robj *sobj, *value;
     if ((sobj = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
         checkType(c,sobj,REDIS_LIST)) return;
-    srclist = sobj->ptr;
-    ln = listLast(srclist);
 
-    if (ln == NULL) {
+    if (lLength(sobj) == 0) {
         addReply(c,shared.nullbulk);
     } else {
         robj *dobj = lookupKeyWrite(c->db,c->argv[2]);
-        robj *ele = listNodeValue(ln);
-        list *dstlist;
-
-        if (dobj && dobj->type != REDIS_LIST) {
-            addReply(c,shared.wrongtypeerr);
-            return;
-        }
+        if (dobj && checkType(c,dobj,REDIS_LIST)) return;
+        value = lPop(sobj,REDIS_TAIL);
 
         /* Add the element to the target list (unless it's directly
          * passed to some BLPOP-ing client */
-        if (!handleClientsWaitingListPush(c,c->argv[2],ele)) {
-            if (dobj == NULL) {
-                /* Create the list if the key does not exist */
-                dobj = createListObject();
+        if (!handleClientsWaitingListPush(c,c->argv[2],value)) {
+            /* Create the list if the key does not exist */
+            if (!dobj) {
+                dobj = createObject(REDIS_LIST,ziplistNew());
+                dobj->encoding = REDIS_ENCODING_ZIPLIST;
                 dictAdd(c->db->dict,c->argv[2],dobj);
                 incrRefCount(c->argv[2]);
             }
-            dstlist = dobj->ptr;
-            listAddNodeHead(dstlist,ele);
-            incrRefCount(ele);
+            lPush(dobj,value,REDIS_HEAD);
         }
 
         /* Send the element to the client as reply as well */
-        addReplyBulk(c,ele);
+        addReplyBulk(c,value);
 
-        /* Finally remove the element from the source list */
-        listDelNode(srclist,ln);
-        if (listLength(srclist) == 0) deleteKey(c->db,c->argv[1]);
+        /* lPop returns an object with its refcount incremented */
+        decrRefCount(value);
+
+        /* Delete the source list when it is empty */
+        if (lLength(sobj) == 0) deleteKey(c->db,c->argv[1]);
         server.dirty++;
     }
 }

From 2796f6da7b7208382edb618b3ab0667f7e2c4f27 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 21:08:07 +0200
Subject: [PATCH 50/66] added rdb save function to directly save long long
 values

---
 redis.c | 44 +++++++++++++++++++-------------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/redis.c b/redis.c
index dc92d76f..be2561b1 100644
--- a/redis.c
+++ b/redis.c
@@ -3544,38 +3544,32 @@ static int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) {
     return 0;
 }
 
+/* Save a long long value as either an encoded string or a string. */
+static int rdbSaveLongLongAsStringObject(FILE *fp, long long value) {
+    unsigned char buf[32];
+    int enclen = rdbEncodeInteger(value,buf);
+    if (enclen > 0) {
+        if (fwrite(buf,enclen,1,fp) == 0) return -1;
+    } else {
+        /* Encode as string */
+        enclen = ll2string((char*)buf,32,value);
+        redisAssert(enclen < 32);
+        if (rdbSaveLen(fp,enclen) == -1) return -1;
+        if (fwrite(buf,enclen,1,fp) == 0) return -1;
+    }
+    return 0;
+}
+
 /* Like rdbSaveStringObjectRaw() but handle encoded objects */
 static int rdbSaveStringObject(FILE *fp, robj *obj) {
-    int retval;
-
     /* Avoid to decode the object, then encode it again, if the
      * object is alrady integer encoded. */
     if (obj->encoding == REDIS_ENCODING_INT) {
-        long val = (long) obj->ptr;
-        unsigned char buf[5];
-        int enclen;
-
-        if ((enclen = rdbEncodeInteger(val,buf)) > 0) {
-            if (fwrite(buf,enclen,1,fp) == 0) return -1;
-            return 0;
-        }
-        /* otherwise... fall throught and continue with the usual
-         * code path. */
-    }
-
-    /* Avoid incr/decr ref count business when possible.
-     * This plays well with copy-on-write given that we are probably
-     * in a child process (BGSAVE). Also this makes sure key objects
-     * of swapped objects are not incRefCount-ed (an assert does not allow
-     * this in order to avoid bugs) */
-    if (obj->encoding != REDIS_ENCODING_RAW) {
-        obj = getDecodedObject(obj);
-        retval = rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr));
-        decrRefCount(obj);
+        return rdbSaveLongLongAsStringObject(fp,(long)obj->ptr);
     } else {
-        retval = rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr));
+        redisAssert(obj->encoding == REDIS_ENCODING_RAW);
+        return rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr));
     }
-    return retval;
 }
 
 /* Save a double value. Doubles are saved as strings prefixed by an unsigned

From b6eb9703949b49d2e5bd71316b4cd545677c5651 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 21:44:12 +0200
Subject: [PATCH 51/66] fixed signedness and disambiguate variable names

---
 redis.c   | 36 ++++++++++++++++++------------------
 ziplist.c | 34 +++++++++++++++++-----------------
 ziplist.h |  8 ++++----
 3 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/redis.c b/redis.c
index be2561b1..0e188f60 100644
--- a/redis.c
+++ b/redis.c
@@ -4788,16 +4788,16 @@ static robj *lPop(robj *subject, int where) {
     robj *value = NULL;
     if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p;
-        char *v;
+        unsigned char *vstr;
         unsigned int vlen;
-        long long vval;
+        long long vlong;
         int pos = (where == REDIS_HEAD) ? 0 : -1;
         p = ziplistIndex(subject->ptr,pos);
-        if (ziplistGet(p,&v,&vlen,&vval)) {
-            if (v) {
-                value = createStringObject(v,vlen);
+        if (ziplistGet(p,&vstr,&vlen,&vlong)) {
+            if (vstr) {
+                value = createStringObject((char*)vstr,vlen);
             } else {
-                value = createStringObjectFromLongLong(vval);
+                value = createStringObjectFromLongLong(vlong);
             }
             /* We only need to delete an element when it exists */
             subject->ptr = ziplistDelete(subject->ptr,&p);
@@ -4902,15 +4902,15 @@ static robj *lGet(lEntry *entry) {
     lIterator *li = entry->li;
     robj *value = NULL;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
-        char *v;
+        unsigned char *vstr;
         unsigned int vlen;
-        long long vval;
+        long long vlong;
         redisAssert(entry->zi != NULL);
-        if (ziplistGet(entry->zi,&v,&vlen,&vval)) {
-            if (v) {
-                value = createStringObject(v,vlen);
+        if (ziplistGet(entry->zi,&vstr,&vlen,&vlong)) {
+            if (vstr) {
+                value = createStringObject((char*)vstr,vlen);
             } else {
-                value = createStringObjectFromLongLong(vval);
+                value = createStringObjectFromLongLong(vlong);
             }
         }
     } else if (li->encoding == REDIS_ENCODING_LIST) {
@@ -5009,15 +5009,15 @@ static void lindexCommand(redisClient *c) {
 
     if (o->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p;
-        char *v;
+        unsigned char *vstr;
         unsigned int vlen;
-        long long vval;
+        long long vlong;
         p = ziplistIndex(o->ptr,index);
-        if (ziplistGet(p,&v,&vlen,&vval)) {
-            if (v) {
-                value = createStringObject(v,vlen);
+        if (ziplistGet(p,&vstr,&vlen,&vlong)) {
+            if (vstr) {
+                value = createStringObject((char*)vstr,vlen);
             } else {
-                value = createStringObjectFromLongLong(vval);
+                value = createStringObjectFromLongLong(vlong);
             }
             addReplyBulk(c,value);
             decrRefCount(value);
diff --git a/ziplist.c b/ziplist.c
index dd94f14e..8b96d6d7 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -63,7 +63,7 @@ typedef struct zlentry {
 } zlentry;
 
 /* Return bytes needed to store integer encoded by 'encoding' */
-static unsigned int zipEncodingSize(char encoding) {
+static unsigned int zipEncodingSize(unsigned char encoding) {
     if (encoding == ZIP_ENC_SHORT) {
         return sizeof(short int);
     } else if (encoding == ZIP_ENC_INT) {
@@ -173,12 +173,12 @@ static int zipPrevLenByteDiff(unsigned char *p, unsigned int len) {
 /* Check if string pointed to by 'entry' can be encoded as an integer.
  * Stores the integer value in 'v' and its encoding in 'encoding'.
  * Warning: this function requires a NULL-terminated string! */
-static int zipTryEncoding(char *entry, long long *v, char *encoding) {
+static int zipTryEncoding(unsigned char *entry, long long *v, unsigned char *encoding) {
     long long value;
     char *eptr;
 
     if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
-        value = strtoll(entry,&eptr,10);
+        value = strtoll((char*)entry,&eptr,10);
         if (eptr[0] != '\0') return 0;
         if (value >= SHRT_MIN && value <= SHRT_MAX) {
             *encoding = ZIP_ENC_SHORT;
@@ -194,7 +194,7 @@ static int zipTryEncoding(char *entry, long long *v, char *encoding) {
 }
 
 /* Store integer 'value' at 'p', encoded as 'encoding' */
-static void zipSaveInteger(unsigned char *p, long long value, char encoding) {
+static void zipSaveInteger(unsigned char *p, long long value, unsigned char encoding) {
     short int s;
     int i;
     long long l;
@@ -213,7 +213,7 @@ static void zipSaveInteger(unsigned char *p, long long value, char encoding) {
 }
 
 /* Read integer encoded as 'encoding' from 'p' */
-static long long zipLoadInteger(unsigned char *p, char encoding) {
+static long long zipLoadInteger(unsigned char *p, unsigned char encoding) {
     short int s;
     int i;
     long long l, ret;
@@ -269,7 +269,7 @@ static unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
 }
 
 /* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
-static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int num) {
+static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsigned int num) {
     unsigned int i, totlen, deleted = 0;
     int nextdiff = 0;
     zlentry first = zipEntry(p);
@@ -306,11 +306,11 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, int n
 }
 
 /* Insert item at "p". */
-static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen) {
+static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
     unsigned int curlen = ZIPLIST_BYTES(zl), reqlen, prevlen = 0;
     unsigned int offset, nextdiff = 0;
     unsigned char *tail;
-    char encoding = ZIP_ENC_RAW;
+    unsigned char encoding = ZIP_ENC_RAW;
     long long value;
     zlentry entry;
 
@@ -372,7 +372,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, char
     return zl;
 }
 
-unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int where) {
+unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
     unsigned char *p;
     p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);
     return __ziplistInsert(zl,p,s,slen);
@@ -455,7 +455,7 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) {
  * on the encoding of the entry. 'e' is always set to NULL to be able
  * to find out whether the string pointer or the integer value was set.
  * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
-unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval) {
+unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) {
     zlentry entry;
     if (p == NULL || p[0] == ZIP_END) return 0;
     if (sstr) *sstr = NULL;
@@ -464,7 +464,7 @@ unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long
     if (entry.encoding == ZIP_ENC_RAW) {
         if (sstr) {
             *slen = entry.len;
-            *sstr = (char*)p+entry.headersize;
+            *sstr = p+entry.headersize;
         }
     } else {
         if (sval) {
@@ -475,7 +475,7 @@ unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long
 }
 
 /* Insert an entry at "p". */
-unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen) {
+unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {
     return __ziplistInsert(zl,p,s,slen);
 }
 
@@ -501,10 +501,10 @@ unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigne
 }
 
 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
-unsigned int ziplistCompare(unsigned char *p, char *sstr, unsigned int slen) {
+unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int slen) {
     zlentry entry;
-    char sencoding;
-    long long val, sval;
+    unsigned char sencoding;
+    long long zval, sval;
     if (p[0] == ZIP_END) return 0;
 
     entry = zipEntry(p);
@@ -519,8 +519,8 @@ unsigned int ziplistCompare(unsigned char *p, char *sstr, unsigned int slen) {
         /* Try to compare encoded values */
         if (zipTryEncoding(sstr,&sval,&sencoding)) {
             if (entry.encoding == sencoding) {
-                val = zipLoadInteger(p+entry.headersize,entry.encoding);
-                return val == sval;
+                zval = zipLoadInteger(p+entry.headersize,entry.encoding);
+                return zval == sval;
             }
         }
     }
diff --git a/ziplist.h b/ziplist.h
index e8522182..6d9037dc 100644
--- a/ziplist.h
+++ b/ziplist.h
@@ -2,15 +2,15 @@
 #define ZIPLIST_TAIL 1
 
 unsigned char *ziplistNew(void);
-unsigned char *ziplistPush(unsigned char *zl, char *s, unsigned int slen, int where);
+unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where);
 unsigned char *ziplistPop(unsigned char *zl, sds *target, int where);
 unsigned char *ziplistIndex(unsigned char *zl, int index);
 unsigned char *ziplistNext(unsigned char *zl, unsigned char *p);
 unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p);
-unsigned int ziplistGet(unsigned char *p, char **sstr, unsigned int *slen, long long *sval);
-unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, char *s, unsigned int slen);
+unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval);
+unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
 unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
 unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
-unsigned int ziplistCompare(unsigned char *p, char *entry, unsigned int elen);
+unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen);
 unsigned int ziplistLen(unsigned char *zl);
 unsigned int ziplistSize(unsigned char *zl);

From 23f964946b815c8d6317071b4a5597151e4cefd2 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 22:25:22 +0200
Subject: [PATCH 52/66] support rdb saving/loading with dual list encoding

---
 redis.c | 84 ++++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 60 insertions(+), 24 deletions(-)

diff --git a/redis.c b/redis.c
index 0e188f60..3abfa44c 100644
--- a/redis.c
+++ b/redis.c
@@ -3622,16 +3622,37 @@ static int rdbSaveObject(FILE *fp, robj *o) {
         if (rdbSaveStringObject(fp,o) == -1) return -1;
     } else if (o->type == REDIS_LIST) {
         /* Save a list value */
-        list *list = o->ptr;
-        listIter li;
-        listNode *ln;
+        if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+            unsigned char *p;
+            unsigned char *vstr;
+            unsigned int vlen;
+            long long vlong;
 
-        if (rdbSaveLen(fp,listLength(list)) == -1) return -1;
-        listRewind(list,&li);
-        while((ln = listNext(&li))) {
-            robj *eleobj = listNodeValue(ln);
+            if (rdbSaveLen(fp,ziplistLen(o->ptr)) == -1) return -1;
+            p = ziplistIndex(o->ptr,0);
+            while(ziplistGet(p,&vstr,&vlen,&vlong)) {
+                if (vstr) {
+                    if (rdbSaveRawString(fp,vstr,vlen) == -1)
+                        return -1;
+                } else {
+                    if (rdbSaveLongLongAsStringObject(fp,vlong) == -1)
+                        return -1;
+                }
+                p = ziplistNext(o->ptr,p);
+            }
+        } else if (o->encoding == REDIS_ENCODING_LIST) {
+            list *list = o->ptr;
+            listIter li;
+            listNode *ln;
 
-            if (rdbSaveStringObject(fp,eleobj) == -1) return -1;
+            if (rdbSaveLen(fp,listLength(list)) == -1) return -1;
+            listRewind(list,&li);
+            while((ln = listNext(&li))) {
+                robj *eleobj = listNodeValue(ln);
+                if (rdbSaveStringObject(fp,eleobj) == -1) return -1;
+            }
+        } else {
+            redisPanic("Unknown list encoding");
         }
     } else if (o->type == REDIS_SET) {
         /* Save a set value */
@@ -3998,34 +4019,49 @@ static int rdbLoadDoubleValue(FILE *fp, double *val) {
 /* Load a Redis object of the specified type from the specified file.
  * On success a newly allocated object is returned, otherwise NULL. */
 static robj *rdbLoadObject(int type, FILE *fp) {
-    robj *o;
+    robj *o, *ele, *dec;
+    size_t len;
 
     redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp));
     if (type == REDIS_STRING) {
         /* Read string value */
         if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
         o = tryObjectEncoding(o);
-    } else if (type == REDIS_LIST || type == REDIS_SET) {
-        /* Read list/set value */
-        uint32_t listlen;
+    } else if (type == REDIS_LIST) {
+        /* Read list value */
+        if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
 
-        if ((listlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
-        o = (type == REDIS_LIST) ? createListObject() : createSetObject();
+        o = createObject(REDIS_LIST,ziplistNew());
+        o->encoding = REDIS_ENCODING_ZIPLIST;
+
+        /* Load every single element of the list */
+        while(len--) {
+            if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
+
+            if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+                dec = getDecodedObject(ele);
+                o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL);
+                decrRefCount(dec);
+                decrRefCount(ele);
+            } else {
+                ele = tryObjectEncoding(ele);
+                listAddNodeTail(o->ptr,ele);
+                incrRefCount(ele);
+            }
+        }
+    } else if (type == REDIS_SET) {
+        /* Read list/set value */
+        if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
+        o = createSetObject();
         /* It's faster to expand the dict to the right size asap in order
          * to avoid rehashing */
-        if (type == REDIS_SET && listlen > DICT_HT_INITIAL_SIZE)
-            dictExpand(o->ptr,listlen);
+        if (len > DICT_HT_INITIAL_SIZE)
+            dictExpand(o->ptr,len);
         /* Load every single element of the list/set */
-        while(listlen--) {
-            robj *ele;
-
+        while(len--) {
             if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
             ele = tryObjectEncoding(ele);
-            if (type == REDIS_LIST) {
-                listAddNodeTail((list*)o->ptr,ele);
-            } else {
-                dictAdd((dict*)o->ptr,ele,NULL);
-            }
+            dictAdd((dict*)o->ptr,ele,NULL);
         }
     } else if (type == REDIS_ZSET) {
         /* Read list/set value */

From 1cd92e7f040702e07e4930e8faa3f0f692658cdc Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 23:10:05 +0200
Subject: [PATCH 53/66] function to create a new ziplist encoded list

---
 redis.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/redis.c b/redis.c
index 3abfa44c..33750b93 100644
--- a/redis.c
+++ b/redis.c
@@ -2984,9 +2984,17 @@ static robj *dupStringObject(robj *o) {
 
 static robj *createListObject(void) {
     list *l = listCreate();
-
+    robj *o = createObject(REDIS_LIST,l);
     listSetFreeMethod(l,decrRefCount);
-    return createObject(REDIS_LIST,l);
+    o->encoding = REDIS_ENCODING_LIST;
+    return o;
+}
+
+static robj *createZiplistObject(void) {
+    unsigned char *zl = ziplistNew();
+    robj *o = createObject(REDIS_LIST,zl);
+    o->encoding = REDIS_ENCODING_ZIPLIST;
+    return o;
 }
 
 static robj *createSetObject(void) {
@@ -4031,8 +4039,7 @@ static robj *rdbLoadObject(int type, FILE *fp) {
         /* Read list value */
         if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
 
-        o = createObject(REDIS_LIST,ziplistNew());
-        o->encoding = REDIS_ENCODING_ZIPLIST;
+        o = createZiplistObject();
 
         /* Load every single element of the list */
         while(len--) {
@@ -5004,8 +5011,7 @@ static void pushGenericCommand(redisClient *c, int where) {
             addReply(c,shared.cone);
             return;
         }
-        lobj = createObject(REDIS_LIST,ziplistNew());
-        lobj->encoding = REDIS_ENCODING_ZIPLIST;
+        lobj = createZiplistObject();
         dictAdd(c->db->dict,c->argv[1],lobj);
         incrRefCount(c->argv[1]);
     } else {
@@ -5293,8 +5299,7 @@ static void rpoplpushcommand(redisClient *c) {
         if (!handleClientsWaitingListPush(c,c->argv[2],value)) {
             /* Create the list if the key does not exist */
             if (!dobj) {
-                dobj = createObject(REDIS_LIST,ziplistNew());
-                dobj->encoding = REDIS_ENCODING_ZIPLIST;
+                dobj = createZiplistObject();
                 dictAdd(c->db->dict,c->argv[2],dobj);
                 incrRefCount(c->argv[2]);
             }

From a03611e1337a207bb771c2c72256047cbb2a4092 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 23:11:28 +0200
Subject: [PATCH 54/66] update SORT to work with the dual list encoding

---
 redis.c | 65 ++++++++++++++++++++++++++++++---------------------------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/redis.c b/redis.c
index 33750b93..07a53748 100644
--- a/redis.c
+++ b/redis.c
@@ -7210,7 +7210,7 @@ static int sortCompare(const void *s1, const void *s2) {
  * is optimized for speed and a bit less for readability */
 static void sortCommand(redisClient *c) {
     list *operations;
-    int outputlen = 0;
+    unsigned int outputlen = 0;
     int desc = 0, alpha = 0;
     int limit_start = 0, limit_count = -1, start, end;
     int j, dontsort = 0, vectorlen;
@@ -7280,7 +7280,7 @@ static void sortCommand(redisClient *c) {
 
     /* Load the sorting vector with all the objects to sort */
     switch(sortval->type) {
-    case REDIS_LIST: vectorlen = listLength((list*)sortval->ptr); break;
+    case REDIS_LIST: vectorlen = lLength(sortval); break;
     case REDIS_SET: vectorlen =  dictSize((dict*)sortval->ptr); break;
     case REDIS_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->dict); break;
     default: vectorlen = 0; redisPanic("Bad SORT type"); /* Avoid GCC warning */
@@ -7289,18 +7289,15 @@ static void sortCommand(redisClient *c) {
     j = 0;
 
     if (sortval->type == REDIS_LIST) {
-        list *list = sortval->ptr;
-        listNode *ln;
-        listIter li;
-
-        listRewind(list,&li);
-        while((ln = listNext(&li))) {
-            robj *ele = ln->value;
-            vector[j].obj = ele;
+        lIterator *li = lInitIterator(sortval,0,REDIS_TAIL);
+        lEntry entry;
+        while(lNext(li,&entry)) {
+            vector[j].obj = lGet(&entry);
             vector[j].u.score = 0;
             vector[j].u.cmpobj = NULL;
             j++;
         }
+        lReleaseIterator(li);
     } else {
         dict *set;
         dictIterator *di;
@@ -7410,8 +7407,12 @@ static void sortCommand(redisClient *c) {
             }
         }
     } else {
-        robj *listObject = createListObject();
-        list *listPtr = (list*) listObject->ptr;
+        robj *sobj;
+        if (outputlen > server.list_max_ziplist_entries) {
+            sobj = createListObject();
+        } else {
+            sobj = createZiplistObject();
+        }
 
         /* STORE option specified, set the sorting result as a List object */
         for (j = start; j <= end; j++) {
@@ -7419,31 +7420,30 @@ static void sortCommand(redisClient *c) {
             listIter li;
 
             if (!getop) {
-                listAddNodeTail(listPtr,vector[j].obj);
-                incrRefCount(vector[j].obj);
-            }
-            listRewind(operations,&li);
-            while((ln = listNext(&li))) {
-                redisSortOperation *sop = ln->value;
-                robj *val = lookupKeyByPattern(c->db,sop->pattern,
-                    vector[j].obj);
+                lPush(sobj,vector[j].obj,REDIS_TAIL);
+            } else {
+                listRewind(operations,&li);
+                while((ln = listNext(&li))) {
+                    redisSortOperation *sop = ln->value;
+                    robj *val = lookupKeyByPattern(c->db,sop->pattern,
+                        vector[j].obj);
 
-                if (sop->type == REDIS_SORT_GET) {
-                    if (!val) {
-                        listAddNodeTail(listPtr,createStringObject("",0));
+                    if (sop->type == REDIS_SORT_GET) {
+                        if (!val) val = createStringObject("",0);
+
+                        /* lPush does an incrRefCount, so we should take care
+                         * care of the incremented refcount caused by either
+                         * lookupKeyByPattern or createStringObject("",0) */
+                        lPush(sobj,val,REDIS_TAIL);
+                        decrRefCount(val);
                     } else {
-                        /* We should do a incrRefCount on val because it is
-                         * added to the list, but also a decrRefCount because
-                         * it is returned by lookupKeyByPattern. This results
-                         * in doing nothing at all. */
-                        listAddNodeTail(listPtr,val);
+                        /* always fails */
+                        redisAssert(sop->type == REDIS_SORT_GET);
                     }
-                } else {
-                    redisAssert(sop->type == REDIS_SORT_GET); /* always fails */
                 }
             }
         }
-        if (dictReplace(c->db->dict,storekey,listObject)) {
+        if (dictReplace(c->db->dict,storekey,sobj)) {
             incrRefCount(storekey);
         }
         /* Note: we add 1 because the DB is dirty anyway since even if the
@@ -7454,6 +7454,9 @@ static void sortCommand(redisClient *c) {
     }
 
     /* Cleanup */
+    if (sortval->type == REDIS_LIST)
+        for (j = 0; j < vectorlen; j++)
+            decrRefCount(vector[j].obj);
     decrRefCount(sortval);
     listRelease(operations);
     for (j = 0; j < vectorlen; j++) {

From d71b98650f64b09cdc4d028144617cad4af00e1b Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 23:35:21 +0200
Subject: [PATCH 55/66] ziplistNext should work as expected when called with a
 pointer to ZIP_END

---
 ziplist.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/ziplist.c b/ziplist.c
index 8b96d6d7..433032e6 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -430,7 +430,16 @@ unsigned char *ziplistIndex(unsigned char *zl, int index) {
 /* Return pointer to next entry in ziplist. */
 unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) {
     ((void) zl);
-    return (p[0] == ZIP_END) ? NULL : p+zipRawEntryLength(p);
+
+    /* "p" could be equal to ZIP_END, caused by ziplistDelete,
+     * and we should return NULL. Otherwise, we should return NULL
+     * when the *next* element is ZIP_END (there is no next entry). */
+    if (p[0] == ZIP_END) {
+        return NULL;
+    } else {
+        p = p+zipRawEntryLength(p);
+        return (p[0] == ZIP_END) ? NULL : p;
+    }
 }
 
 /* Return pointer to previous entry in ziplist. */

From dc845730a9afaadf20a6a892a458c219c7350dff Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 23:35:51 +0200
Subject: [PATCH 56/66] use list wrapper functions in computing the dataset
 digest

---
 redis.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/redis.c b/redis.c
index 07a53748..61941d40 100644
--- a/redis.c
+++ b/redis.c
@@ -10923,16 +10923,14 @@ static void computeDatasetDigest(unsigned char *final) {
             if (o->type == REDIS_STRING) {
                 mixObjectDigest(digest,o);
             } else if (o->type == REDIS_LIST) {
-                list *list = o->ptr;
-                listNode *ln;
-                listIter li;
-
-                listRewind(list,&li);
-                while((ln = listNext(&li))) {
-                    robj *eleobj = listNodeValue(ln);
-
+                lIterator *li = lInitIterator(o,0,REDIS_TAIL);
+                lEntry entry;
+                while(lNext(li,&entry)) {
+                    robj *eleobj = lGet(&entry);
                     mixObjectDigest(digest,eleobj);
+                    decrRefCount(eleobj);
                 }
+                lReleaseIterator(li);
             } else if (o->type == REDIS_SET) {
                 dict *set = o->ptr;
                 dictIterator *di = dictGetIterator(set);

From 9eaef89fbc56f3690d9e4294ca96892cdacfea88 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 23:41:04 +0200
Subject: [PATCH 57/66] small refactor of fwrite* commands for AOF rewrite to
 allow writing a bulk long long

---
 redis.c         | 69 +++++++++++++++++++++----------------------------
 staticsymbols.h |  2 +-
 2 files changed, 30 insertions(+), 41 deletions(-)

diff --git a/redis.c b/redis.c
index 61941d40..6e5fd8a0 100644
--- a/redis.c
+++ b/redis.c
@@ -8730,40 +8730,17 @@ fmterr:
     exit(1);
 }
 
-/* Write an object into a file in the bulk format $<count>\r\n<payload>\r\n */
-static int fwriteBulkObject(FILE *fp, robj *obj) {
-    char buf[128];
-    int decrrc = 0;
-
-    /* Avoid the incr/decr ref count business if possible to help
-     * copy-on-write (we are often in a child process when this function
-     * is called).
-     * Also makes sure that key objects don't get incrRefCount-ed when VM
-     * is enabled */
-    if (obj->encoding != REDIS_ENCODING_RAW) {
-        obj = getDecodedObject(obj);
-        decrrc = 1;
-    }
-    snprintf(buf,sizeof(buf),"$%ld\r\n",(long)sdslen(obj->ptr));
-    if (fwrite(buf,strlen(buf),1,fp) == 0) goto err;
-    if (sdslen(obj->ptr) && fwrite(obj->ptr,sdslen(obj->ptr),1,fp) == 0)
-        goto err;
-    if (fwrite("\r\n",2,1,fp) == 0) goto err;
-    if (decrrc) decrRefCount(obj);
-    return 1;
-err:
-    if (decrrc) decrRefCount(obj);
-    return 0;
-}
-
 /* Write binary-safe string into a file in the bulkformat
  * $<count>\r\n<payload>\r\n */
 static int fwriteBulkString(FILE *fp, char *s, unsigned long len) {
-    char buf[128];
-
-    snprintf(buf,sizeof(buf),"$%ld\r\n",(unsigned long)len);
-    if (fwrite(buf,strlen(buf),1,fp) == 0) return 0;
-    if (len && fwrite(s,len,1,fp) == 0) return 0;
+    char cbuf[128];
+    int clen;
+    cbuf[0] = '$';
+    clen = 1+ll2string(cbuf+1,sizeof(cbuf)-1,len);
+    cbuf[clen++] = '\r';
+    cbuf[clen++] = '\n';
+    if (fwrite(cbuf,clen,1,fp) == 0) return 0;
+    if (len > 0 && fwrite(s,len,1,fp) == 0) return 0;
     if (fwrite("\r\n",2,1,fp) == 0) return 0;
     return 1;
 }
@@ -8780,16 +8757,28 @@ static int fwriteBulkDouble(FILE *fp, double d) {
 }
 
 /* Write a long value in bulk format $<count>\r\n<payload>\r\n */
-static int fwriteBulkLong(FILE *fp, long l) {
-    char buf[128], lbuf[128];
-
-    snprintf(lbuf,sizeof(lbuf),"%ld\r\n",l);
-    snprintf(buf,sizeof(buf),"$%lu\r\n",(unsigned long)strlen(lbuf)-2);
-    if (fwrite(buf,strlen(buf),1,fp) == 0) return 0;
-    if (fwrite(lbuf,strlen(lbuf),1,fp) == 0) return 0;
+static int fwriteBulkLongLong(FILE *fp, long long l) {
+    char bbuf[128], lbuf[128];
+    unsigned int blen, llen;
+    llen = ll2string(lbuf,32,l);
+    blen = snprintf(bbuf,sizeof(bbuf),"$%u\r\n%s\r\n",llen,lbuf);
+    if (fwrite(bbuf,blen,1,fp) == 0) return 0;
     return 1;
 }
 
+/* Delegate writing an object to writing a bulk string or bulk long long. */
+static int fwriteBulkObject(FILE *fp, robj *obj) {
+    /* Avoid using getDecodedObject to help copy-on-write (we are often
+     * in a child process when this function is called). */
+    if (obj->encoding == REDIS_ENCODING_INT) {
+        return fwriteBulkLongLong(fp,(long)obj->ptr);
+    } else if (obj->encoding == REDIS_ENCODING_RAW) {
+        return fwriteBulkString(fp,obj->ptr,sdslen(obj->ptr));
+    } else {
+        redisPanic("Unknown string encoding");
+    }
+}
+
 /* Write a sequence of commands able to fully rebuild the dataset into
  * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
 static int rewriteAppendOnlyFile(char *filename) {
@@ -8821,7 +8810,7 @@ static int rewriteAppendOnlyFile(char *filename) {
 
         /* SELECT the new DB */
         if (fwrite(selectcmd,sizeof(selectcmd)-1,1,fp) == 0) goto werr;
-        if (fwriteBulkLong(fp,j) == 0) goto werr;
+        if (fwriteBulkLongLong(fp,j) == 0) goto werr;
 
         /* Iterate this DB writing every entry */
         while((de = dictNext(di)) != NULL) {
@@ -8941,7 +8930,7 @@ static int rewriteAppendOnlyFile(char *filename) {
                 if (expiretime < now) continue;
                 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
                 if (fwriteBulkObject(fp,key) == 0) goto werr;
-                if (fwriteBulkLong(fp,expiretime) == 0) goto werr;
+                if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
             }
             if (swapped) decrRefCount(o);
         }
diff --git a/staticsymbols.h b/staticsymbols.h
index b3ff3db4..16c3680b 100644
--- a/staticsymbols.h
+++ b/staticsymbols.h
@@ -93,7 +93,7 @@ static struct redisFunctionSym symsTable[] = {
 {"freeStringObject",(unsigned long)freeStringObject},
 {"freeZsetObject",(unsigned long)freeZsetObject},
 {"fwriteBulkDouble",(unsigned long)fwriteBulkDouble},
-{"fwriteBulkLong",(unsigned long)fwriteBulkLong},
+{"fwriteBulkLongLong",(unsigned long)fwriteBulkLongLong},
 {"fwriteBulkObject",(unsigned long)fwriteBulkObject},
 {"fwriteBulkString",(unsigned long)fwriteBulkString},
 {"genRedisInfoString",(unsigned long)genRedisInfoString},

From 6ddc908ab611b1d6f1c94c71497e181734ac53df Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Mon, 31 May 2010 23:49:16 +0200
Subject: [PATCH 58/66] support rewriting the AOF with dual list encoding

---
 redis.c | 43 +++++++++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/redis.c b/redis.c
index 6e5fd8a0..6f48e395 100644
--- a/redis.c
+++ b/redis.c
@@ -8843,18 +8843,41 @@ static int rewriteAppendOnlyFile(char *filename) {
                 if (fwriteBulkObject(fp,o) == 0) goto werr;
             } else if (o->type == REDIS_LIST) {
                 /* Emit the RPUSHes needed to rebuild the list */
-                list *list = o->ptr;
-                listNode *ln;
-                listIter li;
+                char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
+                if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+                    unsigned char *zl = o->ptr;
+                    unsigned char *p = ziplistIndex(zl,0);
+                    unsigned char *vstr;
+                    unsigned int vlen;
+                    long long vlong;
 
-                listRewind(list,&li);
-                while((ln = listNext(&li))) {
-                    char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
-                    robj *eleobj = listNodeValue(ln);
+                    while(ziplistGet(p,&vstr,&vlen,&vlong)) {
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,key) == 0) goto werr;
+                        if (vstr) {
+                            if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
+                                goto werr;
+                        } else {
+                            if (fwriteBulkLongLong(fp,vlong) == 0)
+                                goto werr;
+                        }
+                        p = ziplistNext(zl,p);
+                    }
+                } else if (o->encoding == REDIS_ENCODING_LIST) {
+                    list *list = o->ptr;
+                    listNode *ln;
+                    listIter li;
 
-                    if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                    if (fwriteBulkObject(fp,key) == 0) goto werr;
-                    if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    listRewind(list,&li);
+                    while((ln = listNext(&li))) {
+                        robj *eleobj = listNodeValue(ln);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,key) == 0) goto werr;
+                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    }
+                } else {
+                    redisPanic("Unknown list encoding");
                 }
             } else if (o->type == REDIS_SET) {
                 /* Emit the SADDs needed to rebuild the set */

From b84186ff2dadea9fa67f512a743ab97196390f56 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Tue, 1 Jun 2010 17:57:09 +0200
Subject: [PATCH 59/66] fix signedness errors in ziplist testing code

---
 ziplist.c | 48 ++++++++++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 433032e6..b03ba43f 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -584,10 +584,10 @@ void ziplistRepr(unsigned char *zl) {
 
 unsigned char *createList() {
     unsigned char *zl = ziplistNew();
-    zl = ziplistPush(zl, "foo", 3, ZIPLIST_TAIL);
-    zl = ziplistPush(zl, "quux", 4, ZIPLIST_TAIL);
-    zl = ziplistPush(zl, "hello", 5, ZIPLIST_HEAD);
-    zl = ziplistPush(zl, "1024", 4, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD);
+    zl = ziplistPush(zl, (unsigned char*)"1024", 4, ZIPLIST_TAIL);
     return zl;
 }
 
@@ -596,23 +596,23 @@ unsigned char *createIntList() {
     char buf[32];
 
     sprintf(buf, "100");
-    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
     sprintf(buf, "128000");
-    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
     sprintf(buf, "-100");
-    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_HEAD);
+    zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD);
     sprintf(buf, "4294967296");
-    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_HEAD);
+    zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD);
     sprintf(buf, "non integer");
-    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
     sprintf(buf, "much much longer non integer");
-    zl = ziplistPush(zl, buf, strlen(buf), ZIPLIST_TAIL);
+    zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
     return zl;
 }
 
 int main(int argc, char **argv) {
     unsigned char *zl, *p;
-    char *entry;
+    unsigned char *entry;
     unsigned int elen;
     long long value;
     sds s;
@@ -852,19 +852,19 @@ int main(int argc, char **argv) {
     printf("Delete foo while iterating:\n");
     {
         zl = createList();
-        p = ziplistIndex(zl, 0);
-        while (ziplistGet(p, &entry, &elen, &value)) {
-            if (entry && strncmp("foo", entry, elen) == 0) {
+        p = ziplistIndex(zl,0);
+        while (ziplistGet(p,&entry,&elen,&value)) {
+            if (entry && strncmp("foo",(char*)entry,elen) == 0) {
                 printf("Delete foo\n");
-                zl = ziplistDelete(zl, &p, ZIPLIST_TAIL);
+                zl = ziplistDelete(zl,&p);
             } else {
                 printf("Entry: ");
                 if (entry) {
                     fwrite(entry,elen,1,stdout);
                 } else {
-                    printf("%lld", value);
+                    printf("%lld",value);
                 }
-                p = ziplistNext(p);
+                p = ziplistNext(zl,p);
                 printf("\n");
             }
         }
@@ -879,7 +879,7 @@ int main(int argc, char **argv) {
         int i,len;
         for (i = 0; i < 1000; i++) {
             len = sprintf(buf,"%d",i);
-            zl = ziplistPush(zl,buf,len,ZIPLIST_TAIL);
+            zl = ziplistPush(zl,(unsigned char*)buf,len,ZIPLIST_TAIL);
         }
         for (i = 0; i < 1000; i++) {
             p = ziplistIndex(zl,i);
@@ -896,22 +896,22 @@ int main(int argc, char **argv) {
     printf("Compare strings with ziplist entries:\n");
     {
         zl = createList();
-        p = ziplistIndex(zl, 0);
-        if (!ziplistCompare(p,"hello",5)) {
+        p = ziplistIndex(zl,0);
+        if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
             printf("ERROR: not \"hello\"\n");
             return 1;
         }
-        if (ziplistCompare(p,"hella",5)) {
+        if (ziplistCompare(p,(unsigned char*)"hella",5)) {
             printf("ERROR: \"hella\"\n");
             return 1;
         }
 
-        p = ziplistIndex(zl, 3);
-        if (!ziplistCompare(p,"1024",4)) {
+        p = ziplistIndex(zl,3);
+        if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
             printf("ERROR: not \"1024\"\n");
             return 1;
         }
-        if (ziplistCompare(p,"1025",4)) {
+        if (ziplistCompare(p,(unsigned char*)"1025",4)) {
             printf("ERROR: \"1025\"\n");
             return 1;
         }

From ffc15852676131003d1f5e468467f15f57a8e0bf Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Tue, 1 Jun 2010 17:57:51 +0200
Subject: [PATCH 60/66] added stress test for heavy i/o in ziplists

---
 ziplist.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/ziplist.c b/ziplist.c
index b03ba43f..745a68cb 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -581,6 +581,7 @@ void ziplistRepr(unsigned char *zl) {
 }
 
 #ifdef ZIPLIST_TEST_MAIN
+#include <sys/time.h>
 
 unsigned char *createList() {
     unsigned char *zl = ziplistNew();
@@ -610,6 +611,35 @@ unsigned char *createIntList() {
     return zl;
 }
 
+long long usec(void) {
+    struct timeval tv;
+    gettimeofday(&tv,NULL);
+    return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
+}
+
+void stress(int pos, int num, int maxsize, int dnum) {
+    int i,j,k;
+    unsigned char *zl;
+    char posstr[2][5] = { "HEAD", "TAIL" };
+    long long start;
+    for (i = 0; i < maxsize; i+=dnum) {
+        zl = ziplistNew();
+        for (j = 0; j < i; j++) {
+            zl = ziplistPush(zl,(unsigned char*)"quux",4,ZIPLIST_TAIL);
+        }
+
+        /* Do num times a push+pop from pos */
+        start = usec();
+        for (k = 0; k < num; k++) {
+            zl = ziplistPush(zl,(unsigned char*)"quux",4,pos);
+            zl = ziplistDeleteRange(zl,0,1);
+        }
+        printf("List size: %8d, bytes: %8d, %dx push+pop (%s): %6lld usec\n",
+            i,ZIPLIST_BYTES(zl),num,posstr[pos],usec()-start);
+        zfree(zl);
+    }
+}
+
 int main(int argc, char **argv) {
     unsigned char *zl, *p;
     unsigned char *entry;
@@ -918,6 +948,13 @@ int main(int argc, char **argv) {
         printf("SUCCESS\n");
     }
 
+    printf("Stress with variable ziplist size:\n");
+    {
+        stress(ZIPLIST_HEAD,100000,16384,256);
+        stress(ZIPLIST_TAIL,100000,16384,256);
+    }
+
     return 0;
 }
+
 #endif

From e1f93d4b2cc94fec29ede9b3cedea891ee9bd050 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Tue, 1 Jun 2010 18:55:37 +0200
Subject: [PATCH 61/66] use integer types from stdint.h to be more verbose on
 the size in bytes of encoded elements. update list length to use 2 bytes
 instead of 1.

---
 ziplist.c | 104 ++++++++++++++++++++++++++++--------------------------
 1 file changed, 54 insertions(+), 50 deletions(-)

diff --git a/ziplist.c b/ziplist.c
index 745a68cb..d1d7a151 100644
--- a/ziplist.c
+++ b/ziplist.c
@@ -17,6 +17,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
 #include <assert.h>
 #include <limits.h>
 #include "zmalloc.h"
@@ -33,9 +34,9 @@
 
 /* Entry encoding */
 #define ZIP_ENC_RAW     0
-#define ZIP_ENC_SHORT   1
-#define ZIP_ENC_INT     2
-#define ZIP_ENC_LLONG   3
+#define ZIP_ENC_INT16   1
+#define ZIP_ENC_INT32   2
+#define ZIP_ENC_INT64   3
 #define ZIP_ENCODING(p) ((p)[0] >> 6)
 
 /* Length encoding for raw entries */
@@ -44,15 +45,18 @@
 #define ZIP_LEN_UINT32  2
 
 /* Utility macros */
-#define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
-#define ZIPLIST_TAIL_OFFSET(zl) (*((unsigned int*)((zl)+sizeof(unsigned int))))
-#define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
-#define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
-#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
-#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+ZIPLIST_TAIL_OFFSET(zl))
-#define ZIPLIST_ENTRY_END(zl) ((zl)+ZIPLIST_BYTES(zl)-1)
+#define ZIPLIST_BYTES(zl)       (*((uint32_t*)(zl)))
+#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
+#define ZIPLIST_LENGTH(zl)      (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
+#define ZIPLIST_HEADER_SIZE     (sizeof(uint32_t)*2+sizeof(uint16_t))
+#define ZIPLIST_ENTRY_HEAD(zl)  ((zl)+ZIPLIST_HEADER_SIZE)
+#define ZIPLIST_ENTRY_TAIL(zl)  ((zl)+ZIPLIST_TAIL_OFFSET(zl))
+#define ZIPLIST_ENTRY_END(zl)   ((zl)+ZIPLIST_BYTES(zl)-1)
+
+/* We know a positive increment can only be 1 because entries can only be
+ * pushed one at a time. */
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
-    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
+    if (ZIPLIST_LENGTH(zl) < UINT16_MAX) ZIPLIST_LENGTH(zl)+=incr; }
 
 typedef struct zlentry {
     unsigned int prevrawlensize, prevrawlen;
@@ -64,12 +68,12 @@ typedef struct zlentry {
 
 /* Return bytes needed to store integer encoded by 'encoding' */
 static unsigned int zipEncodingSize(unsigned char encoding) {
-    if (encoding == ZIP_ENC_SHORT) {
-        return sizeof(short int);
-    } else if (encoding == ZIP_ENC_INT) {
-        return sizeof(int);
-    } else if (encoding == ZIP_ENC_LLONG) {
-        return sizeof(long long);
+    if (encoding == ZIP_ENC_INT16) {
+        return sizeof(int16_t);
+    } else if (encoding == ZIP_ENC_INT32) {
+        return sizeof(int32_t);
+    } else if (encoding == ZIP_ENC_INT64) {
+        return sizeof(int64_t);
     }
     assert(NULL);
 }
@@ -180,12 +184,12 @@ static int zipTryEncoding(unsigned char *entry, long long *v, unsigned char *enc
     if (entry[0] == '-' || (entry[0] >= '0' && entry[0] <= '9')) {
         value = strtoll((char*)entry,&eptr,10);
         if (eptr[0] != '\0') return 0;
-        if (value >= SHRT_MIN && value <= SHRT_MAX) {
-            *encoding = ZIP_ENC_SHORT;
-        } else if (value >= INT_MIN && value <= INT_MAX) {
-            *encoding = ZIP_ENC_INT;
+        if (value >= INT16_MIN && value <= INT16_MAX) {
+            *encoding = ZIP_ENC_INT16;
+        } else if (value >= INT32_MIN && value <= INT32_MAX) {
+            *encoding = ZIP_ENC_INT32;
         } else {
-            *encoding = ZIP_ENC_LLONG;
+            *encoding = ZIP_ENC_INT64;
         }
         *v = value;
         return 1;
@@ -194,38 +198,38 @@ static int zipTryEncoding(unsigned char *entry, long long *v, unsigned char *enc
 }
 
 /* Store integer 'value' at 'p', encoded as 'encoding' */
-static void zipSaveInteger(unsigned char *p, long long value, unsigned char encoding) {
-    short int s;
-    int i;
-    long long l;
-    if (encoding == ZIP_ENC_SHORT) {
-        s = value;
-        memcpy(p,&s,sizeof(s));
-    } else if (encoding == ZIP_ENC_INT) {
-        i = value;
-        memcpy(p,&i,sizeof(i));
-    } else if (encoding == ZIP_ENC_LLONG) {
-        l = value;
-        memcpy(p,&l,sizeof(l));
+static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encoding) {
+    int16_t i16;
+    int32_t i32;
+    int64_t i64;
+    if (encoding == ZIP_ENC_INT16) {
+        i16 = value;
+        memcpy(p,&i16,sizeof(i16));
+    } else if (encoding == ZIP_ENC_INT32) {
+        i32 = value;
+        memcpy(p,&i32,sizeof(i32));
+    } else if (encoding == ZIP_ENC_INT64) {
+        i64 = value;
+        memcpy(p,&i64,sizeof(i64));
     } else {
         assert(NULL);
     }
 }
 
 /* Read integer encoded as 'encoding' from 'p' */
-static long long zipLoadInteger(unsigned char *p, unsigned char encoding) {
-    short int s;
-    int i;
-    long long l, ret;
-    if (encoding == ZIP_ENC_SHORT) {
-        memcpy(&s,p,sizeof(s));
-        ret = s;
-    } else if (encoding == ZIP_ENC_INT) {
-        memcpy(&i,p,sizeof(i));
-        ret = i;
-    } else if (encoding == ZIP_ENC_LLONG) {
-        memcpy(&l,p,sizeof(l));
-        ret = l;
+static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
+    int16_t i16;
+    int32_t i32;
+    int64_t i64, ret;
+    if (encoding == ZIP_ENC_INT16) {
+        memcpy(&i16,p,sizeof(i16));
+        ret = i16;
+    } else if (encoding == ZIP_ENC_INT32) {
+        memcpy(&i32,p,sizeof(i32));
+        ret = i32;
+    } else if (encoding == ZIP_ENC_INT64) {
+        memcpy(&i64,p,sizeof(i64));
+        ret = i64;
     } else {
         assert(NULL);
     }
@@ -539,7 +543,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int
 /* Return length of ziplist. */
 unsigned int ziplistLen(unsigned char *zl) {
     unsigned int len = 0;
-    if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) {
+    if (ZIPLIST_LENGTH(zl) < UINT16_MAX) {
         len = ZIPLIST_LENGTH(zl);
     } else {
         unsigned char *p = zl+ZIPLIST_HEADER_SIZE;
@@ -549,7 +553,7 @@ unsigned int ziplistLen(unsigned char *zl) {
         }
 
         /* Re-store length if small enough */
-        if (len < ZIP_BIGLEN) ZIPLIST_LENGTH(zl) = len;
+        if (len < UINT16_MAX) ZIPLIST_LENGTH(zl) = len;
     }
     return len;
 }

From 74e0f445a8851d0cbf6898aa9fbea1814bb8cd1d Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Thu, 3 Jun 2010 00:48:52 +0200
Subject: [PATCH 62/66] use ziplists in SORT STORE until the thresholds are
 determined

---
 redis.c | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/redis.c b/redis.c
index 6f48e395..41bc9c97 100644
--- a/redis.c
+++ b/redis.c
@@ -7407,12 +7407,7 @@ static void sortCommand(redisClient *c) {
             }
         }
     } else {
-        robj *sobj;
-        if (outputlen > server.list_max_ziplist_entries) {
-            sobj = createListObject();
-        } else {
-            sobj = createZiplistObject();
-        }
+        robj *sobj = createZiplistObject();
 
         /* STORE option specified, set the sorting result as a List object */
         for (j = start; j <= end; j++) {

From 178d6903722e80bd68d1b8dacbe3d4fc406d2cb5 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Thu, 3 Jun 2010 00:50:33 +0200
Subject: [PATCH 63/66] update Makefile to include ziplist.o

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b7dfb1cd..718f4521 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@ endif
 CCOPT= $(CFLAGS) $(CCLINK) $(ARCH) $(PROF)
 DEBUG?= -g -rdynamic -ggdb 
 
-OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o
+OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o
 BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o
 CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o linenoise.o
 CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o
@@ -53,6 +53,7 @@ redis.o: redis.c fmacros.h config.h redis.h ae.h sds.h anet.h dict.h \
   adlist.h zmalloc.h lzf.h pqsort.h zipmap.h staticsymbols.h sha1.h
 sds.o: sds.c sds.h zmalloc.h
 zipmap.o: zipmap.c zmalloc.h
+ziplist.o: ziplist.c zmalloc.h
 zmalloc.o: zmalloc.c config.h
 
 redis-server: $(OBJ)

From d0686e070d683d0577c6f6c2f088a55101b74980 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 4 Jun 2010 10:57:31 +0200
Subject: [PATCH 64/66] add thresholds for converting a ziplist to a real list

---
 redis.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 66 insertions(+), 3 deletions(-)

diff --git a/redis.c b/redis.c
index 16384d87..2ec4ca55 100644
--- a/redis.c
+++ b/redis.c
@@ -237,9 +237,11 @@ static char* strencoding[] = {
 #define APPENDFSYNC_ALWAYS 1
 #define APPENDFSYNC_EVERYSEC 2
 
-/* Hashes related defaults */
+/* Zip structure related defaults */
 #define REDIS_HASH_MAX_ZIPMAP_ENTRIES 64
 #define REDIS_HASH_MAX_ZIPMAP_VALUE 512
+#define REDIS_LIST_MAX_ZIPLIST_ENTRIES 1024
+#define REDIS_LIST_MAX_ZIPLIST_VALUE 32
 
 /* We can print the stacktrace, so our assert is defined this way: */
 #define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1)))
@@ -425,9 +427,11 @@ struct redisServer {
     off_t vm_page_size;
     off_t vm_pages;
     unsigned long long vm_max_memory;
-    /* Hashes config */
+    /* Zip structure config */
     size_t hash_max_zipmap_entries;
     size_t hash_max_zipmap_value;
+    size_t list_max_ziplist_entries;
+    size_t list_max_ziplist_value;
     /* Virtual memory state */
     FILE *vm_fp;
     int vm_fd;
@@ -646,6 +650,7 @@ static struct redisCommand *lookupCommand(char *name);
 static void call(redisClient *c, struct redisCommand *cmd);
 static void resetClient(redisClient *c);
 static void convertToRealHash(robj *o);
+static void convertList(robj *o, int enc);
 static int pubsubUnsubscribeAllChannels(redisClient *c, int notify);
 static int pubsubUnsubscribeAllPatterns(redisClient *c, int notify);
 static void freePubsubPattern(void *p);
@@ -1755,6 +1760,8 @@ static void initServerConfig() {
     server.vm_blocked_clients = 0;
     server.hash_max_zipmap_entries = REDIS_HASH_MAX_ZIPMAP_ENTRIES;
     server.hash_max_zipmap_value = REDIS_HASH_MAX_ZIPMAP_VALUE;
+    server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES;
+    server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE;
     server.shutdown_asap = 0;
 
     resetServerSaveParams();
@@ -2033,6 +2040,10 @@ static void loadServerConfig(char *filename) {
             server.hash_max_zipmap_entries = memtoll(argv[1], NULL);
         } else if (!strcasecmp(argv[0],"hash-max-zipmap-value") && argc == 2){
             server.hash_max_zipmap_value = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){
+            server.list_max_ziplist_entries = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2){
+            server.list_max_ziplist_value = memtoll(argv[1], NULL);
         } else {
             err = "Bad directive or wrong number of arguments"; goto loaderr;
         }
@@ -4154,12 +4165,24 @@ static robj *rdbLoadObject(int type, FILE *fp) {
         /* Read list value */
         if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
 
-        o = createZiplistObject();
+        /* Use a real list when there are too many entries */
+        if (len > server.list_max_ziplist_entries) {
+            o = createListObject();
+        } else {
+            o = createZiplistObject();
+        }
 
         /* Load every single element of the list */
         while(len--) {
             if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
 
+            /* If we are using a ziplist and the value is too big, convert
+             * the object to a real list. */
+            if (o->encoding == REDIS_ENCODING_ZIPLIST &&
+                ele->encoding == REDIS_ENCODING_RAW &&
+                sdslen(ele->ptr) > server.list_max_ziplist_value)
+                    convertList(o,REDIS_ENCODING_LIST);
+
             if (o->encoding == REDIS_ENCODING_ZIPLIST) {
                 dec = getDecodedObject(ele);
                 o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL);
@@ -4882,7 +4905,25 @@ static void moveCommand(redisClient *c) {
 }
 
 /* =================================== Lists ================================ */
+
+
+/* Check the argument length to see if it requires us to convert the ziplist
+ * to a real list. Only check raw-encoded objects because integer encoded
+ * objects are never too long. */
+static void listTryConversion(robj *subject, robj *value) {
+    if (subject->encoding != REDIS_ENCODING_ZIPLIST) return;
+    if (value->encoding == REDIS_ENCODING_RAW &&
+        sdslen(value->ptr) > server.list_max_ziplist_value)
+            convertList(subject,REDIS_ENCODING_LIST);
+}
+
 static void lPush(robj *subject, robj *value, int where) {
+    /* Check if we need to convert the ziplist */
+    listTryConversion(subject,value);
+    if (subject->encoding == REDIS_ENCODING_ZIPLIST &&
+        ziplistLen(subject->ptr) > server.list_max_ziplist_entries)
+            convertList(subject,REDIS_ENCODING_LIST);
+
     if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
         int pos = (where == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
         value = getDecodedObject(value);
@@ -5077,6 +5118,27 @@ static void lDelete(lEntry *entry) {
     }
 }
 
+static void convertList(robj *subject, int enc) {
+    lIterator *li;
+    lEntry entry;
+    redisAssert(subject->type == REDIS_LIST);
+
+    if (enc == REDIS_ENCODING_LIST) {
+        list *l = listCreate();
+
+        /* lGet returns a robj with incremented refcount */
+        li = lInitIterator(subject,0,REDIS_TAIL);
+        while (lNext(li,&entry)) listAddNodeTail(l,lGet(&entry));
+        lReleaseIterator(li);
+
+        subject->encoding = REDIS_ENCODING_LIST;
+        zfree(subject->ptr);
+        subject->ptr = l;
+    } else {
+        redisPanic("Unsupported list conversion");
+    }
+}
+
 static void pushGenericCommand(redisClient *c, int where) {
     robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
     if (lobj == NULL) {
@@ -5157,6 +5219,7 @@ static void lsetCommand(redisClient *c) {
     int index = atoi(c->argv[2]->ptr);
     robj *value = c->argv[3];
 
+    listTryConversion(o,value);
     if (o->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p, *zl = o->ptr;
         p = ziplistIndex(zl,index);

From 003f0840ffa7f69fb40836bf9510c9d77be0e830 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 4 Jun 2010 11:04:02 +0200
Subject: [PATCH 65/66] renamed list wrapper functions to be more verbose

---
 redis.c | 152 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 76 insertions(+), 76 deletions(-)

diff --git a/redis.c b/redis.c
index 2ec4ca55..30f05a63 100644
--- a/redis.c
+++ b/redis.c
@@ -650,7 +650,7 @@ static struct redisCommand *lookupCommand(char *name);
 static void call(redisClient *c, struct redisCommand *cmd);
 static void resetClient(redisClient *c);
 static void convertToRealHash(robj *o);
-static void convertList(robj *o, int enc);
+static void listTypeConvert(robj *o, int enc);
 static int pubsubUnsubscribeAllChannels(redisClient *c, int notify);
 static int pubsubUnsubscribeAllPatterns(redisClient *c, int notify);
 static void freePubsubPattern(void *p);
@@ -4181,7 +4181,7 @@ static robj *rdbLoadObject(int type, FILE *fp) {
             if (o->encoding == REDIS_ENCODING_ZIPLIST &&
                 ele->encoding == REDIS_ENCODING_RAW &&
                 sdslen(ele->ptr) > server.list_max_ziplist_value)
-                    convertList(o,REDIS_ENCODING_LIST);
+                    listTypeConvert(o,REDIS_ENCODING_LIST);
 
             if (o->encoding == REDIS_ENCODING_ZIPLIST) {
                 dec = getDecodedObject(ele);
@@ -4910,19 +4910,19 @@ static void moveCommand(redisClient *c) {
 /* Check the argument length to see if it requires us to convert the ziplist
  * to a real list. Only check raw-encoded objects because integer encoded
  * objects are never too long. */
-static void listTryConversion(robj *subject, robj *value) {
+static void listTypeTryConversion(robj *subject, robj *value) {
     if (subject->encoding != REDIS_ENCODING_ZIPLIST) return;
     if (value->encoding == REDIS_ENCODING_RAW &&
         sdslen(value->ptr) > server.list_max_ziplist_value)
-            convertList(subject,REDIS_ENCODING_LIST);
+            listTypeConvert(subject,REDIS_ENCODING_LIST);
 }
 
-static void lPush(robj *subject, robj *value, int where) {
+static void listTypePush(robj *subject, robj *value, int where) {
     /* Check if we need to convert the ziplist */
-    listTryConversion(subject,value);
+    listTypeTryConversion(subject,value);
     if (subject->encoding == REDIS_ENCODING_ZIPLIST &&
         ziplistLen(subject->ptr) > server.list_max_ziplist_entries)
-            convertList(subject,REDIS_ENCODING_LIST);
+            listTypeConvert(subject,REDIS_ENCODING_LIST);
 
     if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
         int pos = (where == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
@@ -4941,7 +4941,7 @@ static void lPush(robj *subject, robj *value, int where) {
     }
 }
 
-static robj *lPop(robj *subject, int where) {
+static robj *listTypePop(robj *subject, int where) {
     robj *value = NULL;
     if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p;
@@ -4978,7 +4978,7 @@ static robj *lPop(robj *subject, int where) {
     return value;
 }
 
-static unsigned long lLength(robj *subject) {
+static unsigned long listTypeLength(robj *subject) {
     if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
         return ziplistLen(subject->ptr);
     } else if (subject->encoding == REDIS_ENCODING_LIST) {
@@ -4995,18 +4995,18 @@ typedef struct {
     unsigned char direction; /* Iteration direction */
     unsigned char *zi;
     listNode *ln;
-} lIterator;
+} listTypeIterator;
 
 /* Structure for an entry while iterating over a list. */
 typedef struct {
-    lIterator *li;
+    listTypeIterator *li;
     unsigned char *zi;  /* Entry in ziplist */
     listNode *ln;       /* Entry in linked list */
-} lEntry;
+} listTypeEntry;
 
 /* Initialize an iterator at the specified index. */
-static lIterator *lInitIterator(robj *subject, int index, unsigned char direction) {
-    lIterator *li = zmalloc(sizeof(lIterator));
+static listTypeIterator *listTypeInitIterator(robj *subject, int index, unsigned char direction) {
+    listTypeIterator *li = zmalloc(sizeof(listTypeIterator));
     li->subject = subject;
     li->encoding = subject->encoding;
     li->direction = direction;
@@ -5021,14 +5021,14 @@ static lIterator *lInitIterator(robj *subject, int index, unsigned char directio
 }
 
 /* Clean up the iterator. */
-static void lReleaseIterator(lIterator *li) {
+static void listTypeReleaseIterator(listTypeIterator *li) {
     zfree(li);
 }
 
 /* Stores pointer to current the entry in the provided entry structure
  * and advances the position of the iterator. Returns 1 when the current
  * entry is in fact an entry, 0 otherwise. */
-static int lNext(lIterator *li, lEntry *entry) {
+static int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
     entry->li = li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         entry->zi = li->zi;
@@ -5055,8 +5055,8 @@ static int lNext(lIterator *li, lEntry *entry) {
 }
 
 /* Return entry or NULL at the current position of the iterator. */
-static robj *lGet(lEntry *entry) {
-    lIterator *li = entry->li;
+static robj *listTypeGet(listTypeEntry *entry) {
+    listTypeIterator *li = entry->li;
     robj *value = NULL;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *vstr;
@@ -5081,8 +5081,8 @@ static robj *lGet(lEntry *entry) {
 }
 
 /* Compare the given object with the entry at the current position. */
-static int lEqual(lEntry *entry, robj *o) {
-    lIterator *li = entry->li;
+static int listTypeEqual(listTypeEntry *entry, robj *o) {
+    listTypeIterator *li = entry->li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         redisAssert(o->encoding == REDIS_ENCODING_RAW);
         return ziplistCompare(entry->zi,o->ptr,sdslen(o->ptr));
@@ -5094,8 +5094,8 @@ static int lEqual(lEntry *entry, robj *o) {
 }
 
 /* Delete the element pointed to. */
-static void lDelete(lEntry *entry) {
-    lIterator *li = entry->li;
+static void listTypeDelete(listTypeEntry *entry) {
+    listTypeIterator *li = entry->li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p = entry->zi;
         li->subject->ptr = ziplistDelete(li->subject->ptr,&p);
@@ -5118,18 +5118,18 @@ static void lDelete(lEntry *entry) {
     }
 }
 
-static void convertList(robj *subject, int enc) {
-    lIterator *li;
-    lEntry entry;
+static void listTypeConvert(robj *subject, int enc) {
+    listTypeIterator *li;
+    listTypeEntry entry;
     redisAssert(subject->type == REDIS_LIST);
 
     if (enc == REDIS_ENCODING_LIST) {
         list *l = listCreate();
 
-        /* lGet returns a robj with incremented refcount */
-        li = lInitIterator(subject,0,REDIS_TAIL);
-        while (lNext(li,&entry)) listAddNodeTail(l,lGet(&entry));
-        lReleaseIterator(li);
+        /* listTypeGet returns a robj with incremented refcount */
+        li = listTypeInitIterator(subject,0,REDIS_TAIL);
+        while (listTypeNext(li,&entry)) listAddNodeTail(l,listTypeGet(&entry));
+        listTypeReleaseIterator(li);
 
         subject->encoding = REDIS_ENCODING_LIST;
         zfree(subject->ptr);
@@ -5158,8 +5158,8 @@ static void pushGenericCommand(redisClient *c, int where) {
             return;
         }
     }
-    lPush(lobj,c->argv[2],where);
-    addReplyLongLong(c,lLength(lobj));
+    listTypePush(lobj,c->argv[2],where);
+    addReplyLongLong(c,listTypeLength(lobj));
     server.dirty++;
 }
 
@@ -5174,7 +5174,7 @@ static void rpushCommand(redisClient *c) {
 static void llenCommand(redisClient *c) {
     robj *o = lookupKeyReadOrReply(c,c->argv[1],shared.czero);
     if (o == NULL || checkType(c,o,REDIS_LIST)) return;
-    addReplyUlong(c,lLength(o));
+    addReplyUlong(c,listTypeLength(o));
 }
 
 static void lindexCommand(redisClient *c) {
@@ -5219,7 +5219,7 @@ static void lsetCommand(redisClient *c) {
     int index = atoi(c->argv[2]->ptr);
     robj *value = c->argv[3];
 
-    listTryConversion(o,value);
+    listTypeTryConversion(o,value);
     if (o->encoding == REDIS_ENCODING_ZIPLIST) {
         unsigned char *p, *zl = o->ptr;
         p = ziplistIndex(zl,index);
@@ -5253,13 +5253,13 @@ static void popGenericCommand(redisClient *c, int where) {
     robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk);
     if (o == NULL || checkType(c,o,REDIS_LIST)) return;
 
-    robj *value = lPop(o,where);
+    robj *value = listTypePop(o,where);
     if (value == NULL) {
         addReply(c,shared.nullbulk);
     } else {
         addReplyBulk(c,value);
         decrRefCount(value);
-        if (lLength(o) == 0) dbDelete(c->db,c->argv[1]);
+        if (listTypeLength(o) == 0) dbDelete(c->db,c->argv[1]);
         server.dirty++;
     }
 }
@@ -5278,11 +5278,11 @@ static void lrangeCommand(redisClient *c) {
     int end = atoi(c->argv[3]->ptr);
     int llen;
     int rangelen, j;
-    lEntry entry;
+    listTypeEntry entry;
 
     if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL
          || checkType(c,o,REDIS_LIST)) return;
-    llen = lLength(o);
+    llen = listTypeLength(o);
 
     /* convert negative indexes */
     if (start < 0) start = llen+start;
@@ -5301,14 +5301,14 @@ static void lrangeCommand(redisClient *c) {
 
     /* Return the result in form of a multi-bulk reply */
     addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",rangelen));
-    lIterator *li = lInitIterator(o,start,REDIS_TAIL);
+    listTypeIterator *li = listTypeInitIterator(o,start,REDIS_TAIL);
     for (j = 0; j < rangelen; j++) {
-        redisAssert(lNext(li,&entry));
-        value = lGet(&entry);
+        redisAssert(listTypeNext(li,&entry));
+        value = listTypeGet(&entry);
         addReplyBulk(c,value);
         decrRefCount(value);
     }
-    lReleaseIterator(li);
+    listTypeReleaseIterator(li);
 }
 
 static void ltrimCommand(redisClient *c) {
@@ -5322,7 +5322,7 @@ static void ltrimCommand(redisClient *c) {
 
     if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.ok)) == NULL ||
         checkType(c,o,REDIS_LIST)) return;
-    llen = lLength(o);
+    llen = listTypeLength(o);
 
     /* convert negative indexes */
     if (start < 0) start = llen+start;
@@ -5358,7 +5358,7 @@ static void ltrimCommand(redisClient *c) {
     } else {
         redisPanic("Unknown list encoding");
     }
-    if (lLength(o) == 0) dbDelete(c->db,c->argv[1]);
+    if (listTypeLength(o) == 0) dbDelete(c->db,c->argv[1]);
     server.dirty++;
     addReply(c,shared.ok);
 }
@@ -5367,7 +5367,7 @@ static void lremCommand(redisClient *c) {
     robj *subject, *obj = c->argv[3];
     int toremove = atoi(c->argv[2]->ptr);
     int removed = 0;
-    lEntry entry;
+    listTypeEntry entry;
 
     subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
     if (subject == NULL || checkType(c,subject,REDIS_LIST)) return;
@@ -5376,29 +5376,29 @@ static void lremCommand(redisClient *c) {
     if (subject->encoding == REDIS_ENCODING_ZIPLIST)
         obj = getDecodedObject(obj);
 
-    lIterator *li;
+    listTypeIterator *li;
     if (toremove < 0) {
         toremove = -toremove;
-        li = lInitIterator(subject,-1,REDIS_HEAD);
+        li = listTypeInitIterator(subject,-1,REDIS_HEAD);
     } else {
-        li = lInitIterator(subject,0,REDIS_TAIL);
+        li = listTypeInitIterator(subject,0,REDIS_TAIL);
     }
 
-    while (lNext(li,&entry)) {
-        if (lEqual(&entry,obj)) {
-            lDelete(&entry);
+    while (listTypeNext(li,&entry)) {
+        if (listTypeEqual(&entry,obj)) {
+            listTypeDelete(&entry);
             server.dirty++;
             removed++;
             if (toremove && removed == toremove) break;
         }
     }
-    lReleaseIterator(li);
+    listTypeReleaseIterator(li);
 
     /* Clean up raw encoded object */
     if (subject->encoding == REDIS_ENCODING_ZIPLIST)
         decrRefCount(obj);
 
-    if (lLength(subject) == 0) dbDelete(c->db,c->argv[1]);
+    if (listTypeLength(subject) == 0) dbDelete(c->db,c->argv[1]);
     addReplySds(c,sdscatprintf(sdsempty(),":%d\r\n",removed));
 }
 
@@ -5422,12 +5422,12 @@ static void rpoplpushcommand(redisClient *c) {
     if ((sobj = lookupKeyWriteOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
         checkType(c,sobj,REDIS_LIST)) return;
 
-    if (lLength(sobj) == 0) {
+    if (listTypeLength(sobj) == 0) {
         addReply(c,shared.nullbulk);
     } else {
         robj *dobj = lookupKeyWrite(c->db,c->argv[2]);
         if (dobj && checkType(c,dobj,REDIS_LIST)) return;
-        value = lPop(sobj,REDIS_TAIL);
+        value = listTypePop(sobj,REDIS_TAIL);
 
         /* Add the element to the target list (unless it's directly
          * passed to some BLPOP-ing client */
@@ -5437,17 +5437,17 @@ static void rpoplpushcommand(redisClient *c) {
                 dobj = createZiplistObject();
                 dbAdd(c->db,c->argv[2],dobj);
             }
-            lPush(dobj,value,REDIS_HEAD);
+            listTypePush(dobj,value,REDIS_HEAD);
         }
 
         /* Send the element to the client as reply as well */
         addReplyBulk(c,value);
 
-        /* lPop returns an object with its refcount incremented */
+        /* listTypePop returns an object with its refcount incremented */
         decrRefCount(value);
 
         /* Delete the source list when it is empty */
-        if (lLength(sobj) == 0) dbDelete(c->db,c->argv[1]);
+        if (listTypeLength(sobj) == 0) dbDelete(c->db,c->argv[1]);
         server.dirty++;
     }
 }
@@ -6070,7 +6070,7 @@ static zskiplistNode *zslFirstWithScore(zskiplist *zsl, double score) {
  * Returns 0 when the element cannot be found, rank otherwise.
  * Note that the rank is 1-based due to the span of zsl->header to the
  * first element. */
-static unsigned long zslGetRank(zskiplist *zsl, double score, robj *o) {
+static unsigned long zslistTypeGetRank(zskiplist *zsl, double score, robj *o) {
     zskiplistNode *x;
     unsigned long rank = 0;
     int i;
@@ -6094,7 +6094,7 @@ static unsigned long zslGetRank(zskiplist *zsl, double score, robj *o) {
 }
 
 /* Finds an element by its rank. The rank argument needs to be 1-based. */
-zskiplistNode* zslGetElementByRank(zskiplist *zsl, unsigned long rank) {
+zskiplistNode* zslistTypeGetElementByRank(zskiplist *zsl, unsigned long rank) {
     zskiplistNode *x;
     unsigned long traversed = 0;
     int i;
@@ -6560,10 +6560,10 @@ static void zrangeGenericCommand(redisClient *c, int reverse) {
     /* check if starting point is trivial, before searching
      * the element in log(N) time */
     if (reverse) {
-        ln = start == 0 ? zsl->tail : zslGetElementByRank(zsl, llen-start);
+        ln = start == 0 ? zsl->tail : zslistTypeGetElementByRank(zsl, llen-start);
     } else {
         ln = start == 0 ?
-            zsl->header->forward[0] : zslGetElementByRank(zsl, start+1);
+            zsl->header->forward[0] : zslistTypeGetElementByRank(zsl, start+1);
     }
 
     /* Return the result in form of a multi-bulk reply */
@@ -6760,7 +6760,7 @@ static void zrankGenericCommand(redisClient *c, int reverse) {
     }
 
     score = dictGetEntryVal(de);
-    rank = zslGetRank(zsl, *score, c->argv[2]);
+    rank = zslistTypeGetRank(zsl, *score, c->argv[2]);
     if (rank) {
         if (reverse) {
             addReplyLongLong(c, zsl->length - rank);
@@ -7407,7 +7407,7 @@ static void sortCommand(redisClient *c) {
 
     /* Load the sorting vector with all the objects to sort */
     switch(sortval->type) {
-    case REDIS_LIST: vectorlen = lLength(sortval); break;
+    case REDIS_LIST: vectorlen = listTypeLength(sortval); break;
     case REDIS_SET: vectorlen =  dictSize((dict*)sortval->ptr); break;
     case REDIS_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->dict); break;
     default: vectorlen = 0; redisPanic("Bad SORT type"); /* Avoid GCC warning */
@@ -7416,15 +7416,15 @@ static void sortCommand(redisClient *c) {
     j = 0;
 
     if (sortval->type == REDIS_LIST) {
-        lIterator *li = lInitIterator(sortval,0,REDIS_TAIL);
-        lEntry entry;
-        while(lNext(li,&entry)) {
-            vector[j].obj = lGet(&entry);
+        listTypeIterator *li = listTypeInitIterator(sortval,0,REDIS_TAIL);
+        listTypeEntry entry;
+        while(listTypeNext(li,&entry)) {
+            vector[j].obj = listTypeGet(&entry);
             vector[j].u.score = 0;
             vector[j].u.cmpobj = NULL;
             j++;
         }
-        lReleaseIterator(li);
+        listTypeReleaseIterator(li);
     } else {
         dict *set;
         dictIterator *di;
@@ -7542,7 +7542,7 @@ static void sortCommand(redisClient *c) {
             listIter li;
 
             if (!getop) {
-                lPush(sobj,vector[j].obj,REDIS_TAIL);
+                listTypePush(sobj,vector[j].obj,REDIS_TAIL);
             } else {
                 listRewind(operations,&li);
                 while((ln = listNext(&li))) {
@@ -7553,10 +7553,10 @@ static void sortCommand(redisClient *c) {
                     if (sop->type == REDIS_SORT_GET) {
                         if (!val) val = createStringObject("",0);
 
-                        /* lPush does an incrRefCount, so we should take care
+                        /* listTypePush does an incrRefCount, so we should take care
                          * care of the incremented refcount caused by either
                          * lookupKeyByPattern or createStringObject("",0) */
-                        lPush(sobj,val,REDIS_TAIL);
+                        listTypePush(sobj,val,REDIS_TAIL);
                         decrRefCount(val);
                     } else {
                         /* always fails */
@@ -11075,14 +11075,14 @@ static void computeDatasetDigest(unsigned char *final) {
             if (o->type == REDIS_STRING) {
                 mixObjectDigest(digest,o);
             } else if (o->type == REDIS_LIST) {
-                lIterator *li = lInitIterator(o,0,REDIS_TAIL);
-                lEntry entry;
-                while(lNext(li,&entry)) {
-                    robj *eleobj = lGet(&entry);
+                listTypeIterator *li = listTypeInitIterator(o,0,REDIS_TAIL);
+                listTypeEntry entry;
+                while(listTypeNext(li,&entry)) {
+                    robj *eleobj = listTypeGet(&entry);
                     mixObjectDigest(digest,eleobj);
                     decrRefCount(eleobj);
                 }
-                lReleaseIterator(li);
+                listTypeReleaseIterator(li);
             } else if (o->type == REDIS_SET) {
                 dict *set = o->ptr;
                 dictIterator *di = dictGetIterator(set);

From dda20542abcccac04a7f630695e6a30304f7dcf8 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pcnoordhuis@gmail.com>
Date: Fri, 4 Jun 2010 11:32:33 +0200
Subject: [PATCH 66/66] safety assert in listTypeNext

---
 redis.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/redis.c b/redis.c
index 30f05a63..1882d742 100644
--- a/redis.c
+++ b/redis.c
@@ -5029,6 +5029,9 @@ static void listTypeReleaseIterator(listTypeIterator *li) {
  * and advances the position of the iterator. Returns 1 when the current
  * entry is in fact an entry, 0 otherwise. */
 static int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
+    /* Protect from converting when iterating */
+    redisAssert(li->subject->encoding == li->encoding);
+
     entry->li = li;
     if (li->encoding == REDIS_ENCODING_ZIPLIST) {
         entry->zi = li->zi;