HyperLogLog: initial sketch of registers access.

src/hyperloglog.c

@@ -0,0 +1,175 @@
+/* hyperloglog.c - Redis HyperLogLog probabilistic cardinality approximation.
+ * This file implements the algorithm and the exported Redis commands.
+ *
+ * Copyright (c) 2014, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "redis.h"
+
+/* The Redis HyperLogLog implementation is based on the following ideas:
+ *
+ * * The use of a 64 bit hash function as proposed in [1], in order to don't
+ *   limited to cardinalities up to 10^9, at the cost of just 1 additional
+ *   bit per register.
+ * * The use of 16384 6-bit registers for a great level of accuracy, using
+ *   a total of 12k per key.
+ * * The use of the Redis string data type. No new type is introduced.
+ * * No attempt is made to compress the data structure as in [1]. Also the
+ *   algorithm used is the original HyperLogLog Algorithm as in [2], with
+ *   the only difference that a 64 bit hash function is used, so no correction
+ *   is performed for values near 2^32 as in [1].
+ *
+ * [1] Heule, Nunkesser, Hall: HyperLogLog in Practice: Algorithmic
+ *     Engineering of a State of The Art Cardinality Estimation Algorithm.
+ *
+ * [2] P. Flajolet, Éric Fusy, O. Gandouet, and F. Meunier. Hyperloglog: The
+ *     analysis of a near-optimal cardinality estimation algorithm.
+ */
+
+#define REDIS_HLL_REGISTERS 16384
+#define REDIS_HLL_BITS 6
+#define REDIS_HLL_SIZE ((REDIS_HLL_REGISTERS*REDIS_HLL_BITS+7)/8)
+
+/* =========================== Low level bit macros ========================= */
+
+/* We need to get and set 6 bit counters in an array of 8 bit bytes.
+ * We use macros to make sure the code is inlined since speed is critical
+ * especially in order to compute the approximated cardinality in
+ * HLLCOUNT where we need to access all the registers at once.
+ *
+ * +--------+--------+--------+------//
+ * |00000011|11112222|22333333|444444
+ * +--------+--------+--------+------//
+ *
+ * Assuming we want to access counter at zero based index 'pos' = 2.
+ * (In the example it is "222222")
+ *
+ * The first byte "b" containing our data is:
+ *   b = 6 * pos / 8 -> 1
+ *
+ *   +--------+
+ *   |11112222|  <- Our byte at "b"
+ *   +--------+
+ *
+ * The amount of left shifting "ls" in the first byte is:
+ *   ls = 6 * pos & 7 -> 4
+ *
+ *   +--------+
+ *   |2222    |  <- Left shift 4 pos.
+ *   +--------+
+ *
+ * To add the bits in the next byte b+1, we need to right shift them right of
+ * "rs" bits positions before xoring it to our current value in the first byte
+ * (after the left shift):
+ *   rs = 8 - ls -> 4
+ *
+ *   +--------+
+ *   |    2233|  <- Byte "b+1" right shifted 4 pos.
+ *   +--------+
+ *
+ * Now we can just bitwise-OR the two bytes and mask for 2^6-1 in order to
+ * clear bits 6 and 7 if they are set, that are not part of our 6 bit unsigned
+ * integer.
+ *
+ * -------------------------------------------------------------------------
+ *
+ * Setting the register is a bit more complex, let's assume that 'val'
+ * is the value we want to set, already in the right range.
+ *
+ * We need two steps, in one we need to clear the bits, and in the other
+ * we need to bitwise-OR the new bits.
+ *
+ * This time let's try with 'pos' = 1, so our first byte at 'b' is 0,
+ * "ls" is 6, and "rs" is 2.
+ *
+ *   +--------+
+ *   |00000011|  <- Our initial byte at "b"
+ *   +--------+
+ *
+ * We store at "mask" the value 255, consisting of a byte with all bits
+ * set to 1. We left-shift it of "rs" bits to the left.
+ *
+ *   +--------+
+ *   |11111100|  <- "mask" after the left-shift of 'rs' bits.
+ *   +--------+
+ *
+ * Now we can bitwise-AND the byte at "b" with the mask, and bitwise-OR
+ * it with "val" right-shifted of "ls" to set the new bits.
+ *
+ * Now let's focus on the next byte b+1:
+ *
+ *   +--------+
+ *   |11112222| <- byte at b+1
+ *   +--------+
+ *
+ * To build the AND mask for the next byte b+1 we left shift it by "rs"
+ * amount of bits a byte with value 2^6-1. Later we negate (bitwise-not) it.
+ *
+ *   +--------+
+ *   |11111100|  <- "filter" set at 2&6-1
+ *   |11110000|  <- "filter" after the left shift of "rs" bits.
+ *   |00001111|  <- "filter" after bitwise not.
+ *   +--------+
+ *
+ * Now we can mask it with b+1 to clear the old bits, and bitwise-OR
+ * with "val" left-shifted by "rs" bits to set the new value.
+ */
+
+/* Note: if we access the last counter, we will also access the b+1 byte
+ * that is out of the array, but sds strings always have an implicit null
+ * term, so the byte exists, and we can skip the conditional (or the need
+ * to allocate 1 byte more explicitly). */
+
+/* Store the value of the register at position 'regnum' into variable 'target'.
+ * 'p' is an array of unsigned bytes. */
+#define HLL_GET_REGISTER(target,p,regnum) do { \
+    int _byte = regnum*REDIS_HLL_BITS/8; \
+    int _leftshift = regnum*REDIS_HLL_BITS&7; \
+    int _rightshift = 8 - _leftshift; \
+    target = ((p[_byte] << _leftshift) | \
+             (p[_byte+1] >> _rightshift)) & \
+             ((1<<REDIS_HLL_BITS)-1); \
+} while(0)
+
+/* Set the value of the register at position 'regnum' to 'val'.
+ * 'p' is an array of unsigned bytes. */
+#define HLL_SET_REGISTER(val,p,regnum) do { \
+    int _byte = regnum*REDIS_HLL_BITS/8; \
+    int _leftshift = regnum*REDIS_HLL_BITS&7; \
+    int _rightshift = 8 - _leftshift; \
+    unsigned int m1 = 255, m2 = (1<<REDIS_HLL_BITS)-1; \
+    p[byte] &= m1 << _rightshift; \
+    p[byte] |= val >> _leftshift; \
+    p[byte+1] &= ~(m2 << _rightshift); \
+    p[byte+1] |= val << _rightshift; \
+} while(0)
+
+/* ========================= HyperLogLog algorithm  ========================= */
+
+/* ========================== HyperLogLog commands ========================== */
+