From 0609380603bb235e63c97049bd8217856c062edb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Mar 2014 11:18:48 +0100 Subject: [PATCH] HyperLogLog: initial sketch of registers access. --- src/hyperloglog.c | 175 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 src/hyperloglog.c diff --git a/src/hyperloglog.c b/src/hyperloglog.c new file mode 100644 index 00000000..01d43b01 --- /dev/null +++ b/src/hyperloglog.c @@ -0,0 +1,175 @@ +/* hyperloglog.c - Redis HyperLogLog probabilistic cardinality approximation. + * This file implements the algorithm and the exported Redis commands. + * + * Copyright (c) 2014, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "redis.h" + +/* The Redis HyperLogLog implementation is based on the following ideas: + * + * * The use of a 64 bit hash function as proposed in [1], in order to don't + * limited to cardinalities up to 10^9, at the cost of just 1 additional + * bit per register. + * * The use of 16384 6-bit registers for a great level of accuracy, using + * a total of 12k per key. + * * The use of the Redis string data type. No new type is introduced. + * * No attempt is made to compress the data structure as in [1]. Also the + * algorithm used is the original HyperLogLog Algorithm as in [2], with + * the only difference that a 64 bit hash function is used, so no correction + * is performed for values near 2^32 as in [1]. + * + * [1] Heule, Nunkesser, Hall: HyperLogLog in Practice: Algorithmic + * Engineering of a State of The Art Cardinality Estimation Algorithm. + * + * [2] P. Flajolet, Éric Fusy, O. Gandouet, and F. Meunier. Hyperloglog: The + * analysis of a near-optimal cardinality estimation algorithm. + */ + +#define REDIS_HLL_REGISTERS 16384 +#define REDIS_HLL_BITS 6 +#define REDIS_HLL_SIZE ((REDIS_HLL_REGISTERS*REDIS_HLL_BITS+7)/8) + +/* =========================== Low level bit macros ========================= */ + +/* We need to get and set 6 bit counters in an array of 8 bit bytes. + * We use macros to make sure the code is inlined since speed is critical + * especially in order to compute the approximated cardinality in + * HLLCOUNT where we need to access all the registers at once. + * + * +--------+--------+--------+------// + * |00000011|11112222|22333333|444444 + * +--------+--------+--------+------// + * + * Assuming we want to access counter at zero based index 'pos' = 2. + * (In the example it is "222222") + * + * The first byte "b" containing our data is: + * b = 6 * pos / 8 -> 1 + * + * +--------+ + * |11112222| <- Our byte at "b" + * +--------+ + * + * The amount of left shifting "ls" in the first byte is: + * ls = 6 * pos & 7 -> 4 + * + * +--------+ + * |2222 | <- Left shift 4 pos. + * +--------+ + * + * To add the bits in the next byte b+1, we need to right shift them right of + * "rs" bits positions before xoring it to our current value in the first byte + * (after the left shift): + * rs = 8 - ls -> 4 + * + * +--------+ + * | 2233| <- Byte "b+1" right shifted 4 pos. + * +--------+ + * + * Now we can just bitwise-OR the two bytes and mask for 2^6-1 in order to + * clear bits 6 and 7 if they are set, that are not part of our 6 bit unsigned + * integer. + * + * ------------------------------------------------------------------------- + * + * Setting the register is a bit more complex, let's assume that 'val' + * is the value we want to set, already in the right range. + * + * We need two steps, in one we need to clear the bits, and in the other + * we need to bitwise-OR the new bits. + * + * This time let's try with 'pos' = 1, so our first byte at 'b' is 0, + * "ls" is 6, and "rs" is 2. + * + * +--------+ + * |00000011| <- Our initial byte at "b" + * +--------+ + * + * We store at "mask" the value 255, consisting of a byte with all bits + * set to 1. We left-shift it of "rs" bits to the left. + * + * +--------+ + * |11111100| <- "mask" after the left-shift of 'rs' bits. + * +--------+ + * + * Now we can bitwise-AND the byte at "b" with the mask, and bitwise-OR + * it with "val" right-shifted of "ls" to set the new bits. + * + * Now let's focus on the next byte b+1: + * + * +--------+ + * |11112222| <- byte at b+1 + * +--------+ + * + * To build the AND mask for the next byte b+1 we left shift it by "rs" + * amount of bits a byte with value 2^6-1. Later we negate (bitwise-not) it. + * + * +--------+ + * |11111100| <- "filter" set at 2&6-1 + * |11110000| <- "filter" after the left shift of "rs" bits. + * |00001111| <- "filter" after bitwise not. + * +--------+ + * + * Now we can mask it with b+1 to clear the old bits, and bitwise-OR + * with "val" left-shifted by "rs" bits to set the new value. + */ + +/* Note: if we access the last counter, we will also access the b+1 byte + * that is out of the array, but sds strings always have an implicit null + * term, so the byte exists, and we can skip the conditional (or the need + * to allocate 1 byte more explicitly). */ + +/* Store the value of the register at position 'regnum' into variable 'target'. + * 'p' is an array of unsigned bytes. */ +#define HLL_GET_REGISTER(target,p,regnum) do { \ + int _byte = regnum*REDIS_HLL_BITS/8; \ + int _leftshift = regnum*REDIS_HLL_BITS&7; \ + int _rightshift = 8 - _leftshift; \ + target = ((p[_byte] << _leftshift) | \ + (p[_byte+1] >> _rightshift)) & \ + ((1<> _leftshift; \ + p[byte+1] &= ~(m2 << _rightshift); \ + p[byte+1] |= val << _rightshift; \ +} while(0) + +/* ========================= HyperLogLog algorithm ========================= */ + +/* ========================== HyperLogLog commands ========================== */ +