Merge pull request #5084 from chendq8/pending-querybuf

limit the size of pending-querybuf in masterclient

src/server.c

@@ -859,6 +859,17 @@ int clientsCronResizeQueryBuffer(client *c) {
     /* Reset the peak again to capture the peak memory usage in the next
      * cycle. */
     c->querybuf_peak = 0;
+
+    if (c->flags & CLIENT_MASTER) {
+        /* There are two conditions to resize the pending query buffer:
+         * 1) Pending Query buffer is > LIMIT_PENDING_QUERYBUF.
+         * 2) used length is smaller than pending_querybuf_size/2 */
+        size_t pending_querybuf_size = sdsAllocSize(c->pending_querybuf);
+        if(pending_querybuf_size > LIMIT_PENDING_QUERYBUF &&
+                sdslen(c->pending_querybuf) < (pending_querybuf_size>>1)){
+            c->pending_querybuf = sdsRemoveFreeSpace(c->pending_querybuf);
+        }
+    }
     return 0;
 }
 

src/server.h

@@ -186,6 +186,8 @@ typedef long long mstime_t; /* millisecond time type. */
 #define LONG_STR_SIZE      21          /* Bytes needed for long -> str + '\0' */
 #define REDIS_AUTOSYNC_BYTES (1024*1024*32) /* fdatasync every 32MB */
 
+#define LIMIT_PENDING_QUERYBUF (4*1024*1024) /* 4mb */
+
 /* When configuring the server eventloop, we setup it so that the total number
  * of file descriptors we can handle are server.maxclients + RESERVED_FDS +
  * a few more to stay safe. Since RESERVED_FDS defaults to 32, we add 96

tests/test_helper.tcl

@@ -61,6 +61,7 @@ set ::all_tests {
     unit/hyperloglog
     unit/lazyfree
     unit/wait
+    unit/pendingquerybuf
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0

tests/unit/pendingquerybuf.tcl

@@ -0,0 +1,35 @@
+proc info_memory {r property} {
+    if {[regexp "\r\n$property:(.*?)\r\n" [{*}$r info memory] _ value]} {
+        set _ $value
+    }
+}
+
+proc prepare_value {size} {
+    set _v "c" 
+    for {set i 1} {$i < $size} {incr i} {
+        append _v 0   
+    }
+    return $_v
+}
+
+start_server {tags {"wait"}} {
+start_server {} {
+    set slave [srv 0 client]
+    set slave_host [srv 0 host]
+    set slave_port [srv 0 port]
+    set master [srv -1 client]
+    set master_host [srv -1 host]
+    set master_port [srv -1 port]
+
+    test "pending querybuf: check size of pending_querybuf after set a big value" {
+        $slave slaveof $master_host $master_port
+        set _v [prepare_value [expr 32*1024*1024]]
+        $master set key $_v 
+        after 2000
+        set m_usedmemory [info_memory $master used_memory]
+        set s_usedmemory [info_memory $slave used_memory]
+        if { $s_usedmemory > $m_usedmemory + 10*1024*1024 } {
+            fail "the used_memory of slave is too larger than master.Master:$m_usedmemory Slave:$s_usedmemory"
+        }
+    }  
+}}