Sentinel test: basic failover tested. Framework improvements.

tests/sentinel-tests/00-base.tcl

@@ -4,16 +4,19 @@ test "Sentinels aren't monitoring any master" {
     }
 }
 
-test "Create a master-slaves cluster of 3 instances" {
+set redis_slaves 4
-    create_redis_master_slave_cluster 3
+test "Create a master-slaves cluster of [expr $redis_slaves+1] instances" {
+    create_redis_master_slave_cluster [expr {$redis_slaves+1}]
 }
+set master_id 0
 
 test "Sentinels can start monitoring a master" {
     set sentinels [llength $::sentinel_instances]
     set quorum [expr {$sentinels/2+1}]
     foreach_sentinel_id id {
-        S $id SENTINEL MONITOR mymaster [get_instance_attrib redis 0 host] \
+        S $id SENTINEL MONITOR mymaster \
-              [get_instance_attrib redis 0 port] $quorum
+              [get_instance_attrib redis $master_id host] \
+              [get_instance_attrib redis $master_id port] $quorum
     }
     foreach_sentinel_id id {
         assert {[S $id sentinel master mymaster] ne {}}
@@ -30,3 +33,42 @@ test "Sentinels are able to auto-discover other sentinels" {
         }
     }
 }
+
+test "Sentinels are able to auto-discover slaves" {
+    foreach_sentinel_id id {
+        wait_for_condition 100 50 {
+            [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves
+        } else {
+            fail "At least some sentinel can't detect some slave"
+        }
+    }
+}
+
+test "Can change master parameters via SENTINEL SET" {
+    foreach_sentinel_id id {
+        S $id SENTINEL SET mymaster down-after-milliseconds 2000
+    }
+    foreach_sentinel_id id {
+        assert {[dict get [S $id sentinel master mymaster] down-after-milliseconds] == 2000}
+    }
+}
+
+test "Basic failover works if the master is down" {
+    set old_port [RI $master_id tcp_port]
+    set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
+    assert {[lindex $addr 1] == $old_port}
+    R $master_id debug sleep 5
+    foreach_sentinel_id id {
+        wait_for_condition 100 50 {
+            [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port
+        } else {
+            fail "At least one Sentinel did not received failover info"
+        }
+    }
+    set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster]
+    set master_id [get_instance_id_by_port redis [lindex $addr 1]]
+}
+
+test "New master [join $addr {:}] role matches" {
+    assert {[RI $master_id role] eq {master}}
+}

tests/sentinel.tcl

@@ -116,7 +116,7 @@ proc test {descr code} {
 proc run_tests {} {
     set tests [lsort [glob ../sentinel-tests/*]]
     foreach test $tests {
-        puts [colorstr green "### [lindex [file split $test] end]"]
+        puts [colorstr yellow "Testing unit: [lindex [file split $test] end]"]
         source $test
     }
 }
@@ -159,18 +159,26 @@ proc RI {n field} {
 }
 
 # Iterate over IDs of sentinel or redis instances.
-proc foreach_sentinel_id {idvar code} {
+proc foreach_instance_id {instances idvar code} {
     upvar 1 $idvar id
-    for {set id 0} {$id < [llength $::sentinel_instances]} {incr id} {
+    for {set id 0} {$id < [llength $instances]} {incr id} {
-        uplevel 1 $code
+        set errcode [catch {uplevel 1 $code} result]
+        if {$errcode == 1} {
+            error $result $::errorInfo $::errorCode
+        } elseif {$errcode != 0} {
+            return -code $errcode $result
+        }
     }
 }
 
+proc foreach_sentinel_id {idvar code} {
+    set errcode [catch {uplevel 1 [list foreach_instance_id $::sentinel_instances $idvar $code]} result]
+    return -code $errcode $result
+}
+
 proc foreach_redis_id {idvar code} {
-    upvar 1 $idvar id
+    set errcode [catch {uplevel 1 [list foreach_instance_id $::redis_instances $idvar $code]} result]
-    for {set id 0} {$id < [llength $::redis_instances]} {incr id} {
+    return -code $errcode $result
-        uplevel 1 $code
-    }
 }
 
 # Get the specific attribute of the specified instance type, id.
@@ -203,6 +211,15 @@ proc create_redis_master_slave_cluster n {
     }
 }
 
+proc get_instance_id_by_port {type port} {
+    foreach_${type}_id id {
+        if {[get_instance_attrib $type $id port] == $port} {
+            return $id
+        }
+    }
+    fail "Instance $type port $port not found."
+}
+
 if {[catch main e]} {
     puts $::errorInfo
     cleanup