From 8e553ec67cdc1d2720b2c37d5ccbacf1924bd3a9 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 18 Feb 2014 16:31:52 +0100 Subject: [PATCH] Sentinel test: basic failover tested. Framework improvements. --- tests/sentinel-tests/00-base.tcl | 50 +++++++++++++++++++++++++++++--- tests/sentinel.tcl | 33 ++++++++++++++++----- 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/tests/sentinel-tests/00-base.tcl b/tests/sentinel-tests/00-base.tcl index 2bb80e75..0b70cc26 100644 --- a/tests/sentinel-tests/00-base.tcl +++ b/tests/sentinel-tests/00-base.tcl @@ -4,16 +4,19 @@ test "Sentinels aren't monitoring any master" { } } -test "Create a master-slaves cluster of 3 instances" { - create_redis_master_slave_cluster 3 +set redis_slaves 4 +test "Create a master-slaves cluster of [expr $redis_slaves+1] instances" { + create_redis_master_slave_cluster [expr {$redis_slaves+1}] } +set master_id 0 test "Sentinels can start monitoring a master" { set sentinels [llength $::sentinel_instances] set quorum [expr {$sentinels/2+1}] foreach_sentinel_id id { - S $id SENTINEL MONITOR mymaster [get_instance_attrib redis 0 host] \ - [get_instance_attrib redis 0 port] $quorum + S $id SENTINEL MONITOR mymaster \ + [get_instance_attrib redis $master_id host] \ + [get_instance_attrib redis $master_id port] $quorum } foreach_sentinel_id id { assert {[S $id sentinel master mymaster] ne {}} @@ -30,3 +33,42 @@ test "Sentinels are able to auto-discover other sentinels" { } } } + +test "Sentinels are able to auto-discover slaves" { + foreach_sentinel_id id { + wait_for_condition 100 50 { + [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves + } else { + fail "At least some sentinel can't detect some slave" + } + } +} + +test "Can change master parameters via SENTINEL SET" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster down-after-milliseconds 2000 + } + foreach_sentinel_id id { + assert {[dict get [S $id sentinel master mymaster] down-after-milliseconds] == 2000} + } +} + +test "Basic failover works if the master is down" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + R $master_id debug sleep 5 + foreach_sentinel_id id { + wait_for_condition 100 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} diff --git a/tests/sentinel.tcl b/tests/sentinel.tcl index 38243abd..0001a15f 100644 --- a/tests/sentinel.tcl +++ b/tests/sentinel.tcl @@ -116,7 +116,7 @@ proc test {descr code} { proc run_tests {} { set tests [lsort [glob ../sentinel-tests/*]] foreach test $tests { - puts [colorstr green "### [lindex [file split $test] end]"] + puts [colorstr yellow "Testing unit: [lindex [file split $test] end]"] source $test } } @@ -159,18 +159,26 @@ proc RI {n field} { } # Iterate over IDs of sentinel or redis instances. -proc foreach_sentinel_id {idvar code} { +proc foreach_instance_id {instances idvar code} { upvar 1 $idvar id - for {set id 0} {$id < [llength $::sentinel_instances]} {incr id} { - uplevel 1 $code + for {set id 0} {$id < [llength $instances]} {incr id} { + set errcode [catch {uplevel 1 $code} result] + if {$errcode == 1} { + error $result $::errorInfo $::errorCode + } elseif {$errcode != 0} { + return -code $errcode $result + } } } +proc foreach_sentinel_id {idvar code} { + set errcode [catch {uplevel 1 [list foreach_instance_id $::sentinel_instances $idvar $code]} result] + return -code $errcode $result +} + proc foreach_redis_id {idvar code} { - upvar 1 $idvar id - for {set id 0} {$id < [llength $::redis_instances]} {incr id} { - uplevel 1 $code - } + set errcode [catch {uplevel 1 [list foreach_instance_id $::redis_instances $idvar $code]} result] + return -code $errcode $result } # Get the specific attribute of the specified instance type, id. @@ -203,6 +211,15 @@ proc create_redis_master_slave_cluster n { } } +proc get_instance_id_by_port {type port} { + foreach_${type}_id id { + if {[get_instance_attrib $type $id port] == $port} { + return $id + } + } + fail "Instance $type port $port not found." +} + if {[catch main e]} { puts $::errorInfo cleanup