Testing/understanding various node.js redis-sentinel implementations

results-docusign-sentinel.log

Using test hash test-sentinel-662038
sentinel client [no role] [no port] got 'end' { '0': undefined, '1': undefined, '2': undefined, '3': undefined }
sentinel client [no role] [no port] got 'connect' {}
sentinel client master 5379 got 'ready' {}
---- knock 1 ----
1 Set? true
1 check integrity: all good
---- knock 2 ----
2 Set? true
2 check integrity: all good
---- knock 3 ----
3 Set? true
3 check integrity: all good
---- knock 4 ----
4 Set? true
4 check integrity: all good
---- knock 5 ----
5 Set? true
5 check integrity: all good
**** Killing master on 20992 ****
... should failover now!
sentinel client master 5379 got 'end' { '0': undefined, '1': undefined, '2': undefined, '3': undefined }
error [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED]
sentinel client master 5379 got 'error' { '0': [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED] }
error [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED]
sentinel client master 5379 got 'error' { '0': [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED] }
---- knock 6 ----
error [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED]
sentinel client master 5379 got 'error' { '0': [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED] }
---- knock 7 ----
error [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED]
sentinel client master 5379 got 'error' { '0': [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED] }
---- knock 8 ----
---- knock 9 ----
---- knock 10 ----
error [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED]
sentinel client master 5379 got 'error' { '0': [Error: sentinel-127.0.0.1:8379-mymaster master error: Redis connection to 127.0.0.1:5379 failed - connect ECONNREFUSED] }
---- knock 11 ----
---- knock 12 ----
---- knock 13 ----
---- knock 14 ----
---- knock 15 ----
---- knock 16 ----
---- knock 17 ----
sentinel client master 5379 got 'connect' {}
sentinel client master 5380 got 'ready' {}
6 Set? true
7 Set? true
8 Set? true
9 Set? true
10 Set? true
11 Set? true
12 Set? true
13 Set? true
14 Set? true
15 Set? true
16 Set? true
17 Set? true
6 check integrity: all good
7 check integrity: all good
8 check integrity: all good
9 check integrity: all good
10 check integrity: all good
11 check integrity: all good
12 check integrity: all good
13 check integrity: all good
14 check integrity: all good
15 check integrity: all good
16 check integrity: all good
17 check integrity: all good
---- knock 18 ----
18 Set? true
18 check integrity: all good
---- knock 19 ----
19 Set? true
19 check integrity: all good

results-jamessharp-sentinel.log

Using test hash test-sentinel-63989
master [no role] [no port] got 'connect' {}
master [no role] [no port] got 'drain' {}
master master 5379 got 'drain' {}
master master 5379 got 'ready' {}
---- knock 1 ----
master master 5379 got 'drain' {}
master master 5379 got 'drain' {}
1 Set? true
master master 5379 got 'drain' {}
1 check integrity: all good
---- knock 2 ----
master master 5379 got 'drain' {}
master master 5379 got 'drain' {}
2 Set? true
master master 5379 got 'drain' {}
2 check integrity: all good
---- knock 3 ----
master master 5379 got 'drain' {}
master master 5379 got 'drain' {}
3 Set? true
master master 5379 got 'drain' {}
3 check integrity: all good
---- knock 4 ----
master master 5379 got 'drain' {}
master master 5379 got 'drain' {}
4 Set? true
master master 5379 got 'drain' {}
4 check integrity: all good
---- knock 5 ----
master master 5379 got 'drain' {}
master master 5379 got 'drain' {}
5 Set? true
master master 5379 got 'drain' {}
5 check integrity: all good
**** Killing master on 21055 ****
... should failover now!
master master 5379 got 'end' {}
master master 5379 got 'reconnecting' { '0': { delay: 255, attempt: 2 } }
master master 5379 got 'reconnecting' { '0': { delay: 433, attempt: 3 } }
master master 5379 got 'reconnecting' { '0': { delay: 736, attempt: 4 } }
---- knock 6 ----
6 Set? false
master master 5379 got 'reconnecting' { '0': { delay: 1251, attempt: 5 } }
---- knock 7 ----
7 Set? false
master master 5379 got 'reconnecting' { '0': { delay: 2126, attempt: 6 } }
---- knock 8 ----
---- knock 9 ----
8 Set? false
9 Set? false
master master 5379 got 'reconnecting' { '0': { delay: 3614, attempt: 7 } }
---- knock 10 ----
---- knock 11 ----
---- knock 12 ----
---- knock 13 ----
10 Set? false
11 Set? false
12 Set? false
13 Set? false
master master 5379 got 'reconnecting' { '0': { delay: 6143, attempt: 8 } }
---- knock 14 ----
---- knock 15 ----
---- knock 16 ----
---- knock 17 ----
---- knock 18 ----
---- knock 19 ----
14 Set? false
15 Set? false
16 Set? false
17 Set? false
18 Set? false
19 Set? false
master master 5379 got 'reconnecting' { '0': { delay: 10443, attempt: 9 } }
---- knock 20 ----
---- knock 21 ----
---- knock 22 ----
---- knock 23 ----
---- knock 24 ----
---- knock 25 ----
---- knock 26 ----
---- knock 27 ----
---- knock 28 ----
---- knock 29 ----
---- knock 30 ----
20 Set? false
21 Set? false
22 Set? false
23 Set? false
24 Set? false
25 Set? false
26 Set? false
27 Set? false
28 Set? false
29 Set? false
30 Set? false
master master 5379 got 'reconnecting' { '0': { delay: 17753, attempt: 10 } }
---- knock 31 ----
---- knock 32 ----
---- knock 33 ----
---- knock 34 ----
---- knock 35 ----
---- knock 36 ----
---- knock 37 ----
---- knock 38 ----
---- knock 39 ----
---- knock 40 ----
---- knock 41 ----
---- knock 42 ----
---- knock 43 ----
---- knock 44 ----
---- knock 45 ----
---- knock 46 ----
---- knock 47 ----
master master 5379 got 'connect' {}
master master 5379 got 'drain' {}
master master 5380 got 'drain' {}
master master 5380 got 'ready' {}
master master 5380 got 'drain' {}
master master 5380 got 'drain' {}
31 Set? true
32 Set? true
33 Set? true
34 Set? true
35 Set? true
36 Set? true
37 Set? true
38 Set? true
39 Set? true
40 Set? true
41 Set? true
42 Set? true
43 Set? true
44 Set? true
45 Set? true
46 Set? true
47 Set? true
master master 5380 got 'drain' {}
31 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
32 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
33 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
34 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
35 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
36 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
37 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
38 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
39 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
40 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
41 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
42 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
43 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
44 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
45 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
46 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
47 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
---- knock 48 ----
master master 5380 got 'drain' {}
master master 5380 got 'drain' {}
48 Set? true
master master 5380 got 'drain' {}
48 check integrity: missing 6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30

test-docusign-sentinel.coffee

###
test DocuSignDev's redis-sentinel implementation

to use this,
  - git clone https://github.com/DocuSignDev/node_redis.git ./node_modules/redis
  - start a redis master on 5379
  - start a redis slave on 5380
  - start a redis sentinel talking to the master on 8379

  - run this (coffee ./[filename])
###

redis = require 'redis'
util = require 'util'

sentinelClient = redis.createClient 8379, '127.0.0.1', { sentinel:true }

for event in ['error', 'reconnecting', 'end', 'drain', 'ready', 'connect']
  do (event)->
    try
      sentinelClient.on event, ->
        console.log "sentinel client " +
          (sentinelClient?.server_info?.role ? '[no role]') + " " +
          (sentinelClient?.server_info?.tcp_port ? '[no port]') +
          " got '#{event}'", util.inspect(arguments,false,1)
    catch e
      console.error "can't listen to #{event}"


# should maintain persistent connection to master ...?
require('./test-failover')(sentinelClient)

test-failover.coffee

child_process = require 'child_process'

handleError = (error)->
  console.error "Error", error


module.exports = (client)->
  hashKey = "test-sentinel-" + Math.round( Math.random() * 1000000 )
  console.log "Using test hash", hashKey

  # put an incremental n:n entry in a hash,
  # to see if we've missed anything during the failover.

  runCount = 0
  doIO = ->
    runCount++
    
    do (runCount)->
      console.log "---- knock #{runCount} ----"

      n = runCount.toString()
      client.hset hashKey, n, n, (error)->
        if error then handleError error

        client.hget hashKey, n, (error, val)->
          if error then handleError error
          console.log "#{runCount} Set?", (val is n).toString()

          # count how many entries we're missing
          client.hgetall hashKey, (error, hash)->
            missing = []
            if error then handleError error
            else unless typeof hash is 'object' then handleError new Error "Missing hash #{hashKey}"
            else
              for i in [1..runCount]
                if not hash[ i.toString() ]? then missing.push i
              console.log "#{runCount} check integrity: " + (if missing.length is 0 then "all good" else "missing " + missing.join(','))


  # continously hit the current master
  setInterval doIO, 1000


  # kill the master after 5 seconds
  setTimeout ->
    # find the PID of the master on 5379
    child_process.exec 'ps -ef | grep "redis-server" | grep "port 5379" | grep -v "grep" | awk \'{print $2}\''
    , (error, stdout, stderr)->
      # console.log "finding pid", console.log(arguments)
      pid = stdout.trim()
      if error or (stderr.trim() isnt '') or (not pid.match /^[0-9]*$/)
        console.error "Missing master pid, can't force failover", [ error, stderr, pid ]
        process.exit 1
      else
        # kill the master
        console.log "**** Killing master on #{pid} ****"
        child_process.exec "kill #{pid}", (error, stdout, stderr)->
          if error or (stderr.trim() isnt '')
            console.error "Failed to kill master pid", [ error, stderr ]
          else
            console.log "... should failover now!"
  , 5000

test-jamessharp-sentinel.coffee

###
test jamessharp's https://github.com/ortoo/node-redis-sentinel

to use this,
  - npm install redis-sentinel
  - start a redis master on 5379
  - start a redis slave on 5380
  - start a redis sentinel talking to the master on 8379

  - run this (coffee ./[filename])
###

sentinel = require 'redis-sentinel'
util = require 'util'

endpoints = [
  # sentinel
  { port: 8379, host: '127.0.0.1' }

  # master
  { host: '127.0.0.1', port: 5379 }

  # slave
  { host: '127.0.0.1', port: 5380 }
]

clients =
  master: sentinel.createClient(endpoints, 'mymaster', {role: 'master'})
  # slave: sentinel.createClient(endpoints, 'mymaster', {role: 'slave'})
  # sentinel: sentinel.createClient(endpoints, {role: 'sentinel'})

# console.log 'clients', util.inspect(clients,false,2)

# ?? sentinel is not an eventemitter / real client?

for clientName, client of clients

  for event in ['error', 'reconnecting', 'end', 'drain', 'ready', 'connect']
    do (clientName, client, event)->
      try
        client.on event, ->
          console.log clientName + " " +
            (client?.server_info?.role ? '[no role]') + " " +
            (client?.server_info?.tcp_port ? '[no port]') +
            " got '#{event}'", util.inspect(arguments,false,1)
      catch e
        console.error "#{clientName} can't listen to #{event}"


# should maintain persistent connection to master ...?
require('./test-failover')(clients.master)

test-separated-sentinel-client.coffee

###
test DocuSignDev's redis-sentinel implementation,
separated from node_redis fork (https://github.com/DocuSignDev/node_redis.git)
into its own module (https://github.com/DocuSignDev/node-redis-sentinel-client)

to use this,
  - git clone https://github.com/DocuSignDev/node-redis-sentinel-client.git ./node_modules/redis-sentinel-client
    [@todo make it an npm module!]
  - start a redis master on 5379
  - start a redis slave on 5380
  - start a redis sentinel talking to the master on 8379

  - run this (coffee ./[filename])
###

sentinel = require 'redis-sentinel-client'
util = require 'util'

sentinelClient = sentinel.createClient 8379, '127.0.0.1'

for event in ['error', 'reconnecting', 'end', 'drain', 'ready', 'connect']
  do (event)->
    try
      sentinelClient.on event, ->
        console.log "sentinel client " +
          (sentinelClient?.server_info?.role ? '[no role]') + " " +
          (sentinelClient?.server_info?.tcp_port ? '[no port]') +
          " got '#{event}'", util.inspect(arguments,false,1)
    catch e
      console.error "can't listen to #{event}"


# should maintain persistent connection to master ...?
require('./test-failover')(sentinelClient)