raft_ha_standby.sh

#!/bin/bash

set -aex

# Kill any Vault process that is running
for process in $(lsof -Pn | grep ':8200' | awk '{print $1}')
do
pkill -9 $process || true
done

cd dev

# Enable loopback addresses for different Vault nodes
sudo ifconfig lo0 alias 127.0.0.2
sudo ifconfig lo0 alias 127.0.0.3
sudo ifconfig lo0 alias 127.0.0.4

vault_2() {
    VAULT_ADDR=http://127.0.0.2:8200 vault $@
}

vault_3() {
    VAULT_ADDR=http://127.0.0.3:8200 vault $@
}

vault_4() {
    VAULT_ADDR=http://127.0.0.4:8200 vault $@
}

# Run a Vault node that acts as auto-unsealer for another Vault cluster
tee inmem.hcl <<EOF
storage "inmem" {}
listener "tcp" {
  address = "127.0.0.1:8200"
  tls_disable = true
}
api_addr = "http://127.0.0.1:8200"
disable_mlock = true
EOF
vault server -log-level=trace -config inmem.hcl > /tmp/inmem.log 2>&1 &
sleep 5s
initResult=$(vault operator init -format json -key-shares 1 -key-threshold 1)
unsealKey=$(echo -n $initResult | jq -r '.unseal_keys_b64[0]')
rootToken=$(echo -n $initResult | jq -r '.root_token')
echo -n $unsealKey > unsealKey
echo -n $rootToken > rootToken
vault operator unseal `cat unsealKey`
vault login $rootToken
vault secrets enable transit

# Bring up three Raft nodes
tee raft1.hcl <<EOF
storage "raft" {
  path    = "/tmp/raft1"
  node_id = "node1"
  retry_join{
    leader_api_addr = "http://127.0.0.2:8200"
  }
  retry_join {
    leader_api_addr = "http://127.0.0.3:8200"
  }
  retry_join {
    leader_api_addr = "http://127.0.0.4:8200"
  }
}
listener "tcp" {
  address = "127.0.0.2:8200"
  tls_disable = true
}
seal "transit" {
  address            = "http://127.0.0.1:8200"
  token              = "$rootToken"
  disable_renewal    = "false"

  // Key configuration
  key_name           = "unseal_key"
  mount_path         = "transit/"
}
disable_mlock = true
api_addr="https://127.0.0.2:8200"
cluster_addr="https://127.0.0.2:8201"
EOF

tee raft2.hcl <<EOF
storage "raft" {
  path    = "/tmp/raft2"
  node_id = "node2"
  retry_join{
    leader_api_addr = "http://127.0.0.2:8200"
  }
  retry_join {
    leader_api_addr = "http://127.0.0.3:8200"
  }
  retry_join {
    leader_api_addr = "http://127.0.0.4:8200"
  }
}
listener "tcp" {
  address = "127.0.0.3:8200"
  tls_disable = true
}
seal "transit" {
  address            = "http://127.0.0.1:8200"
  token              = "$rootToken"
  disable_renewal    = "false"

  // Key configuration
  key_name           = "unseal_key"
  mount_path         = "transit/"
}
disable_mlock = true
api_addr="https://127.0.0.3:8200"
cluster_addr="https://127.0.0.3:8201"
EOF

tee raft3.hcl <<EOF
storage "raft" {
  path    = "/tmp/raft3"
  node_id = "node3"
  retry_join{
    leader_api_addr = "http://127.0.0.2:8200"
  }
  retry_join {
    leader_api_addr = "http://127.0.0.3:8200"
  }
  retry_join {
    leader_api_addr = "http://127.0.0.4:8200"
  }
}
listener "tcp" {
  address = "127.0.0.4:8200"
  tls_disable = true
}
seal "transit" {
  address            = "http://127.0.0.1:8200"
  token              = "$rootToken"
  disable_renewal    = "false"

  // Key configuration
  key_name           = "unseal_key"
  mount_path         = "transit/"
}
disable_mlock = true
api_addr="https://127.0.0.4:8200"
cluster_addr="https://127.0.0.4:8201"
EOF
rm -rf /tmp/raft1/
rm -rf /tmp/raft2/
rm -rf /tmp/raft3/
mkdir /tmp/raft1
mkdir /tmp/raft2
mkdir /tmp/raft3

vault server -log-level=trace -config raft1.hcl > /tmp/raft1.log 2>&1 &
vault server -log-level=trace -config raft2.hcl > /tmp/raft2.log 2>&1 &
vault server -log-level=trace -config raft3.hcl > /tmp/raft3.log 2>&1 &

sleep 10s

# Initialize node1 and let all the others automatically join the raft cluster
initResult2=$(vault_2 operator init -format=json -key-shares 1 -key-threshold 1)
rootToken2=$(echo $initResult2 | jq -r .root_token)
sleep 10s
vault_2 login $rootToken2
vault_2 operator raft list-peers -format json

# See if one of the nodes is active and other two are standby and that all the nodes are unsealed
vault_2 status || true
vault_3 status || true
vault_4 status || true

# Remove the second node
vault_2 operator raft remove-peer node2
vault_2 operator raft list-peers -format json

# Check the status of all the nodes again. It is normal for the second node to
# be removed from the raft cluster but be present as a standby node.
vault_2 status || true
vault_3 status || true
vault_4 status || true