diff.patch

From b536eb89d895cb80489ac970923e389a0d73636e Mon Sep 17 00:00:00 2001
From: "Sean M. Collins" <sean@coreitpro.com>
Date: Mon, 11 Dec 2017 10:47:15 -0500
Subject: [PATCH] Evict wtp-k8s-3 from the cluster

Hardware issues with node 3 means we'll promote node 4 to a master
---
 k8s-envs/vpcbeta-production/hosts | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/k8s-envs/vpcbeta-production/hosts b/k8s-envs/vpcbeta-production/hosts
index 01faddb..15492f4 100644
--- a/k8s-envs/vpcbeta-production/hosts
+++ b/k8s-envs/vpcbeta-production/hosts
@@ -1,18 +1,20 @@
 wtp-k8s-1.projwrigley.com ansible_host=10.171.203.112 ip=10.171.203.112 ansible_user=root database_node_ord=1
 wtp-k8s-2.projwrigley.com ansible_host=10.171.203.193 ip=10.171.203.193 ansible_user=root database_node_ord=2
-wtp-k8s-3.projwrigley.com ansible_host=10.171.201.20 ip=10.171.201.20 ansible_user=root database_node_ord=3
-wtp-k8s-4.projwrigley.com ansible_host=10.171.201.249 ip=10.171.201.249 ansible_user=root
+wtp-k8s-4.projwrigley.com ansible_host=10.171.201.249 ip=10.171.201.249 ansible_user=root database_node_ord=3
 wtp-k8s-5.projwrigley.com ansible_host=10.171.201.112 ip=10.171.201.112 ansible_user=root
 wtp-k8s-6.projwrigley.com ansible_host=10.171.201.66 ip=10.171.201.66 ansible_user=root
 
 [kube-master]
-wtp-k8s-[1:3].projwrigley.com
+wtp-k8s-[1:2].projwrigley.com
+wtp-k8s-4.projwrigley.com
 
 [etcd]
-wtp-k8s-[1:3].projwrigley.com
+wtp-k8s-[1:2].projwrigley.com
+wtp-k8s-4.projwrigley.com
 
 [kube-node]
-wtp-k8s-[1:6].projwrigley.com
+wtp-k8s-[1:2].projwrigley.com
+wtp-k8s-[4:6].projwrigley.com
 
 [database:children]
 kube-master

gistfile1.txt

2017-12-11 16:02:03.146978 | TASK [etcd : Configure | Check if cluster is healthy] **************************
2017-12-11 16:02:03.147120 | Monday 11 December 2017  16:02:03 +0000 (0:00:02.783)       0:01:06.562 ******* 
2017-12-11 16:02:13.490291 | ok: [wtp-k8s-1.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 cluster-health | grep -q 'cluster is healthy'", "delta": "0:00:10.048627", "end": "2017-12-11 10:02:13.471955", "failed": false, "rc": 0, "start": "2017-12-11 10:02:03.423328", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
2017-12-11 16:02:13.503112 | ok: [wtp-k8s-2.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 cluster-health | grep -q 'cluster is healthy'", "delta": "0:00:10.047946", "end": "2017-12-11 10:02:13.485331", "failed": false, "rc": 0, "start": "2017-12-11 10:02:03.437385", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
2017-12-11 16:02:13.520347 | ok: [wtp-k8s-4.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 cluster-health | grep -q 'cluster is healthy'", "delta": "0:00:10.049270", "end": "2017-12-11 10:02:13.503088", "failed": false, "rc": 0, "start": "2017-12-11 10:02:03.453818", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
2017-12-11 16:02:13.529611 | 
2017-12-11 16:02:13.529663 | TASK [etcd : Configure | Check if member is in cluster] ************************
2017-12-11 16:02:13.529811 | Monday 11 December 2017  16:02:13 +0000 (0:00:10.382)       0:01:16.944 ******* 
2017-12-11 16:02:13.855650 | ok: [wtp-k8s-1.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --no-sync --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member list | grep -q 10.171.203.112", "delta": "0:00:00.033135", "end": "2017-12-11 10:02:13.839410", "failed": false, "rc": 0, "start": "2017-12-11 10:02:13.806275", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
2017-12-11 16:02:13.875416 | ok: [wtp-k8s-2.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --no-sync --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member list | grep -q 10.171.203.193", "delta": "0:00:00.033358", "end": "2017-12-11 10:02:13.859151", "failed": false, "rc": 0, "start": "2017-12-11 10:02:13.825793", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
2017-12-11 16:02:13.902197 | fatal: [wtp-k8s-4.projwrigley.com]: FAILED! => {"changed": false, "cmd": "/usr/local/bin/etcdctl --no-sync --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member list | grep -q 10.171.201.249", "delta": "0:00:00.033663", "end": "2017-12-11 10:02:13.887385", "failed": true, "msg": "non-zero return code", "rc": 1, "start": "2017-12-11 10:02:13.853722", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
2017-12-11 16:02:13.902343 | ...ignoring
2017-12-11 16:02:13.911672 | 
2017-12-11 16:02:13.911739 | TASK [etcd : Configure | Add member to the cluster if it is not there] *********
2017-12-11 16:02:13.911885 | Monday 11 December 2017  16:02:13 +0000 (0:00:00.382)       0:01:17.326 ******* 
2017-12-11 16:02:16.293740 | fatal: [wtp-k8s-4.projwrigley.com]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member add etcd3 https://10.171.201.249:2380", "delta": "0:00:02.042013", "end": "2017-12-11 10:02:16.273793", "failed": true, "msg": "non-zero return code", "rc": 1, "start": "2017-12-11 10:02:14.231780", "stderr": "client: etcd cluster is unavailable or misconfigured; error #0: client: etcd member https://10.171.203.193:2379 has no leader\n; error #1: client: etcd member https://10.171.203.112:2379 has no leader\n; error #2: client: endpoint https://10.171.201.20:2379 exceeded header timeout", "stderr_lines": ["client: etcd cluster is unavailable or misconfigured; error #0: client: etcd member https://10.171.203.193:2379 has no leader", "; error #1: client: etcd member https://10.171.203.112:2379 has no leader", "; error #2: client: endpoint https://10.171.201.20:2379 exceeded header timeout"], "stdout": "", "stdout_lines": []}