my swarm loadbalancer experiences

swarm.md

Manager:

[root@centosvm ~]# docker ps
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES
a593d485050a        ddewaele/springboot.crud.sample:latest   "sh -c 'java $JAVA_OP"   7 minutes ago       Up 7 minutes                            springbootcrudsample.1.5syc6j4c8i3bnerdqq4e1yelm

Node1:

[root@node1 ~]# docker ps
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES
d3b3fbc0f2c5        ddewaele/springboot.crud.sample:latest   "sh -c 'java $JAVA_OP"   4 minutes ago       Up 4 minutes                            springbootcrudsample.3.7y1oyjyrifgkmxlr20oai5ppl

Node 2:

[root@node2 ~]# docker ps
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES
ebca8f24ec3a        ddewaele/springboot.crud.sample:latest   "sh -c 'java $JAVA_OP"   7 minutes ago       Up 7 minutes                            springbootcrudsample.2.4tqjad7od8ep047s55485na1t

Killing the docker container at node 2 (swarm should create a new container as we have a replication=3 on the service)

[root@node1 ~]# docker kill d3b3fbc0f2c5
d3b3fbc0f2c5

Container gone

[root@node1 ~]# docker ps
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES

New container up

[root@node1 ~]# docker ps
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES
b8c9a7a5cf97        ddewaele/springboot.crud.sample:latest   "sh -c 'java $JAVA_OP"   11 seconds ago      Up 9 seconds                            springbootcrudsample.3.9v4cnhi8dvq7n8afb2kvp28sk

In the output however, when container d3b3fbc0f2c5 was killed, why didn't the swarm loadbalancer detect this ? why did it still sent traffic its way ?

[root@centosvm ~]# while :; do curl http://localhost:8080/env/hostname ; echo "" ; sleep 1; done
{"hostname":"d3b3fbc0f2c5"}
{"hostname":"a593d485050a"}
{"hostname":"ebca8f24ec3a"}
{"hostname":"d3b3fbc0f2c5"}
{"hostname":"a593d485050a"}
{"hostname":"ebca8f24ec3a"}
{"hostname":"d3b3fbc0f2c5"}
{"hostname":"a593d485050a"}
{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
curl: (7) Failed connect to localhost:8080; Connection refused

{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
curl: (7) Failed connect to localhost:8080; Connection refused

{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
curl: (7) Failed connect to localhost:8080; Connection refused

{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
curl: (7) Failed connect to localhost:8080; Connection refused

{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
curl: (7) Failed connect to localhost:8080; Connection refused

{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
curl: (7) Failed connect to localhost:8080; Connection refused

{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
{"hostname":"b8c9a7a5cf97"}
{"hostname":"ebca8f24ec3a"}
{"hostname":"a593d485050a"}
{"hostname":"b8c9a7a5cf97"}