Created
September 22, 2016 06:16
-
-
Save ecelis/c664de6bbb80650500e7cedef7195463 to your computer and use it in GitHub Desktop.
Docker cluster sample setup
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ######################################################################################################################################################## | |
| ####### The following commands are for Module 4: Building your Swarm Infrastructure | |
| ####### | |
| ####### Remember to substitute hostnames and IPs etc etc for the appropriate values in your environment | |
| ######################################################################################################################################################## | |
| ####### CONSUL BUILD COMMANDS | |
| NODE1 | |
| docker run --restart=unless-stopped -d -h consul1 --name consul1 -v /mnt:/data \ | |
| -p 10.0.1.5:8300:8300 \ | |
| -p 10.0.1.5:8301:8301 \ | |
| -p 10.0.1.5:8301:8301/udp \ | |
| -p 10.0.1.5:8302:8302 \ | |
| -p 10.0.1.5:8302:8302/udp \ | |
| -p 10.0.1.5:8400:8400 \ | |
| -p 10.0.1.5:8500:8500 \ | |
| -p 172.17.0.1:53:53/udp \ | |
| progrium/consul -server -advertise 10.0.1.5 -bootstrap-expect 3 | |
| NODE2 | |
| docker run --restart=unless-stopped -d -h consul2 --name consul2 -v /mnt:/data \ | |
| -p 10.0.2.5:8300:8300 \ | |
| -p 10.0.2.5:8301:8301 \ | |
| -p 10.0.2.5:8301:8301/udp \ | |
| -p 10.0.2.5:8302:8302 \ | |
| -p 10.0.2.5:8302:8302/udp \ | |
| -p 10.0.2.5:8400:8400 \ | |
| -p 10.0.2.5:8500:8500 \ | |
| -p 172.17.0.1:53:53/udp \ | |
| progrium/consul -server -advertise 10.0.2.5 -join 10.0.1.5 | |
| NODE3 | |
| docker run --restart=unless-stopped -d -h consul3 --name consul3 -v /mnt:/data \ | |
| -p 10.0.3.5:8300:8300 \ | |
| -p 10.0.3.5:8301:8301 \ | |
| -p 10.0.3.5:8301:8301/udp \ | |
| -p 10.0.3.5:8302:8302 \ | |
| -p 10.0.3.5:8302:8302/udp \ | |
| -p 10.0.3.5:8400:8400 \ | |
| -p 10.0.3.5:8500:8500 \ | |
| -p 172.17.0.1:53:53/udp \ | |
| progrium/consul -server -advertise 10.0.3.5 -join 10.0.1.5 | |
| ####### Perform the following from any of the consul server nodes to verify the status of Consul | |
| docker exec -it consul<1/2/3> bash | |
| consul members | |
| ####### CONSUL CLIENT BUILDS ON NODES 1-3 | |
| NODE1 | |
| docker run --restart=unless-stopped -d -h consul-agt1 --name consul-agt1 \ | |
| -p 8300:8300 \ | |
| -p 8301:8301 -p 8301:8301/udp \ | |
| -p 8302:8302 -p 8302:8302/udp \ | |
| -p 8400:8400 \ | |
| -p 8500:8500 \ | |
| -p 8600:8600/udp \ | |
| progrium/consul -rejoin -advertise 10.0.4.5 -join 10.0.1.5 | |
| NODE2 | |
| docker run --restart=unless-stopped -d -h consul-agt2 --name consul-agt2 \ | |
| -p 8300:8300 \ | |
| -p 8301:8301 -p 8301:8301/udp \ | |
| -p 8302:8302 -p 8302:8302/udp \ | |
| -p 8400:8400 \ | |
| -p 8500:8500 \ | |
| -p 8600:8600/udp \ | |
| progrium/consul -rejoin -advertise 10.0.4.5 -join 10.0.1.5 | |
| NODE3 | |
| docker run --restart=unless-stopped -d -h consul-agt3 --name consul-agt3 \ | |
| -p 8300:8300 \ | |
| -p 8301:8301 -p 8301:8301/udp \ | |
| -p 8302:8302 -p 8302:8302/udp \ | |
| -p 8400:8400 \ | |
| -p 8500:8500 \ | |
| -p 8600:8600/udp \ | |
| progrium/consul -rejoin -advertise 10.0.4.5 -join 10.0.1.5 | |
| ####### SWARM MANAGER BUILD COMMANDS | |
| NODE1 | |
| docker run --restart=unless-stopped -h mgr1 --name mgr1 -d -p 3375:2375 swarm manage --replication --advertise 10.0.1.5:3375 consul://10.0.1.5:8500/ | |
| NODE2 | |
| docker run --restart=unless-stopped -h mgr2 --name mgr2 -d -p 3375:2375 swarm manage --replication --advertise 10.0.2.5:3375 consul://10.0.2.5:8500/ | |
| NODE3 | |
| docker run --restart=unless-stopped -h mgr3 --name mgr3 -d -p 3375:2375 swarm manage --replication --advertise 10.0.3.5:3375 consul://10.0.3.5:8500/ | |
| ####### SWARM JOIN COMMANDS TO JOIN NODES TO THE CLUSTER | |
| NODE1 | |
| docker run -d swarm join --advertise=10.0.4.5:2375 consul://10.0.4.5:8500/ | |
| NODE2 | |
| docker run -d swarm join --advertise=10.0.5.5:2375 consul://10.0.5.5:8500/ | |
| NODE3 | |
| docker run -d swarm join --advertise=10.0.6.5:2375 consul://10.0.6.5:8500/ | |
| ####### To launch a local registrator service you can use the following command on each node - | |
| docker run -d --name registrator -h registrator -v /var/run/docker.sock:/tmp/docker.sock gliderlabs/registrator:latest consul://10.0.1.5:8500 | |
| ######################################################################################################################################################## | |
| ####### The following commands are for mModule 5: Securing your Swarm Cluster | |
| ####### | |
| ####### Remember to substitute hostnames and IPs etc for the appropriate values in your environment | |
| ######################################################################################################################################################## | |
| CREATE CA | |
| openssl genrsa -out ca-key.pem 2048 | |
| openssl req -config /usr/lib/ssl/openssl.cnf -new -key ca-key.pem -x509 -days 1825 -out ca-cert.pem | |
| CREATE MANAGER AND NODE KEYS | |
| openssl genrsa -out manager1-key.pem 2048 | |
| openssl req -subj "/CN=manager1" -new -key manager1-key.pem -out manager1.csr | |
| echo subjectAltName = IP:10.0.1.5,IP:127.0.0.1 > extfile.cnf | |
| openssl x509 -req -days 365 -in manager1.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out manager1-cert.pem -extfile extfile.cnf | |
| openssl genrsa -out manager2-key.pem 2048 | |
| openssl req -subj "/CN=manager2" -new -key manager2-key.pem -out manager2.csr | |
| echo subjectAltName = IP:10.0.2.5,IP:127.0.0.1 > extfile.cnf | |
| openssl x509 -req -days 365 -in manager2.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out manager2-cert.pem -extfile extfile.cnf | |
| openssl genrsa -out manager3-key.pem 2048 | |
| openssl req -subj "/CN=manager3" -new -key manager3-key.pem -out manager3.csr | |
| echo subjectAltName = IP:10.0.3.5,IP:127.0.0.1 > extfile.cnf | |
| openssl x509 -req -days 365 -in manager3.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out manager3-cert.pem -extfile extfile.cnf | |
| openssl genrsa -out node1-key.pem 2048 | |
| openssl req -subj "/CN=node1" -new -key node1-key.pem -out node1.csr | |
| echo subjectAltName = IP:10.0.4.5,IP:127.0.0.1 > extfile.cnf | |
| openssl x509 -req -days 365 -in node1.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out node1-cert.pem -extfile extfile.cnf | |
| openssl genrsa -out node2-key.pem 2048 | |
| openssl req -subj "/CN=node2" -new -key node2-key.pem -out node2.csr | |
| echo subjectAltName = IP:10.0.5.5,IP:127.0.0.1 > extfile.cnf | |
| openssl x509 -req -days 365 -in node2.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out node2-cert.pem -extfile extfile.cnf | |
| openssl genrsa -out node3-key.pem 2048 | |
| openssl req -subj "/CN=node3" -new -key node3-key.pem -out node3.csr | |
| echo subjectAltName = IP:10.0.6.5,IP:127.0.0.1 > extfile.cnf | |
| openssl x509 -req -days 365 -in node3.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out node3-cert.pem -extfile extfile.cnf | |
| CREATE CLIENT KEYS | |
| openssl genrsa -out client-key.pem 2048 | |
| openssl req -subj "/CN=client" -new -key client-key.pem -out client.csr | |
| echo extendedKeyUsage = clientAuth > extfile.cnf | |
| openssl x509 -req -days 365 -in client.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -extfile extfile.cnf | |
| COPY KEYS | |
| ####### The following SCP commands are those used in the course and use the `-i eu-west-1-key.pem` certificate required for my own personal AWS environment. | |
| ####### Be sure to either replace this with the appropriate key for your environment, or remove it if your environment does not require a key for SSH/SCP | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@manager1:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./manager1-cert.pem ubuntu@manager1:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./manager1-key.pem ubuntu@manager1:/home/ubuntu/.docker/key.pem | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@manager2:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./manager2-cert.pem ubuntu@manager2:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./manager2-key.pem ubuntu@manager2:/home/ubuntu/.docker/key.pem | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@manager3:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./manager3-cert.pem ubuntu@manager3:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./manager3-key.pem ubuntu@manager3:/home/ubuntu/.docker/key.pem | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@node1:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./node1-cert.pem ubuntu@node1:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./node1-key.pem ubuntu@node1:/home/ubuntu/.docker/key.pem | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@node2:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./node2-cert.pem ubuntu@node2:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./node2-key.pem ubuntu@node2:/home/ubuntu/.docker/key.pem | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@node3:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./node3-cert.pem ubuntu@node3:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./node3-key.pem ubuntu@node3:/home/ubuntu/.docker/key.pem | |
| scp -i eu-west-1-key.pem ./ca-cert.pem ubuntu@client:/home/ubuntu/.docker/ca.pem | |
| scp -i eu-west-1-key.pem ./client-cert.pem ubuntu@client:/home/ubuntu/.docker/cert.pem | |
| scp -i eu-west-1-key.pem ./client-key.pem ubuntu@client:/home/ubuntu/.docker/key.pem | |
| DOCKER DAEMON restarts | |
| vim /etc/default/docker | |
| -H tcp://0.0.0.0:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem | |
| service docker start | |
| service docker status | |
| START NEW CONSUL SERVERS | |
| MANAGER1 | |
| docker -H tcp://manager1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul1 --name consul1 -v /mnt:/data -p 10.0.1.5:8300:8300 -p 10.0.1.5:8301:8301 -p 10.0.1.5:8301:8301/udp -p 10.0.1.5:8302:8302 -p 10.0.1.5:8302:8302/udp -p 10.0.1.5:8400:8400 -p 10.0.1.5:8500:8500 -p 172.17.0.1:53:53/udp progrium/consul -server -advertise 10.0.1.5 -join 10.0.2.5 | |
| MANAGER2 | |
| docker -H tcp://manager2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul2 --name consul2 -v /mnt:/data -p 10.0.2.5:8300:8300 -p 10.0.2.5:8301:8301 -p 10.0.2.5:8301:8301/udp -p 10.0.2.5:8302:8302 -p 10.0.2.5:8302:8302/udp -p 10.0.2.5:8400:8400 -p 10.0.2.5:8500:8500 -p 172.17.0.1:53:53/udp progrium/consul -server -advertise 10.0.1.5 -join 10.0.1.5 | |
| MANAGER3 | |
| docker -H tcp://manager3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul3 --name consul3 -v /mnt:/data -p 10.0.3.5:8300:8300 -p 10.0.3.5:8301:8301 -p 10.0.3.5:8301:8301/udp -p 10.0.3.5:8302:8302 -p 10.0.3.5:8302:8302/udp -p 10.0.3.5:8400:8400 -p 10.0.3.5:8500:8500 -p 172.17.0.1:53:53/udp progrium/consul -server -advertise 10.0.1.5 -join 10.0.1.5 | |
| docker -H tcp://manager1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr1 --name mgr1 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.1.5:2376 consul://10.0.1.5:8500/ | |
| START CONSUL CLIENTS | |
| NODE1 | |
| docker -H tcp://node1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul-agt1 --name consul-agt1 \ | |
| -p 8300:8300 \ | |
| -p 8301:8301 -p 8301:8301/udp \ | |
| -p 8302:8302 -p 8302:8302/udp \ | |
| -p 8400:8400 \ | |
| -p 8500:8500 \ | |
| -p 8600:8600/udp \ | |
| progrium/consul -rejoin -advertise 10.0.4.5 -join 10.0.1.5 | |
| NODE2 | |
| docker -H tcp://node2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul-agt2 --name consul-agt2 \ | |
| -p 8300:8300 \ | |
| -p 8301:8301 -p 8301:8301/udp \ | |
| -p 8302:8302 -p 8302:8302/udp \ | |
| -p 8400:8400 \ | |
| -p 8500:8500 \ | |
| -p 8600:8600/udp \ | |
| progrium/consul -rejoin -advertise 10.0.5.5 -join 10.0.1.5 | |
| NODE3 | |
| docker -H tcp://node3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul-agt3 --name consul-agt3 \ | |
| -p 8300:8300 \ | |
| -p 8301:8301 -p 8301:8301/udp \ | |
| -p 8302:8302 -p 8302:8302/udp \ | |
| -p 8400:8400 \ | |
| -p 8500:8500 \ | |
| -p 8600:8600/udp \ | |
| progrium/consul -rejoin -advertise 10.0.6.5 -join 10.0.1.5 | |
| START SWARM MANAGERS | |
| MNAGER1 | |
| docker -H tcp://manager1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr1 --name mgr1 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.1.5:2376 consul://10.0.1.5:8500/ | |
| MANAGER2 | |
| docker -H tcp://manager2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr2 --name mgr2 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.2.5:2376 consul://10.0.2.5:8500/ | |
| MANAGER3 | |
| docker -H tcp://manager3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr3 --name mgr3 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.3.5:2376 consul://10.0.3.5:8500/ | |
| START SWARM JOIN CONTAIENRS ON EACH NODE | |
| NODE1 | |
| docker -H tcp://node1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run -d -h join --name join swarm join --advertise=10.0.4.5:2376 consul://10.0.4.5:8500/ | |
| NODE2 | |
| docker -H tcp://node2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run -d -h join --name join swarm join --advertise=10.0.5.5:2376 consul://10.0.5.5:8500/ | |
| NODE3 | |
| docker -H tcp://node3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run -d -h join --name join swarm join --advertise=10.0.6.5:2376 consul://10.0.6.5:8500/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment