Skip to content

Instantly share code, notes, and snippets.

@r6m

r6m/nomad-consul-production-minmal-setup-part1.md

Forked from jonaskahn/nomad-consul-production-minmal-setup-part1.md
Created November 26, 2022 19:43
Show Gist options
  • Save r6m/861357d437ea5b4bc04c3862a4bef7b2 to your computer and use it in GitHub Desktop.
Save r6m/861357d437ea5b4bc04c3862a4bef7b2 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
nomad-consul-production-minmal-setup-part1.md

Practice Consul, Nomad in Production Part 1- Setup Cluster

This cluster will be set up for 3 servers, each server will run (Nomad server/client + Consul server/client)

Server Information

workload-info

Plans

Setup

Prerequisites

  • Ubuntu 22.04 LTS
  • Consul 1.13.3
  • Nomad 1.4.2

Update the system and install some required software ( Run on all nodes )

  • Install docker and some other packages
bash -c "$(curl -fsSL https://gist.github.com/tuyendev/3a3bca421b6689f73c670257090b5d34/raw/c31f3de439c5ac86c4e4c7b50bf95a26988b0bd5/nomad-consul-prerequisite.sh)"
  • Install nomad & consul packages
bash -c "$(curl -fsSL https://gist.github.com/tuyendev/63a0a75c1abd354199fd12ef04bacafe/raw/71c7f72cc21943fd8679ad148c4494a55f627dc9/consul-nomad-install.sh)"

Note

structure

  • A nomad client connect to one consul

Setup Core Node

[Name: sg-server-core-1 <-> IP: 10.238.22.122]

Setup consul

  1. Generate a secrect key
consul keygen

##### OUTPUT ##### 
+dnwtBflWAtk1QxpKjFS463Ytxd1VZdtjupwYxw1qgg=

This key will be used for share configuration in all consul nodes, take note of the key.

  1. Generate TLS certificates for RPC encryption

Consul can use TLS to verify the authenticity of servers and clients.

  • Create the Certificate Authority
mkdir ~/certs

cd ~/certs && consul tls ca create
	
##### OUTPUT #####
==> Saved consul-agent-ca.pem
==> Saved consul-agent-ca-key.pem
  • Create the certificates
cd ~/certs && consul tls cert create -server -dc saigon

##### OUTPUT #####
==> WARNING: Server Certificates grants authority to become a
    server and access all state in the cluster including root keys
    and all ACL tokens. Do not distribute them to production hosts
    that are not server nodes. Store them as securely as CA keys.
==> Using consul-agent-ca.pem and consul-agent-ca-key.pem
==> Saved saigon-server-consul-0.pem
==> Saved saigon-server-consul-0-key.pem
  • Result
ls -al ~/certs

##### OUTPUT #####
total 24
drwxrwxr-x 2 ubuntu ubuntu 4096 Nov 21 10:26 .
drwxr-x--- 5 ubuntu ubuntu 4096 Nov 21 10:23 ..
-rw-r--r-- 1 consul consul  227 Nov 21 17:28 consul-agent-ca-key.pem
-rw-r--r-- 1 consul consul 1074 Nov 21 17:28 consul-agent-ca.pem
-rw-r--r-- 1 consul consul  227 Nov 21 17:28 saigon-server-consul-0-key.pem
-rw-r--r-- 1 consul consul  973 Nov 21 17:28 saigon-server-consul-0.pem
  • Copy to configuration folder /opt/consul/certs & keep a backup
sudo cp -R ~/certs/* /opt/consul/certs
sudo chown -R consul:consul /opt/consul && sudo chmod a+r -R /opt/consul/certs
  1. Configuration
  • /etc/consul.d/consul.hcl
sudo nano /etc/consul.d/consul.hcl

##### content #####
datacenter             = "saigon"
node_name              = "sg-core-consul-1"
data_dir               = "/opt/consul"
encrypt                = "+dnwtBflWAtk1QxpKjFS463Ytxd1VZdtjupwYxw1qgg=" ### Encrypt key from Step 1
verify_incoming        = true
verify_outgoing        = true
verify_server_hostname = true
ca_file                = "/opt/consul/certs/consul-agent-ca.pem"
cert_file              = "/opt/consul/certs/saigon-server-consul-0.pem"
key_file               = "/opt/consul/certs/saigon-server-consul-0-key.pem"

auto_encrypt {
  allow_tls = true
}

retry_join = ["10.238.22.122", "10.238.22.182", "10.238.22.50"] ### List of all consul server ( CORE NODES )

acl {
  enabled                  = true
  default_policy           = "allow"
  enable_token_persistence = true
}

performance {
  raft_multiplier = 1
}
  • /etc/consul.d/server.hcl
sudo nano /etc/consul.d/server.hcl

##### content #####
server           = true
bootstrap_expect = 3
bind_addr        = "10.238.22.122"
client_addr      = "0.0.0.0"
connect {
  enabled = true
}
addresses {
  grpc = "127.0.0.1"
}
ports {
  grpc = 8502
}
ui_config {
  enabled = true
}
  1. Start service
sudo systemctl restart consul
sudo systemctl status consul

Setup nomad

  1. Configuration
  • /etc/nomad.d/nomad.hcl
sudo nano /etc/nomad.d/nomad.hcl

##### content #####
datacenter = "saigon"
data_dir   = "/opt/nomad"
bind_addr  = "10.238.22.122"
acl {
  enabled = true
}
telemetry {
  collection_interval = "1s"
  disable_hostname = true
  prometheus_metrics = true
  publish_allocation_metrics = true
  publish_node_metrics = true
}
plugin "docker" {
  config {
    endpoint = "unix:///var/run/docker.sock"
    volumes {
      enabled      = true
      selinuxlabel = "z"
    }
    extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"]

    gc {
      image       = true
      image_delay = "10m"
      container   = true

      dangling_containers {
        enabled        = true
        dry_run        = false
        period         = "5m"
        creation_grace = "5m"
      }
    }
    allow_privileged = true
    
  }
}
  • /etc/nomad.d/server.hcl
sudo nano /etc/nomad.d/server.hcl

##### content #####
server {
  enabled          = true
  bootstrap_expect = 3
  server_join {
    retry_join = ["10.238.22.122:4648", "10.238.22.182:4648", "10.238.22.50:4648"]  # List IP of nomad server ( CORE NODES )
  }
}
  • /etc/nomad.d/client.hcl
sudo nano /etc/nomad.d/client.hcl

##### content #####
client {
  enabled    = true
  node_class = "core"
  server_join {
    #NOMAD SERVER LIST
    retry_join = ["10.238.22.122:4647", "10.238.22.182:4647", "10.238.22.50:4647"]
  }
}
  • /etc/nomad.d/consul.hcl
sudo nano /etc/nomad.d/consul.hcl

##### content #####
consul {
  address = "127.0.0.1:8500"
  server_service_name = "sg-core-nomad-server-1"
  client_service_name = "sg-core-nomad-client-1"
  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
}
  1. Start service
sudo systemctl start nomad && sudo systemctl status nomad

[Name: sg-server-core-2 <-> IP: 10.238.22.182]

Setup consul

  1. Copy backup certs from sg-server-core-1 to sg-server-core-2
sudo cp -R ~/certs/* /opt/consul/certs
sudo chown -R consul:consul /opt/consul && sudo chmod a+r -R /opt/consul/certs
  1. Configuration
  • /etc/consul.d/consul.hcl
sudo nano /etc/consul.d/consul.hcl

##### content #####
datacenter             = "saigon"
node_name              = "sg-core-consul-2"
data_dir               = "/opt/consul"
encrypt                = "+dnwtBflWAtk1QxpKjFS463Ytxd1VZdtjupwYxw1qgg=" ### Encrypt key from Step 1
verify_incoming        = true
verify_outgoing        = true
verify_server_hostname = true
ca_file                = "/opt/consul/certs/consul-agent-ca.pem"
cert_file              = "/opt/consul/certs/saigon-server-consul-0.pem"
key_file               = "/opt/consul/certs/saigon-server-consul-0-key.pem"

auto_encrypt {
  allow_tls = true
}

retry_join = ["10.238.22.122", "10.238.22.182", "10.238.22.50"] ### List of all consul server ( CORE NODES )

acl {
  enabled                  = true
  default_policy           = "allow"
  enable_token_persistence = true
}

performance {
  raft_multiplier = 1
}
  • /etc/consul.d/server.hcl
sudo nano /etc/consul.d/server.hcl

##### content #####
server           = true
bootstrap_expect = 3
bind_addr        = "10.238.22.182"
client_addr      = "0.0.0.0"
connect {
  enabled = true
}
addresses {
  grpc = "127.0.0.1"
}
ports {
  grpc = 8502
}
ui_config {
  enabled = true
}
  1. Start service
sudo systemctl restart consul
sudo systemctl status consul

Setup nomad

  1. Configuration
  • /etc/nomad.d/nomad.hcl
sudo nano /etc/nomad.d/nomad.hcl

##### content #####
datacenter = "saigon"
data_dir   = "/opt/nomad"
bind_addr  = "10.238.22.182"
acl {
  enabled = true
}
telemetry {
  collection_interval = "1s"
  disable_hostname = true
  prometheus_metrics = true
  publish_allocation_metrics = true
  publish_node_metrics = true
}
plugin "docker" {
  config {
    endpoint = "unix:///var/run/docker.sock"
    volumes {
      enabled      = true
      selinuxlabel = "z"
    }
    extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"]

    gc {
      image       = true
      image_delay = "10m"
      container   = true

      dangling_containers {
        enabled        = true
        dry_run        = false
        period         = "5m"
        creation_grace = "5m"
      }
    }
    allow_privileged = true
    
  }
}
  • /etc/nomad.d/server.hcl
sudo nano /etc/nomad.d/server.hcl

##### content #####
server {
  enabled          = true
  bootstrap_expect = 3
  server_join {
    retry_join = ["10.238.22.122:4648", "10.238.22.182:4648", "10.238.22.50:4648"]  # List IP of nomad server ( CORE NODES )
  }
}
  • /etc/nomad.d/client.hcl
sudo nano /etc/nomad.d/client.hcl

##### content #####
client {
  enabled    = true
  node_class = "core"
  server_join {
    #NOMAD SERVER LIST
    retry_join = ["10.238.22.122:4647", "10.238.22.182:4647", "10.238.22.50:4647"]
  }
}
  • /etc/nomad.d/consul.hcl
sudo nano /etc/nomad.d/consul.hcl

##### content #####
consul {
  address = "127.0.0.1:8500"
  server_service_name = "sg-core-nomad-server-2"
  client_service_name = "sg-core-nomad-client-2"
  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
}
  1. Start service
sudo systemctl start nomad && sudo systemctl status nomad

[Name: sg-server-core-3 <-> IP: 10.238.22.50]

Setup consul

  1. Copy backup certs from sg-server-core-1 to sg-server-core-3
sudo cp -R ~/certs/* /opt/consul/certs
sudo chown -R consul:consul /opt/consul && sudo chmod a+r -R /opt/consul/certs
  1. Configuration
  • /etc/consul.d/consul.hcl
sudo nano /etc/consul.d/consul.hcl

##### content #####
datacenter             = "saigon"
node_name              = "sg-core-consul-3"
data_dir               = "/opt/consul"
encrypt                = "+dnwtBflWAtk1QxpKjFS463Ytxd1VZdtjupwYxw1qgg=" ### Encrypt key from Step 1
verify_incoming        = true
verify_outgoing        = true
verify_server_hostname = true
ca_file                = "/opt/consul/certs/consul-agent-ca.pem"
cert_file              = "/opt/consul/certs/saigon-server-consul-0.pem"
key_file               = "/opt/consul/certs/saigon-server-consul-0-key.pem"

auto_encrypt {
  allow_tls = true
}

retry_join = ["10.238.22.122", "10.238.22.182", "10.238.22.50"] ### List of all consul server ( CORE NODES )

acl {
  enabled                  = true
  default_policy           = "allow"
  enable_token_persistence = true
}

performance {
  raft_multiplier = 1
}
  • /etc/consul.d/server.hcl
sudo nano /etc/consul.d/server.hcl

##### content #####
server           = true
bootstrap_expect = 3
bind_addr        = "10.238.22.50"
client_addr      = "0.0.0.0"
connect {
  enabled = true
}
addresses {
  grpc = "127.0.0.1"
}
ports {
  grpc = 8502
}
ui_config {
  enabled = true
}
  1. Start service
sudo systemctl restart consul && journalctl -f

Setup nomad

  1. Configuration
  • /etc/nomad.d/nomad.hcl
sudo nano /etc/nomad.d/nomad.hcl

##### content #####
datacenter = "saigon"
data_dir   = "/opt/nomad"
bind_addr  = "10.238.22.50"
acl {
  enabled = true
}
telemetry {
  collection_interval = "1s"
  disable_hostname = true
  prometheus_metrics = true
  publish_allocation_metrics = true
  publish_node_metrics = true
}
plugin "docker" {
  config {
    endpoint = "unix:///var/run/docker.sock"
    volumes {
      enabled      = true
      selinuxlabel = "z"
    }
    extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"]

    gc {
      image       = true
      image_delay = "10m"
      container   = true

      dangling_containers {
        enabled        = true
        dry_run        = false
        period         = "5m"
        creation_grace = "5m"
      }
    }
    allow_privileged = true
    
  }
}
  • /etc/nomad.d/server.hcl
sudo nano /etc/nomad.d/server.hcl

##### content #####
server {
  enabled          = true
  bootstrap_expect = 3
  server_join {
    retry_join = ["10.238.22.122:4648", "10.238.22.182:4648", "10.238.22.50:4648"]  # List IP of nomad server ( CORE NODES )
  }
}
  • /etc/nomad.d/client.hcl
sudo nano /etc/nomad.d/client.hcl

##### content #####
client {
  enabled    = true
  node_class = "core"
  server_join {
    #NOMAD SERVER LIST
    retry_join = ["10.238.22.122:4647", "10.238.22.182:4647", "10.238.22.50:4647"]
  }
}
  • /etc/nomad.d/consul.hcl
sudo nano /etc/nomad.d/consul.hcl

##### content #####
consul {
  address = "127.0.0.1:8500"
  server_service_name = "sg-core-nomad-server-3"
  client_service_name = "sg-core-nomad-client-3"
  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
}
  1. Start service
sudo systemctl start nomad && sudo systemctl status nomad

Result

Access any link to see the result

enter image description here

Secure Nomad & Consul Server ( Basic )

Secure Consul

  1. Bootstrap ACL ( run in any core node )
consul acl bootstrap

##### OUTPUT #####
AccessorID:       bbf8c714-a61d-3643-619e-fc5852405e62
SecretID:         c5a19133-4591-7cf7-20b4-15f072401324 <==== TAKE NOTE FOR THE KEY
Description:      Bootstrap Token (Global Management)
Local:            false
Create Time:      2022-11-21 11:46:48.650513634 +0700 +07
Policies:
   00000000-0000-0000-0000-000000000001 - global-management
  1. Export this key to bash env (run on all core nodes)
echo '
export CONSUL_HTTP_TOKEN="c5a19133-4591-7cf7-20b4-15f072401324"  
export CONSUL_MGMT_TOKEN="c5a19133-4591-7cf7-20b4-15f072401324"
' | tee -a ~/.bashrc && source ~/.bashrc
  1. Make default policy
cd ~ && \
echo "agent_prefix \"\" { 
 policy = \"write\" 
} 
node_prefix \"\" { 
 policy = \"write\" 
} 
service_prefix \"\" { 
 policy = \"read\" 
} 
session_prefix \"\" { 
 policy = \"read\" 
}" | tee -a node-policy.hcl
  1. Apply policy
consul acl policy create -token=${CONSUL_MGMT_TOKEN} -name node-policy -rules @node-policy.hcl
  1. Create a token from bootstrap policy
consul acl token create -token=${CONSUL_MGMT_TOKEN} -description "node token" -policy-name node-policy

##### OUTPUT #####
AccessorID:       335279ff-d0cb-6151-1ffa-6cf713e581b0
SecretID:         76967b87-ceb4-ca3a-85c6-b2b4033eda4c <============== secret key cho node-token 
Description:      node token
Local:            false
Create Time:      2022-11-21 11:52:38.26291528 +0700 +07
Policies:
   edc0c6af-32f7-98ea-fe9e-369d2b7b2b5c - node-policy
  1. Apply token to consul node
consul acl set-agent-token -token=${CONSUL_MGMT_TOKEN} agent "76967b87-ceb4-ca3a-85c6-b2b4033eda4c"
  1. Add token to consul configuration
sudo nano /etc/consul.d/consul.hcl

### OUPUT
datacenter             = "saigon"
data_dir               = "/opt/consul"
......
acl {
  enabled                  = true
  default_policy           = "deny" # Change allow to deny
  enable_token_persistence = true
  tokens {
    agent = "76967b87-ceb4-ca3a-85c6-b2b4033eda4c"
  }
}
  1. Create Consul ACL for Nomad
  • Make configurations
cd ~ && \
echo " 
agent_prefix \"\" { 
 policy = \"read\" 
} 
node_prefix \"\" { 
 policy = \"read\" 
} 
service_prefix \"\" { 
 policy = \"write\" 
} 
acl = \"write\" 
" | sudo tee -a  nomad-server-policy.hcl
consul acl policy create -name "nomad-server" -description "Nomad Server Policy" -rules @nomad-server-policy.hcl
echo " 
agent_prefix \"\" { 
 policy = \"read\" 
}
node_prefix \"\" { 
 policy = \"read\" 
} 
service_prefix \"\" { 
 policy = \"write\" 
} 
" | sudo tee -a  nomad-client-policy.hcl
consul acl policy create -name "nomad-client" -description "Nomad Client Policy" -rules @nomad-client-policy.hcl
  • Create token
consul acl token create -description "Nomad Agent Token" -policy-name "nomad-server" -policy-name "nomad-client" | tee nomad-agent.token

##### OUTPUT #####
nomad-agent.token
AccessorID:       903ff2de-025c-5715-f618-666c01365606
SecretID:         4637c1d8-8b40-71b5-cfa1-86a11b8aca6c
Description:      Nomad Agent Token
Local:            false
Create Time:      2022-11-21 13:46:45.482691825 +0700 +07
Policies:
   e5075f22-fc42-ec25-dae1-9541cedf5549 - nomad-server
   b6a6903f-90bb-1c0a-5316-11e6d3151fd3 - nomad-client
  • Apply token ( Do this on all CORE-NODE )
sudo nano /etc/nomad.d/consul.hcl

#### CONTENT ######
consul {
  address             = "127.0.0.1:8500"
  server_service_name = "sg-core-nomad-server-1"
  client_service_name = "sg-core-nomad-client-1"
  auto_advertise      = true
  server_auto_join    = true
  client_auto_join    = true
  token               = "4637c1d8-8b40-71b5-cfa1-86a11b8aca6c" <========== add this line
}
  1. Add agent read for anonymous token
cd ~ && \
echo "
agent_prefix \"\" {
  policy = \"read\"
}
node_prefix \"\" {
  policy = \"read\"
}
service_prefix \"\" {
  policy = \"read\"
}
" | sudo tee  anonymous-read.hcl
consul acl policy create -name "allowed-anonymous-agent-read" -description "Allowed anonymous to read agent " -rules @anonymous-read.hcl

##### OUTPUT #####
ID:           97370dcb-85e7-1eea-c39e-300f11721251 <================= This ID will be used for next command
Name:         allowed-anonymous-agent-read
Description:  Allowed anonymous to read agent 
Datacenters:  
Rules:
 
agent_prefix "" {
  policy = "read"
}
node_prefix "" {
  policy = "read"
}
service_prefix "" {
  policy = "read"
}
consul acl token update -id anonymous -policy-id 97370dcb-85e7-1eea-c39e-300f11721251

##### OUTPUT #####
AccessorID:       00000000-0000-0000-0000-000000000002
SecretID:         anonymous
Description:      Anonymous Token
Local:            false
Create Time:      2022-11-23 10:51:30.161487171 +0700 +07
Policies:
   97370dcb-85e7-1eea-c39e-300f11721251 - allowed-anonymous-agent-read
  1. Restart service (Do this on all CORE-NODE)
sudo systemctl restart consul && sudo systemctl restart nomad

Secure Nomad

  • Create ACL Token
nomad acl bootstrap  -address=http://10.238.22.50:4646

##### OUPUT #####
Accessor ID  = e9aec4a6-c462-3731-c8ce-c55c0bcbbc33
Secret ID    = 5f4899ed-321b-e641-b032-0fa08d417fe9  <=========== Take note for this key
Name         = Bootstrap Token
Type         = management
Global       = true
Create Time  = 2022-11-21 07:10:38.641009746 +0000 UTC
Expiry Time  = <none>
Create Index = 210
Modify Index = 210
Policies     = n/a
Roles        = n/a
  • Add the token to bash env .bashrc ( all CORE-NODE)
echo 'export NOMAD_TOKEN="5f4899ed-321b-e641-b032-0fa08d417fe9"' | tee -a ~/.bashrc && source ~/.bashrc
  • Test
nomad node status --address=http://10.238.22.50:4646

##### OUTPUT #####
ID        DC      Name              Class  Drain  Eligibility  Status
febbd751  saigon  sg-server-core-2  core   false  eligible     ready
292504c3  saigon  sg-server-core-3  core   false  eligible     ready
6a69a5fe  saigon  sg-server-core-1  core   false  eligible     ready

Take a look to create concrete ACL for your system NOMAD ACL

SETUP Agent Node

[Name: sg-server-monitor <-> IP: 10.238.22.193]

  1. Copy backup certs from sg-server-core-1 to sg-server-monitor

Don't need to copy consul-agent-ca-key.pem

sudo cp -R ~/certs/* /opt/consul/certs
sudo chown -R consul:consul /opt/consul && sudo chmod a+r -R /opt/consul/certs
  1. Configuration
  • /etc/consul.d/consul.hcl
sudo nano /etc/consul.d/consul.hcl

##### content #####
datacenter             = "saigon"
node_name              = "sg-server-monitor"
data_dir               = "/opt/consul"
encrypt                = "+dnwtBflWAtk1QxpKjFS463Ytxd1VZdtjupwYxw1qgg=" ### Encrypt key from Step 1
verify_incoming        = true
verify_outgoing        = true
verify_server_hostname = true
ca_file                = "/opt/consul/certs/consul-agent-ca.pem"
cert_file              = "/opt/consul/certs/saigon-server-consul-0.pem"
key_file               = "/opt/consul/certs/saigon-server-consul-0-key.pem"

retry_join = ["10.238.22.122", "10.238.22.182", "10.238.22.50"] ### List of all consul server ( CORE NODES )

acl {
  enabled                  = true
  default_policy           = "deny"
  enable_token_persistence = true
  tokens {
    agent = "76967b87-ceb4-ca3a-85c6-b2b4033eda4c"
  }
}

performance {
  raft_multiplier = 1
}
  • /etc/consul.d/server.hcl
sudo nano /etc/consul.d/server.hcl

##### content #####
server           = false
bind_addr        = "10.238.22.193"
client_addr      = "0.0.0.0"
connect {
  enabled = true
}
addresses {
  grpc = "127.0.0.1"
}
ports {
  grpc = 8502
}
ui_config {
  enabled = true
}
  1. Start service
sudo systemctl restart consul
sudo systemctl status consul

Setup nomad

  1. Configuration
  • /etc/nomad.d/nomad.hcl
sudo nano /etc/nomad.d/nomad.hcl

##### content #####
datacenter = "saigon"
data_dir   = "/opt/nomad"
bind_addr  = "10.238.22.193"
acl {
  enabled = true
}
telemetry {
  collection_interval = "1s"
  disable_hostname = true
  prometheus_metrics = true
  publish_allocation_metrics = true
  publish_node_metrics = true
}
plugin "docker" {
  config {
    endpoint = "unix:///var/run/docker.sock"
    volumes {
      enabled      = true
      selinuxlabel = "z"
    }
    extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"]

    gc {
      image       = true
      image_delay = "10m"
      container   = true

      dangling_containers {
        enabled        = true
        dry_run        = false
        period         = "5m"
        creation_grace = "5m"
      }
    }
    allow_privileged = true
    
  }
}
  • /etc/nomad.d/server.hcl
sudo nano /etc/nomad.d/server.hcl

##### content #####
server {
  enabled          = false
}
  • /etc/nomad.d/client.hcl
sudo nano /etc/nomad.d/client.hcl

##### content #####
client {
  enabled    = true
  node_class = "monitor"
  server_join {
    #NOMAD SERVER LIST
    retry_join = ["10.238.22.122:4647", "10.238.22.182:4647", "10.238.22.50:4647"]
  }
}
  • /etc/nomad.d/consul.hcl
sudo nano /etc/nomad.d/consul.hcl

##### content #####
consul {
  address             = "127.0.0.1:8500"
  client_service_name = "sg-monitor-nomad-client-1"
  auto_advertise      = true
  client_auto_join    = true
  token               = "4637c1d8-8b40-71b5-cfa1-86a11b8aca6c"
}
  1. Start service
echo 'export NOMAD_TOKEN="5f4899ed-321b-e641-b032-0fa08d417fe9"' | tee -a ~/.bashrc && source ~/.bashrc
sudo systemctl restart nomad && sudo systemctl status nomad

[Name: sg-server-agent-1<-> IP: 10.238.22.35]

  1. Copy backup certs from sg-server-core-1 to sg-server-agent-1

Dont need to copy consul-agent-ca-key.pem

sudo cp -R ~/certs/* /opt/consul/certs
sudo chown -R consul:consul /opt/consul && sudo chmod a+r -R /opt/consul/certs
  1. Configuration
  • /etc/consul.d/consul.hcl
sudo nano /etc/consul.d/consul.hcl

##### content #####
datacenter             = "saigon"
node_name              = "sg-agent-consul-1"
data_dir               = "/opt/consul"
encrypt                = "+dnwtBflWAtk1QxpKjFS463Ytxd1VZdtjupwYxw1qgg=" ### Encrypt key from Step 1
verify_incoming        = true
verify_outgoing        = true
verify_server_hostname = true
ca_file                = "/opt/consul/certs/consul-agent-ca.pem"
cert_file              = "/opt/consul/certs/saigon-server-consul-0.pem"
key_file               = "/opt/consul/certs/saigon-server-consul-0-key.pem"

retry_join = ["10.238.22.122", "10.238.22.182", "10.238.22.50"] ### List of all consul server ( CORE NODES )

acl {
  enabled                  = true
  default_policy           = "deny"
  enable_token_persistence = true
  tokens {
    agent = "76967b87-ceb4-ca3a-85c6-b2b4033eda4c"
  }
}

performance {
  raft_multiplier = 1
}
  • /etc/consul.d/server.hcl
sudo nano /etc/consul.d/server.hcl

##### content #####
server           = false
bind_addr        = "10.238.22.35"
client_addr      = "0.0.0.0"
connect {
  enabled = true
}
addresses {
  grpc = "127.0.0.1"
}
ports {
  grpc = 8502
}
ui_config {
  enabled = true
}
  1. Start service
sudo systemctl restart consul
sudo systemctl status consul

Setup nomad

  1. Configuration
  • /etc/nomad.d/nomad.hcl
sudo nano /etc/nomad.d/nomad.hcl

##### content #####
datacenter = "saigon"
data_dir   = "/opt/nomad"
bind_addr  = "10.238.22.35"
acl {
  enabled = true
}
telemetry {
  collection_interval = "1s"
  disable_hostname = true
  prometheus_metrics = true
  publish_allocation_metrics = true
  publish_node_metrics = true
}
plugin "docker" {
  config {
    endpoint = "unix:///var/run/docker.sock"
    volumes {
      enabled      = true
      selinuxlabel = "z"
    }
    extra_labels = ["job_name", "job_id", "task_group_name", "task_name", "namespace", "node_name", "node_id"]

    gc {
      image       = true
      image_delay = "10m"
      container   = true

      dangling_containers {
        enabled        = true
        dry_run        = false
        period         = "5m"
        creation_grace = "5m"
      }
    }
    allow_privileged = true
    
  }
}
  • /etc/nomad.d/server.hcl
sudo nano /etc/nomad.d/server.hcl

##### content #####
server {
  enabled          = false
}
  • /etc/nomad.d/client.hcl
sudo nano /etc/nomad.d/client.hcl

##### content #####
client {
  enabled    = true
  node_class = "agent"
  server_join {
    #NOMAD SERVER LIST
    retry_join = ["10.238.22.122:4647", "10.238.22.182:4647", "10.238.22.50:4647"]
  }
}
  • /etc/nomad.d/consul.hcl
sudo nano /etc/nomad.d/consul.hcl

##### content #####
consul {
  address             = "127.0.0.1:8500"
  client_service_name = "sg-agent-nomad-client-1"
  auto_advertise      = true
  client_auto_join    = true
  token               = "4637c1d8-8b40-71b5-cfa1-86a11b8aca6c"
}
  1. Start service
echo 'export NOMAD_TOKEN="5f4899ed-321b-e641-b032-0fa08d417fe9"' | tee -a ~/.bashrc && source ~/.bashrc
sudo systemctl restart nomad && sudo systemctl status nomad

OTHER AGENT NODES WOULD BE THE SAME SETUP

RESULT services node

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment