NGINX Controller v3 Install

agent-fix.sh

#!/bin/bash

### Run this on the nplus instances to fix the issue where they start before the controller is running

sudo mkdir -p /etc/systemd/system/controller-agent.service.d/
{ echo "[Service]";
  echo "Restart=on-failure";
  echo "RestartSec=60s";
} | sudo tee /etc/systemd/system/controller-agent.service.d/10-restart-delay.conf
sudo systemctl daemon-reload

echo "To start this without it waiting run the following"
echo "sudo systemctl --no-block start controller-agent.service"

cleandatabase.sh

#!/bin/bash
for i in common data nats system vault naas; do sudo -u postgres psql -c "DROP DATABASE $i"; done
sudo -u postgres psql -c "CREATE DATABASE naas WITH ENCODING='UTF8'"

database-install.sh

#!/bin/bash

if [ "$EUID" -eq 0 ]
  then echo "Please run as a user and not root"
  exit
fi

# Update hosts and check these
controllerip="10.1.1.4"
databaseip="10.1.1.5"

if grep -q "controller.f5demo.udf" /etc/hosts
then
    echo "controller in hosts already"
else
    sudo echo "$controllerip controller.f5demo.udf" >> /etc/hosts
fi

if grep -q "database.f5demo.udf" /etc/hosts
then
    echo "database in hosts already"
else
    sudo echo "$databaseip database.f5demo.udf" >> /etc/hosts
fi

# set hostname
sudo hostnamectl set-hostname --static database.f5demo.udf

sudo yum -y update
#reboot if kernel is updated

sudo yum -y install vim wget git curl bash-completion yum-utils

### Install Dependencies - jq
sudo rpm -Uvh http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/o/oniguruma-5.9.5-3.el7.x86_64.rpm
sudo rpm -Uvh http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/j/jq-1.5-1.el7.x86_64.rpm

# Set selinux permissive
sudo setenforce Permissive
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

### Firewall
# Install if not there
sudo yum -y install firewalld
sudo systemctl enable firewalld --now
sudo firewall-cmd --add-service=ssh --permanent
sudo firewall-cmd --add-port=5432/tcp --permanent
sudo firewall-cmd --reload

# Install postgresql
sudo rpm -ivh https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
sudo yum -y groups install "PostgreSQL Database Server 9.5 PGDG"
sudo /usr/pgsql-9.5/bin/postgresql95-setup initdb

# Configure access we enable all 10.* addresses for naas user
sudo echo "host    all             naas            10.0.0.0/8              md5" >> /var/lib/pgsql/9.5/data/pg_hba.conf
sudo sed -i "s/#listen_addresses = 'localhost'/listen_addresses = '*'/" /var/lib/pgsql/9.5/data/postgresql.conf

sudo systemctl enable postgresql-9.5 --now

# configure the naas db and user
sudo -u postgres psql -c "CREATE ROLE naas CREATEDB LOGIN"
sudo -u postgres psql -c "ALTER ROLE naas WITH PASSWORD 'foobar'"
sudo -u postgres psql -c "CREATE DATABASE naas WITH ENCODING='UTF8'"

# Ready for install

install-controller.sh

#!/bin/bash

if [ "$EUID" -eq 0 ]
  then echo "Please run as a user and not root"
  exit
fi

### Variables
# Update hosts and check these
controllerip="10.1.1.4"
databaseip="10.1.1.5"
adminpassword="P@ssw0rd20"

if grep -q "controller.f5demo.udf" /etc/hosts
then
    echo "controller in hosts already"
else
    sudo echo "$controllerip controller.f5demo.udf" >> /etc/hosts
fi

if grep -q "database.f5demo.udf" /etc/hosts
then
    echo "database in hosts already"
else
    sudo echo "$databaseip database.f5demo.udf" >> /etc/hosts
fi

# set hostname
if [ "$HOSTNAME" = "controller.f5demo.ud" ]; then
    printf '%s\n' "hostname already set"
else
    sudo hostnamectl set-hostname --static controller.f5demo.udf
fi

sudo yum -y update
#reboot if kernel is updated

sudo yum -y install vim wget git curl bash-completion \
  yum-utils \
  device-mapper-persistent-data \
  lvm2

### Install Dependencies - jq
sudo rpm -Uvh http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/o/oniguruma-5.9.5-3.el7.x86_64.rpm
sudo rpm -Uvh http://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/j/jq-1.6-1.el7.x86_64.rpm

### Install Docker
sudo yum install -y docker

# Modify the docker sysconfig file NOT the daemon.json (they will conflict)
sudo sed -i 's/journald/json-file --log-opt max-size=10m --log-opt max-file=3/g' /etc/sysconfig/docker
sudo systemctl daemon-reload
sudo systemctl enable docker --now

# Set selinux permissive
sudo setenforce Permissive
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

# Disable swap (check with free -h)
sudo swapoff -a
sudo sed -i 's/^\(.*swap.*\)$/#\1/' /etc/fstab 

# load netfilter probe specifically
sudo modprobe br_netfilter

# Enable IP forwarding
sudo bash -c 'cat <<EOF >  /etc/sysctl.d/90-k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF'
sudo sysctl --system


### Firewall
# Install if not there
sudo yum -y install firewalld
sudo systemctl enable firewalld --now
sudo firewall-cmd --add-service=ssh

# Open firewall ports
sudo firewall-cmd --add-masquerade --permanent

for service in http https ssh
do
  sudo firewall-cmd --permanent --add-service=$service
done

for port in 8080 6443 8443 10250
do
  sudo firewall-cmd --permanent --add-port=$port/tcp
done

sudo firewall-cmd --reload

echo "Now installing Controller...."

cd controller-installer
yes n|./install.sh -y -m localhost -x 25 -b false -g false -d $databaseip -k 5432 -r naas -s foobar -j noreply@f5demo.udf -e admin@f5demo.udf -p $adminpassword -f controller.f5demo.udf -c -a F5Demo -t Hawk -u Man --tsdb-volume-type local -o

# Add Agent Install
export API_KEY=$(curl -k -X POST -d '{"email":"admin@f5demo.udf", "password":"P@ssw0rd20"}' -H "Content-Type: application/json" https://localhost/sapi/auth/login/ | jq -r .api_key)

# Install Ansible
sudo rpm -Uvh https://releases.ansible.com/ansible/rpm/release/epel-7-x86_64/ansible-2.9.6-1.el7.ans.noarch.rpm

# Add License
ansible-playbook ../nginx_controller_license.yml

# Install agent
# v3.1 doesn't work well with a reinstall of controller and agent so we
# need to really clear out the agent stuff
for i in {1..4}
do
    ssh nplus$i sudo yum remove nginx-controller-agent nginx-plus-module-metrics -y
    ssh nplus$i sudo rm -rf /etc/controller-agent/
    ssh nplus$i sudo rm -f /etc/yum.repos.d/nginx-controller-metrics.repo /etc/yum.repos.d/nginx-controller.repo
    ssh nplus$i wget --no-check-certificate https://controller.f5demo.udf:8443/1.4/install/controller/ -O /home/centos/install.sh
    ssh -t nplus$i API_KEY=$API_KEY sh -l /home/centos/install.sh -y
    ssh nplus$i /home/centos/agent-fix.sh
done

# Lock kubectl so it doesn't update to incompatible version
#sudo yum -y install yum-versionlock
#sudo yum versionlock add kubectl kubelet kubernetes-cni kubeadm

nginx_controller_license.yml

---
# ansible-playbook nginx_controller_license.yaml -i controller -e "controller_user_email=user@company.com controller_password=userPassword" 
- hosts: localhost

  tasks:
  #- name: get controller internal DNS, written to inventory file
  #  shell: cat "{{playbook_dir}}/controller"
  #  register: controller

  - name: set the controller private fqdn
    set_fact:
      controller_dns: "controller.f5demo.udf"

  - debug:
      var: controller_dns

- hosts: controller
  remote_user: centos
  become: true
  become_method: sudo
  gather_facts: yes

  vars:
    # base64 encoded, one line, no line endings or carrage returns
    license: "{{ lookup('file', 'license-encoded.txt') }}"

  tasks:
  - name: login to Controller
    uri:
      url: "https://{{hostvars['localhost']['controller_dns']}}/api/v1/platform/login"
      method: "POST"
      body: 
        credentials:
          type: "BASIC"
          #username: "{{controller_user_email}}"
          #password: "{{controller_password}}"
          username: "admin@f5demo.udf"
          password: "P@ssw0rd20"
      body_format: json
      return_content: yes
      status_code: 204
      validate_certs: false
    register: auth

  - name: POST controller license
    uri:
      url: "https://{{hostvars['localhost']['controller_dns']}}/api/v1/platform/license-file"
      method: POST
      body: 
        content: "{{ license }}"
      body_format: json
      return_content: yes
      status_code: 200
      validate_certs: false
      headers:
        Cookie: "{{auth.set_cookie}}"
    register: controller_response

  - name: GET controller license status
    uri:
      url: "https://{{hostvars['localhost']['controller_dns']}}/api/v1/platform/license"
      method: GET
      status_code: 200
      validate_certs: false
      headers:
        Cookie: "{{auth.set_cookie}}"
    register: controller_license

reset-controller.sh

#!/bin/bash
./uninstall-controller.sh
./install-controller.sh

uninstall-controller.sh

#!/bin/bash
cd ~/controller-installer
./uninstall.sh --yes-delete-controller --yes-delete-database --yes-delete-k8s
cd ..
kubeadm reset
sudo yum remove -y kubeadm kubectl kubelet kubernetes-cni kube*
sudo rm -rf ~/.kube
ssh database ~/cleandatabase.sh