ccamacho/deploy-offline-cdk-with-lxd.sh

## deploy-offline-cdk-with-lxd.sh
#!/usr/bin/env bash

set -e
set -u

## Variables
set_vars(){
    APT_MIRROR_HOST="mirror"
    LXDKVM_SSTREAM_HOST="mirror"
    JUJU_SSTREAM_HOST="mirror"
    BOOTSTRAP_NODE_IP=10.10.0.20
    CDK_SHRINKWRAP_PATH=$(ls -dt $HOME/cdk-shrinkwrap/*/ | head -1)
    PRIV_REGISTRY_HOST="mirror:5000"
    MACHINE_IPS="10.10.0.21 \
    10.10.0.22 \
    10.10.0.23 \
    10.10.0.24 \
    10.10.0.25 \
    10.10.0.26 \
    10.10.0.27 \
    10.10.0.28 \
    10.10.0.29 \
    10.10.0.30 \
    10.10.0.31"
    KUBE_VER="1.11/stable"
    ETCD_VER="3.2/stable"
    SERIES=xenial
    CERT_PATH="/etc/pki/tls/certs/"
    CERT_FILE="mirror.crt"
    FAN_CONFIG="10.10.0.0/24=252.0.0.0/8"
}

check_pkg(){
  if dpkg --get-selections | grep -q "^$1[[:space:]]*install$" >/dev/null; then
    return
  else
    echo "package $1 is missing."
    exit 1
  fi
}

check_prerequisite(){
   check_pkg "jq"
   check_pkg "juju-2.0"
}

## Bootstrap juju
juju_bootstrap(){
    tee ~/mycloud.yaml > /dev/null << EOL
clouds:
  manual:
    type: manual
    endpoint: $BOOTSTRAP_NODE_IP
EOL
    juju add-cloud --replace manual ~/mycloud.yaml
    echo "bootstrapping juju..."
    ssh-keyscan $BOOTSTRAP_NODE_IP
    sleep 2
    ssh-keyscan $BOOTSTRAP_NODE_IP > ~/.ssh/known_hosts
    juju bootstrap --no-gui \
    --bootstrap-series=$SERIES \
    --config apt-mirror=http://${APT_MIRROR_HOST}/archive.ubuntu.com/ubuntu/ \
    --config agent-stream=release \
    --config container-image-metadata-url=https://${LXDKVM_SSTREAM_HOST}/lxdkvm/_latest \
    --config agent-metadata-url=https://${JUJU_SSTREAM_HOST}/juju/ \
    --debug manual manual-controller > juju-bootstrap.log 2>&1
    juju model-config fan-config=$FAN_CONFIG
    juju model-config container-networking-method=fan
}

## add machines
juju_add_machines(){
    for machine in $MACHINE_IPS
    do
      ssh-keyscan $machine
      sleep 2
      ssh-keyscan $machine >> ~/.ssh/known_hosts
      juju add-machine --debug ssh:ubuntu@$machine > juju-add-machine-$machine.log 2>&1 &
    done
    sleep 60
    ## prepare machines for deploy
    len=$(wc -w <<< "$MACHINE_IPS")
    for id in $(seq 0 $(expr $len - 1))
    do
      while [ $(juju status --format json | jq -r ".machines[\"$id\"].\"juju-status\".current") != "started" ]
      do
        echo "waiting for machine $id to be ready..."
        sleep 5
      done
      echo "machine $id is ready. configuring machine now"
      juju scp ${CDK_SHRINKWRAP_PATH}/resources/core.snap $id:
      juju scp ${CERT_PATH}${CERT_FILE} $id:
    done
    juju run --all "sudo snap install --dangerous /home/ubuntu/core.snap"
    juju run --all "sudo mv /home/ubuntu/$CERT_FILE /usr/local/share/ca-certificates/"
    juju run --all "sudo update-ca-certificates"
    # juju won't replace securiy.ubuntu.com to value in --apt-mirror
    juju run --all -- sudo sed -i 's/security.ubuntu.com/${APT_MIRROR_HOST}\/archive.ubuntu.com/g' /etc/apt/sources.list

}

# deploy all
juju_deploy(){
    # deploy etcd
    juju deploy -n 3 --resource etcd=${CDK_SHRINKWRAP_PATH}/resources/etcd/etcd.snap --resource snapshot=${CDK_SHRINKWRAP_PATH}/resources/etcd/snapshot.gz \
    --to 0,1,2 ${CDK_SHRINKWRAP_PATH}/charms/etcd --config ~/cdk-config.yaml
    # deploy kubernetes-master
    juju deploy -n 2 --resource cdk-addons=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/cdk-addons.snap \
    --resource kube-apiserver=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-apiserver.snap \
    --resource kube-controller-manager=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-controller-manager.snap \
    --resource kube-scheduler=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-scheduler.snap \
    --resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kubectl.snap --to 3,4 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-master --config ~/cdk-config.yaml
    # deploy flannel
    juju deploy --resource flannel-amd64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-amd64.gz --resource flannel-arm64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-arm64.gz \
    --resource flannel-s390x=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-s390x.gz ${CDK_SHRINKWRAP_PATH}/charms/flannel
    # deploy easyrsa
    juju deploy -n 1 --resource easyrsa=${CDK_SHRINKWRAP_PATH}/resources/easyrsa/easyrsa.tgz --to lxd:8 ${CDK_SHRINKWRAP_PATH}/charms/easyrsa
    # deploy kubernetes-worker
    juju deploy -n 5 --resource cni-amd64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-amd64.tgz --resource cni-arm64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-arm64.tgz \
    --resource cni-s390x=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-s390x.tgz --resource kube-proxy=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kube-proxy.snap \
    --resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubectl.snap --resource kubelet=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubelet.snap \
    --to 5,6,7 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-worker --config ~/cdk-config.yaml
    # deploy ceph-osd
    juju deploy -n 3 --to 5,6,7 $CDK_SHRINKWRAP_PATH/charms/ceph-osd
    # deploy ceph-mon
    juju deploy -n 3 --to lxd:5,lxd:6,lxd:7 $CDK_SHRINKWRAP_PATH/charms/ceph-mon
    # deploy kubeapi-load-balancer
    juju deploy -n 1  --to 8 ${CDK_SHRINKWRAP_PATH}/charms/kubeapi-load-balancer
    # deploy prometheus2
    juju deploy -n 1 --resource prometheus=${CDK_SHRINKWRAP_PATH}/resources/prometheus/prometheus.snap --to 9 ${CDK_SHRINKWRAP_PATH}/charms/prometheus2
    # deploy grafana
    juju deploy -n 1 --to lxd:9 ${CDK_SHRINKWRAP_PATH}/charms/grafana --config ~/cdk-config.yaml
    # deploy telegraf
    juju deploy ${CDK_SHRINKWRAP_PATH}/charms/telegraf --config ~/cdk-config.yaml
    # deploy elasticsearch
    juju deploy -n 1 --to lxd:8 ${CDK_SHRINKWRAP_PATH}/charms/elasticsearch --config ~/cdk-config.yaml
    # deploy filebeat
    juju deploy ${CDK_SHRINKWRAP_PATH}/charms/filebeat --config ~/cdk-config.yaml
    # deploy graylog
    juju deploy -n 1 --resource graylog=${CDK_SHRINKWRAP_PATH}/resources/graylog/graylog.snap --to 10 ${CDK_SHRINKWRAP_PATH}/charms/graylog
    # deploy mongodb
    juju deploy -n 1 --to lxd:10 ${CDK_SHRINKWRAP_PATH}/charms/mongodb
    # deploy apache2
    juju deploy -n 1 --to lxd:10 ${CDK_SHRINKWRAP_PATH}/charms/apache2 --config ~/cdk-config.yaml
}


make_config_yaml(){
    tee ~/cdk-config.yaml > /dev/null << EOL
kubernetes-master:
  channel: $KUBE_VER
kubernetes-worker:
  kubelet-extra-args: "pod-infra-container-image=$PRIV_REGISTRY_HOST/google_containers/pause-amd64:3.1"
  docker-opts: "--insecure-registry=$PRIV_REGISTRY_HOST"
  nginx-image: "$PRIV_REGISTRY_HOST/nginx-ingress-controller:0.16.1"
  default-backend-image: "$PRIV_REGISTRY_HOST/defaultbackend:1.4"
  channel: $KUBE_VER
etcd:
  channel: $ETCD_VER
apache2:
  enable_modules: "headers proxy_html proxy_http"
grafana:
  install_sources: "deb http://${APT_MIRROR_HOST}/packagecloud.io/grafana/stable/debian/ stretch main"
elasticsearch:
  apt-key-url: "http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch"
  apt-repository: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
filebeat:
  logpath: '/var/log/*.log /var/log/containers/*.log'
  install_sources: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
  install_keys: $(echo -e "|\n      - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch | sed -e 's/^/         /')")
telegraf:
  install_sources: "deb http://${APT_MIRROR_HOST}/ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main"
  install_keys: $(echo -e "|\n      - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-telegraf | sed -e 's/^/          /')")
EOL
}


juju_add_relations(){
    # add relations
    juju relate kubernetes-master:kube-api-endpoint kubeapi-load-balancer:apiserver
    juju relate kubernetes-master:loadbalancer kubeapi-load-balancer:loadbalancer
    juju relate kubernetes-master:kube-control kubernetes-worker:kube-control
    juju relate kubernetes-master:certificates easyrsa:client
    juju relate etcd:certificates easyrsa:client
    juju relate kubernetes-master:etcd etcd:db
    juju relate kubernetes-worker:certificates easyrsa:client
    juju relate kubernetes-worker:kube-api-endpoint kubeapi-load-balancer:website
    juju relate kubeapi-load-balancer:certificates easyrsa:client
    juju relate flannel:etcd etcd:db
    juju relate flannel:cni kubernetes-master:cni
    juju relate flannel:cni kubernetes-worker:cni
    juju relate apache2:reverseproxy graylog:website
    juju relate graylog:elasticsearch elasticsearch:client
    juju relate graylog:mongodb mongodb:database
    juju relate filebeat:beats-host kubernetes-master:juju-info
    juju relate filebeat:beats-host kubernetes-worker:juju-info
    juju relate filebeat:logstash graylog:beats
    juju relate prometheus2:grafana-source grafana:grafana-source
    juju relate telegraf:prometheus-client prometheus2:target
    juju relate kubernetes-master:juju-info telegraf:juju-info
    juju relate kubernetes-worker:juju-info telegraf:juju-info
    juju relate ceph-mon:osd ceph-osd:mon
    juju relate kubernetes-master:ceph-storage ceph-mon:admin
}

adjust_kube_master(){
    # kubernetes-master charm doesn't support to change registry address for cdk-addons
    kube_master_machines=$(juju status kubernetes-master --format json | jq -r '.machines|length')
    installed_cnt=0
    while [ $installed_cnt -ne $kube_master_machines ]
    do
      sleep 10
      installed_cnt=$(juju run --application kubernetes-master "snap list | grep cdk-addons" | grep -c cdk-addons || true)
    done
    juju run --application kubernetes-master -- snap set cdk-addons registry="$PRIV_REGISTRY_HOST"
}

# main
set_vars
check_prerequisite
juju_bootstrap
juju_add_machines
make_config_yaml
juju_deploy
juju_add_relations

# after deployment
echo "sleep for 3 minutes..."
sleep 180
adjust_kube_master
# following is required when deploying charms that uses snap resources to lxd container, because core.snap is not installed in lxd containers
#juju run --machine $id -- "sudo find /var/lib/lxd/containers/ -path "/var/lib/lxd/containers/juju-*/rootfs/home/ubuntu" -type d -exec cp /home/ubuntu/core.snap {} \;"
#juju run --machine $id -- "sudo find /var/lib/lxd/containers/ -path "/var/lib/lxd/containers/juju-*/rootfs/home/ubuntu" -type d -exec cp /home/ubuntu/core.snap {} \;"
#juju run --application prometheus2 "sudo snap install --dangerous /home/ubuntu/core.snap"
#juju run --application graylog "sudo snap install --dangerous /home/ubuntu/core.snap"

echo "disk devices are still not set in ceph-osd. ex) set devices by,"
echo "juju config ceph-osd osd-devices=\"/dev/sdb /dev/sdc\""

## deploy-offline-cdk.sh
#!/usr/bin/env bash

set -e
set -u

## Variables
set_vars(){
    APT_MIRROR_HOST=10.12.1.2
    LXDKVM_SSTREAM_HOST=10.12.1.2
    JUJU_SSTREAM_HOST=10.12.1.2
    BOOTSTRAP_NODE_IP=10.12.1.20
    CDK_SHRINKWRAP_PATH=$(ls -dt $HOME/cdk-shrinkwrap/*/ | head -1)
    PRIV_REGISTRY_HOST=10.12.1.2:5000
    MACHINE_IPS="10.12.1.21 \
    10.12.1.22 \
    10.12.1.23 \
    10.12.1.24 \
    10.12.1.25 \
    10.12.1.26 \
    10.12.1.27 \
    10.12.1.28 \
    10.12.1.29 \
    10.12.1.30 \
    10.12.1.31 \
    10.12.1.32"
    KUBE_VER="1.11/stable"
    ETCD_VER="3.2/stable"
    SERIES=xenial
}


## Bootstrap juju
juju_bootstrap(){
    tee ~/mycloud.yaml > /dev/null << EOL
clouds:
  manual:
    type: manual
    endpoint: $BOOTSTRAP_NODE_IP
EOL
    juju add-cloud --replace manual ~/mycloud.yaml
    echo "bootstrapping juju..."
    ssh-keyscan $BOOTSTRAP_NODE_IP
    sleep 2
    ssh-keyscan $BOOTSTRAP_NODE_IP > ~/.ssh/known_hosts
    juju bootstrap --no-gui \
    --bootstrap-series=$SERIES \
    --config apt-mirror=http://${APT_MIRROR_HOST}/archive.ubuntu.com/ubuntu/ \
    --config agent-stream=release \
    --config container-image-metadata-url=https://${LXDKVM_SSTREAM_HOST}/lxdkvm/ \
    --config agent-metada-url=${JUJU_SSTREAM_HOST}/juju/ \
    --debug manual manual-controller > juju-bootstrap.log 2>&1
}

## add machines
juju_add_machines(){
    for machine in $MACHINE_IPS
    do
    ssh-keyscan $machine
    sleep 2
    ssh-keyscan $machine >> ~/.ssh/known_hosts
    juju add-machine --debug ssh:ubuntu@$machine > juju-add-machine-$machine.log 2>&1
    done


    ## prepare machines for deploy
    len=$(wc -w <<< "$MACHINE_IPS")
    for id in $(seq 0 $(expr $len - 1))
    do
    juju scp ${CDK_SHRINKWRAP_PATH}/resources/core.snap $id:
    #juju scp ${CDK_SHRINKWRAP_PATH}/resources/lxd.snap $id:
    juju run --machine $id "sudo snap install --dangerous /home/ubuntu/core.snap"
    #juju run --machine $id "sudo snap install --dangerous /home/ubuntu/lxd.snap"
    done
}

# deploy all
juju_deploy(){
    # deploy etcd
    juju deploy -n 3 --resource etcd=${CDK_SHRINKWRAP_PATH}/resources/etcd/etcd.snap --resource snapshot=${CDK_SHRINKWRAP_PATH}/resources/etcd/snapshot.gz \
    --to 0,1,2 ${CDK_SHRINKWRAP_PATH}/charms/etcd --config ~/cdk-config.yaml
    # deploy kubernetes-master
    juju deploy -n 2 --resource cdk-addons=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/cdk-addons.snap \
    --resource kube-apiserver=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-apiserver.snap \
    --resource kube-controller-manager=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-controller-manager.snap \
    --resource kube-scheduler=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-scheduler.snap \
    --resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kubectl.snap --to 3,4 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-master --config ~/cdk-config.yaml
    # deploy flannel
    juju deploy --resource flannel-amd64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-amd64.gz --resource flannel-arm64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-arm64.gz \
    --resource flannel-s390x=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-s390x.gz ${CDK_SHRINKWRAP_PATH}/charms/flannel
    # deploy easyrsa
    juju deploy -n 1 --resource easyrsa=${CDK_SHRINKWRAP_PATH}/resources/easyrsa/easyrsa.tgz --to 0 ${CDK_SHRINKWRAP_PATH}/charms/easyrsa
    # deploy kubernetes-worker
    juju deploy -n 5 --resource cni-amd64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-amd64.tgz --resource cni-arm64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-arm64.tgz \
    --resource cni-s390x=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-s390x.tgz --resource kube-proxy=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kube-proxy.snap \
    --resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubectl.snap --resource kubelet=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubelet.snap \
    --to 5,6,7,8,9 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-worker --config ~/cdk-config.yaml
    # deploy kubeapi-load-balancer
    juju deploy -n 1  --to 10 ${CDK_SHRINKWRAP_PATH}/charms/kubeapi-load-balancer
    # deploy prometheus
    #juju deploy -n 1 --to 10 ${CDK_SHRINKWRAP_PATH}/charms/prometheus
    juju deploy -n 1 --resource prometheus=${CDK_SHRINKWRAP_PATH}/resources/prometheus/prometheus.snap --to 10 ${CDK_SHRINKWRAP_PATH}/charms/prometheus2
    # deploy grafana
    juju deploy -n 1 --to 10 ${CDK_SHRINKWRAP_PATH}/charms/grafana --config ~/cdk-config.yaml
    # deploy telegraf
    juju deploy ${CDK_SHRINKWRAP_PATH}/charms/telegraf --config ~/cdk-config.yaml
    # deploy elasticsearch
    juju deploy -n 1 --to 11 ${CDK_SHRINKWRAP_PATH}/charms/elasticsearch --config ~/cdk-config.yaml
    # deploy filebeat
    juju deploy ${CDK_SHRINKWRAP_PATH}/charms/filebeat --config ~/cdk-config.yaml
    # deploy graylog
    juju deploy -n 1 --resource graylog=${CDK_SHRINKWRAP_PATH}/resources/graylog/graylog.snap --to 11 ${CDK_SHRINKWRAP_PATH}/charms/graylog
    # deploy mongodb
    juju deploy -n 1 --to 11 ${CDK_SHRINKWRAP_PATH}/charms/mongodb
    # deploy apache2
    juju deploy -n 1 --to 11 ${CDK_SHRINKWRAP_PATH}/charms/apache2 --config ~/cdk-config.yaml
    # deploy ceph-osd
    juju deploy -n 5 --to 5,6,7,8,9 $CDK_SHRINKWRAP_PATH/charms/ceph-osd
    # deploy ceph-mon
    juju deploy -n 5 --to 5,6,7,8,9 $CDK_SHRINKWRAP_PATH/charms/ceph-mon
}


make_config_yaml(){
    tee ~/cdk-config.yaml > /dev/null << EOL
kubernetes-master:
  channel: $KUBE_VER
kubernetes-worker:
  kubelet-extra-args: "pod-infra-container-image=$PRIV_REGISTRY_HOST/google_containers/pause-amd64:3.1"
  docker-opts: "--insecure-registry=$PRIV_REGISTRY_HOST"
  nginx-image: "$PRIV_REGISTRY_HOST/nginx-ingress-controller:0.16.1"
  default-backend-image: "$PRIV_REGISTRY_HOST/defaultbackend:1.4"
  channel: $KUBE_VER
etcd:
  channel: $ETCD_VER
apache2:
  enable_modules: "headers proxy_html proxy_http"
grafana:
  install_sources: "deb http://${APT_MIRROR_HOST}/packagecloud.io/grafana/stable/debian/ stretch main"
elasticsearch:
  apt-key-url: "http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch"
  apt-repository: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
filebeat:
  logpath: '/var/log/*.log /var/log/containers/*.log'
  install_sources: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
  install_keys: $(echo -e "|\n      - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch | sed -e 's/^/         /')")
telegraf:
  install_sources: "deb http://${APT_MIRROR_HOST}/ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main"
  install_keys: $(echo -e "|\n      - |\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-telegraf | sed -e 's/^/          /')")
EOL
}


juju_add_relations(){
    # add relations
    juju relate kubernetes-master:kube-api-endpoint kubeapi-load-balancer:apiserver
    juju relate kubernetes-master:loadbalancer kubeapi-load-balancer:loadbalancer
    juju relate kubernetes-master:kube-control kubernetes-worker:kube-control
    juju relate kubernetes-master:certificates easyrsa:client
    juju relate etcd:certificates easyrsa:client
    juju relate kubernetes-master:etcd etcd:db
    juju relate kubernetes-worker:certificates easyrsa:client
    juju relate kubernetes-worker:kube-api-endpoint kubeapi-load-balancer:website
    juju relate kubeapi-load-balancer:certificates easyrsa:client
    juju relate flannel:etcd etcd:db
    juju relate flannel:cni kubernetes-master:cni
    juju relate flannel:cni kubernetes-worker:cni
    juju relate apache2:reverseproxy graylog:website
    juju relate graylog:elasticsearch elasticsearch:client
    juju relate graylog:mongodb mongodb:database
    juju relate filebeat:beats-host kubernetes-master:juju-info
    juju relate filebeat:beats-host kubernetes-worker:juju-info
    juju relate filebeat:logstash graylog:beats
    #juju relate prometheus:grafana-source grafana:grafana-source
    #juju relate telegraf:prometheus-client prometheus:target
    juju relate prometheus2:grafana-source grafana:grafana-source
    juju relate telegraf:prometheus-client prometheus2:target
    juju relate kubernetes-master:juju-info telegraf:juju-info
    juju relate kubernetes-worker:juju-info telegraf:juju-info
    juju relate ceph-mon:osd ceph-osd:mon
    juju relate kubernetes-master:ceph-storage ceph-mon:admin
}


# main
set_vars
juju_bootstrap
juju_add_machines
make_config_yaml
juju_deploy
juju_add_relations

# after deployment
sleep 300
# juju won't replace securiy.ubuntu.com to value in --apt-mirror
juju run --all -- sudo sed -i 's/security.ubuntu.com/10.12.1.2\/archive.ubuntu.com/g' /etc/apt/sources.list
# kubernetes-master charm doesn't support to change registry address for cdk-addons
juju run --application kubernetes-master -- snap set cdk-addons registry="$PRIV_REGISTRY_HOST"
# elasticsearch charm doesn't support to pass GPG key, but only url
#juju run --application elasticsearch -- wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch | sudo apt-key add -

echo "disk devices are still not set in ceph-osd. set devices by,"
echo "juju config ceph-osd osd-devices=\"/dev/sdb /dev/sdc\""

## prepare-offline-cdk.sh
#!/usr/bin/env bash

## Variables
APT_MIRROR_PATH="/var/spool/apt-mirror/"
SSTREAM_PATH="/var/spool/sstreams/"
GPG_KEY_TELEGRAF=C94406F5
GPG_KEY_ELASTICSEARCH=D88E42B4
SERIES=xenial
CERT_PATH="/etc/pki/tls/certs/"
PRIV_KEY_PATH="/etc/pki/tls/private/"
CERT_FILE="mirror.crt"
PRIV_KEY_FILE="mirror.key"

## Install all necessary packages
echo "Installing Missing Packages & Repositories"
sudo apt-add-repository -y ppa:telegraf-devs/ppa
sudo apt-add-repository -y ppa:juju/stable
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
wget -qO - https://packagecloud.io/gpg.key | sudo apt-key add -
sudo apt update && sudo apt install -y apt-mirror docker.io git apache2 python3-pip unzip juju
sudo pip3 install pyyaml
sudo pip3 install pyaml
sudo snap install kubectl --classic
sudo snap install charm

## Setup apt-mirror
echo "Configuring apt-mirror"
sudo tee /etc/apt/mirror.list > /dev/null <<EOL
set base_path    $APT_MIRROR_PATH
set nthreads     20
set _tilde 0

deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse

deb-src http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse

deb http://ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main
deb https://artifacts.elastic.co/packages/5.x/apt stable main
deb https://packagecloud.io/grafana/stable/debian/ stretch main

clean http://archive.ubuntu.com/ubuntu
clean http://ppa.launchpad.net/telegraf-devs/ppa/ubuntu
clean https://artifacts.elastic.co/packages/5.x/apt
clean https://packagecloud.io/grafana/stable/debian/
EOL
# save GPG keys
apt_key_path="$APT_MIRROR_PATH"mirror/keys/
sudo mkdir -p $apt_key_path
apt-key export $GPG_KEY_TELEGRAF | sudo tee "$apt_key_path"GPG-KEY-telegraf
apt-key export $GPG_KEY_ELASTICSEARCH | sudo tee "$apt_key_path"GPG-KEY-elasticsearch


echo "Start syncing Files, this will take few hours"
# syncing docker images
unset -e
sudo docker run -d -p 5000:5000 --restart=always --name registry registry:2
export REGISTRY="localhost:5000"
set -e
sudo docker pull gcr.io/google_containers/pause-amd64:3.1
sudo docker tag gcr.io/google_containers/pause-amd64:3.1 "$REGISTRY"/google_containers/pause-amd64:3.1
sudo docker push "$REGISTRY"/google_containers/pause-amd64:3.1

sudo docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1
sudo docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1 "$REGISTRY"/nginx-ingress-controller:0.16.1
sudo docker push "$REGISTRY"/nginx-ingress-controller:0.16.1

sudo docker pull k8s.gcr.io/defaultbackend:1.4
sudo docker tag k8s.gcr.io/defaultbackend:1.4 "$REGISTRY"/defaultbackend:1.4
sudo docker push "$REGISTRY"/defaultbackend:1.4

sudo docker pull cdkbot/addon-resizer-amd64:1.8.1
sudo docker tag cdkbot/addon-resizer-amd64:1.8.1 ${REGISTRY}/addon-resizer-amd64:1.8.1
sudo docker push ${REGISTRY}/addon-resizer-amd64:1.8.1

sudo docker pull k8s.gcr.io/heapster-amd64:v1.5.3
sudo docker tag k8s.gcr.io/heapster-amd64:v1.5.3 ${REGISTRY}/heapster-amd64:v1.5.3
sudo docker push ${REGISTRY}/heapster-amd64:v1.5.3

sudo docker pull k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
sudo docker tag k8s.gcr.io/heapster-influxdb-amd64:v1.3.3 ${REGISTRY}/heapster-influxdb-amd64:v1.3.3
sudo docker push ${REGISTRY}/heapster-influxdb-amd64:v1.3.3

sudo docker pull k8s.gcr.io/heapster-grafana-amd64:v4.4.3
sudo docker tag k8s.gcr.io/heapster-grafana-amd64:v4.4.3 ${REGISTRY}/heapster-grafana-amd64:v4.4.3
sudo docker push ${REGISTRY}/heapster-grafana-amd64:v4.4.3

sudo docker pull k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10
sudo docker tag k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10 ${REGISTRY}/k8s-dns-kube-dns-amd64:1.14.10
sudo docker push ${REGISTRY}/k8s-dns-kube-dns-amd64:1.14.10

sudo docker pull k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10
sudo docker tag k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10 ${REGISTRY}/k8s-dns-dnsmasq-nanny-amd64:1.14.10
sudo docker push ${REGISTRY}/k8s-dns-dnsmasq-nanny-amd64:1.14.10

sudo docker pull k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10
sudo docker tag k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10 ${REGISTRY}/k8s-dns-sidecar-amd64:1.14.10
sudo docker push ${REGISTRY}/k8s-dns-sidecar-amd64:1.14.10

sudo docker pull k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3
sudo docker tag k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3 ${REGISTRY}/kubernetes-dashboard-amd64:v1.8.3
sudo docker push ${REGISTRY}/kubernetes-dashboard-amd64:v1.8.3

sudo docker pull k8s.gcr.io/metrics-server-amd64:v0.2.1
sudo docker tag k8s.gcr.io/metrics-server-amd64:v0.2.1 ${REGISTRY}/metrics-server-amd64:v0.2.1
sudo docker push ${REGISTRY}/metrics-server-amd64:v0.2.1
# pull nexus for a private registry  (optional)
sudo docker pull sonatype/nexus3:latest
sudo docker tag sonatype/nexus3:latest ${REGISTRY}/nexus3:latest
sudo docker push ${REGISTRY}/nexus3:latest

# pull rancher for management (optional)
sudo docker pull rancher/rancher:latest
sudo docker tag rancher/rancher:latest ${REGISTRY}/rancher:latest
sudo docker push ${REGISTRY}/rancher:latest

# synch apt packages (this will take several hours)
sudo apt-mirror

# synch simplestreams metadata
workdir=${SSTREAM_PATH}juju
sudo sstream-mirror --no-verify --progress --max=1 --path=streams/v1/index2.sjson https://streams.canonical.com/juju/tools/ $workdir 'arch=amd64' 'release~(xenial|bionic)' 'version~(2.2|2.3|2.4)'
workdir=${SSTREAM_PATH}lxdkvm
sudo sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg --progress --max=1 --path=streams/v1/index.json https://cloud-images.ubuntu.com/releases/ $workdir/_latest 'arch=amd64' 'release~(trusty|xenial)' 'ftype~(lxd.tar.xz|squashfs|root.tar.xz|root.tar.gz|disk1.img|.json|.sjson)'

# Running CDK Shrink Wrap
if [ -d cdk-shrinkwrap ]; then
  cd cdk-shrinkwrap
  git pull
else
  git clone https://github.com/juju-solutions/cdk-shrinkwrap.git
  cd cdk-shrinkwrap
fi
./shrinkwrap.py canonical-kubernetes --channel stable
cdk_shrinkwrap_name=$(ls -t canonical-kubernetes-stable-*.tar.gz | head -1 | cut -d'.' -f1)
tar xf ${cdk_shrinkwrap_name}.tar.gz
cdk_shrinkwrap_path=$HOME/cdk-shrinkwrap/${cdk_shrinkwrap_name}

# Pull missing charms
charms="ceph-osd ceph-mon vault prometheus2 prometheus grafana telegraf elasticsearch filebeat graylog mongodb apache2"
for charm in $charms
do
  charm pull $SERIES/$charm ${cdk_shrinkwrap_path}/charms/$charm >> downloaded.txt
done
container_charms="canal calico"
for charm in $container_charms
do
  charm pull cs:~containers/$SERIES/$charm ${cdk_shrinkwrap_path}/charms/$charm >> downloaded.txt
done

# Pull missing snaps for graylog and prometheus
snap download graylog --stable
#snap download prometheus --stable
snap download --channel=2/stable prometheus
mkdir -p ${cdk_shrinkwrap_path}/resources/graylog
mkdir -p ${cdk_shrinkwrap_path}/resources/prometheus
rm *.assert
mv graylog* ${cdk_shrinkwrap_path}/resources/graylog/graylog.snap
mv prometheus* ${cdk_shrinkwrap_path}/resources/prometheus/prometheus.snap

# Configure Mirror to Serve Repo to other nodes
sudo tee /etc/apache2/sites-available/sstreams-mirror.conf > /dev/null <<EOL
<VirtualHost *:443>
    ServerName sstreams.cdk-juju
    ServerAlias *
    DocumentRoot ${SSTREAM_PATH}
    SSLCACertificatePath /etc/ssl/certs
    SSLCertificateFile /etc/pki/tls/certs/mirror.crt
    SSLEngine On
    SSLCertificateKeyFile /etc/pki/tls/private/mirror.key
    LogLevel info
    ErrorLog /var/log/apache2/mirror-lxdkvm-error.log
    CustomLog /var/log/apache2/mirror-lxdkvm-access.log combined
    <Directory ${SSTREAM_PATH}>
    Options Indexes FollowSymLinks
    AllowOverride None
    Require all granted
    </Directory>
</VirtualHost>
EOL

sudo tee /etc/apache2/sites-available/ubuntu-mirror.conf > /dev/null <<EOL
<VirtualHost *:80>
    ServerName cdk-juju
    DocumentRoot ${APT_MIRROR_PATH}mirror/

    LogLevel info
    ErrorLog /var/log/apache2/mirror-error.log
    CustomLog /var/log/apache2/mirror-access.log combined
    <Directory $APT_MIRROR_PATH>
    Options Indexes FollowSymLinks
    AllowOverride None
    Require all granted
    </Directory>

</VirtualHost>
EOL

# Generate SSL cert to be used by apache
sudo mkdir -p $PRIV_KEY_PATH
sudo mkdir -p $CERT_PATH

PRIMARYIP=`hostname -i`
sudo tee /root/$HOSTNAME.conf > /dev/null <<EOL
[ req ]
prompt = no
default_bits = 4096
distinguished_name = req_distinguished_name
req_extensions = req_ext
[ req_distinguished_name ]
C=GB
ST=London
L=London
O=Canonical
OU=Canonical
CN=$HOSTNAME
[ req_ext ]
subjectAltName = @alt_names
[alt_names]
DNS.1 = $HOSTNAME
DNS.2 = $PRIMARYIP
IP.1 = $PRIMARYIP
EOL

sudo openssl req \
    -new \
    -newkey rsa:4096 \
    -days 3650 \
    -nodes \
    -x509 \
    -config /root/$HOSTNAME.conf \
    -keyout ${PRIV_KEY_PATH}${PRIV_KEY_FILE} \
    -out ${CERT_PATH}${CERT_FILE}

sudo a2enmod ssl
sudo a2ensite sstreams-mirror.conf
sudo a2ensite ubuntu-mirror.conf
sudo systemctl restart apache2

echo "Repo configuration and sync done, exiting...!"
echo "Ubuntu repo: http://$HOSTNAME/ubuntu/"
echo "LXD and KVM metadata: https://$HOSTNAME/lxdkvm/_latest"
echo "Juju metadata: https://$HOSTNAME/juju/"
echo "You should also be able to use the IP address"
	#!/usr/bin/env bash

	set -e
	set -u

	## Variables
	set_vars(){
	APT_MIRROR_HOST="mirror"
	LXDKVM_SSTREAM_HOST="mirror"
	JUJU_SSTREAM_HOST="mirror"
	BOOTSTRAP_NODE_IP=10.10.0.20
	CDK_SHRINKWRAP_PATH=$(ls -dt $HOME/cdk-shrinkwrap/*/ \| head -1)
	PRIV_REGISTRY_HOST="mirror:5000"
	MACHINE_IPS="10.10.0.21 \
	10.10.0.22 \
	10.10.0.23 \
	10.10.0.24 \
	10.10.0.25 \
	10.10.0.26 \
	10.10.0.27 \
	10.10.0.28 \
	10.10.0.29 \
	10.10.0.30 \
	10.10.0.31"
	KUBE_VER="1.11/stable"
	ETCD_VER="3.2/stable"
	SERIES=xenial
	CERT_PATH="/etc/pki/tls/certs/"
	CERT_FILE="mirror.crt"
	FAN_CONFIG="10.10.0.0/24=252.0.0.0/8"
	}

	check_pkg(){
	if dpkg --get-selections \| grep -q "^$1[[:space:]]*install$" >/dev/null; then
	return
	else
	echo "package $1 is missing."
	exit 1
	fi
	}

	check_prerequisite(){
	check_pkg "jq"
	check_pkg "juju-2.0"
	}

	## Bootstrap juju
	juju_bootstrap(){
	tee ~/mycloud.yaml > /dev/null << EOL
	clouds:
	manual:
	type: manual
	endpoint: $BOOTSTRAP_NODE_IP
	EOL
	juju add-cloud --replace manual ~/mycloud.yaml
	echo "bootstrapping juju..."
	ssh-keyscan $BOOTSTRAP_NODE_IP
	sleep 2
	ssh-keyscan $BOOTSTRAP_NODE_IP > ~/.ssh/known_hosts
	juju bootstrap --no-gui \
	--bootstrap-series=$SERIES \
	--config apt-mirror=http://${APT_MIRROR_HOST}/archive.ubuntu.com/ubuntu/ \
	--config agent-stream=release \
	--config container-image-metadata-url=https://${LXDKVM_SSTREAM_HOST}/lxdkvm/_latest \
	--config agent-metadata-url=https://${JUJU_SSTREAM_HOST}/juju/ \
	--debug manual manual-controller > juju-bootstrap.log 2>&1
	juju model-config fan-config=$FAN_CONFIG
	juju model-config container-networking-method=fan
	}

	## add machines
	juju_add_machines(){
	for machine in $MACHINE_IPS
	do
	ssh-keyscan $machine
	sleep 2
	ssh-keyscan $machine >> ~/.ssh/known_hosts
	juju add-machine --debug ssh:ubuntu@$machine > juju-add-machine-$machine.log 2>&1 &
	done
	sleep 60
	## prepare machines for deploy
	len=$(wc -w <<< "$MACHINE_IPS")
	for id in $(seq 0 $(expr $len - 1))
	do
	while [ $(juju status --format json \| jq -r ".machines[\"$id\"].\"juju-status\".current") != "started" ]
	do
	echo "waiting for machine $id to be ready..."
	sleep 5
	done
	echo "machine $id is ready. configuring machine now"
	juju scp ${CDK_SHRINKWRAP_PATH}/resources/core.snap $id:
	juju scp ${CERT_PATH}${CERT_FILE} $id:
	done
	juju run --all "sudo snap install --dangerous /home/ubuntu/core.snap"
	juju run --all "sudo mv /home/ubuntu/$CERT_FILE /usr/local/share/ca-certificates/"
	juju run --all "sudo update-ca-certificates"
	# juju won't replace securiy.ubuntu.com to value in --apt-mirror
	juju run --all -- sudo sed -i 's/security.ubuntu.com/${APT_MIRROR_HOST}\/archive.ubuntu.com/g' /etc/apt/sources.list

	}

	# deploy all
	juju_deploy(){
	# deploy etcd
	juju deploy -n 3 --resource etcd=${CDK_SHRINKWRAP_PATH}/resources/etcd/etcd.snap --resource snapshot=${CDK_SHRINKWRAP_PATH}/resources/etcd/snapshot.gz \
	--to 0,1,2 ${CDK_SHRINKWRAP_PATH}/charms/etcd --config ~/cdk-config.yaml
	# deploy kubernetes-master
	juju deploy -n 2 --resource cdk-addons=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/cdk-addons.snap \
	--resource kube-apiserver=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-apiserver.snap \
	--resource kube-controller-manager=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-controller-manager.snap \
	--resource kube-scheduler=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kube-scheduler.snap \
	--resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-master/kubectl.snap --to 3,4 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-master --config ~/cdk-config.yaml
	# deploy flannel
	juju deploy --resource flannel-amd64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-amd64.gz --resource flannel-arm64=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-arm64.gz \
	--resource flannel-s390x=${CDK_SHRINKWRAP_PATH}/resources/flannel/flannel-s390x.gz ${CDK_SHRINKWRAP_PATH}/charms/flannel
	# deploy easyrsa
	juju deploy -n 1 --resource easyrsa=${CDK_SHRINKWRAP_PATH}/resources/easyrsa/easyrsa.tgz --to lxd:8 ${CDK_SHRINKWRAP_PATH}/charms/easyrsa
	# deploy kubernetes-worker
	juju deploy -n 5 --resource cni-amd64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-amd64.tgz --resource cni-arm64=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-arm64.tgz \
	--resource cni-s390x=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/cni-s390x.tgz --resource kube-proxy=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kube-proxy.snap \
	--resource kubectl=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubectl.snap --resource kubelet=${CDK_SHRINKWRAP_PATH}/resources/kubernetes-worker/kubelet.snap \
	--to 5,6,7 ${CDK_SHRINKWRAP_PATH}/charms/kubernetes-worker --config ~/cdk-config.yaml
	# deploy ceph-osd
	juju deploy -n 3 --to 5,6,7 $CDK_SHRINKWRAP_PATH/charms/ceph-osd
	# deploy ceph-mon
	juju deploy -n 3 --to lxd:5,lxd:6,lxd:7 $CDK_SHRINKWRAP_PATH/charms/ceph-mon
	# deploy kubeapi-load-balancer
	juju deploy -n 1 --to 8 ${CDK_SHRINKWRAP_PATH}/charms/kubeapi-load-balancer
	# deploy prometheus2
	juju deploy -n 1 --resource prometheus=${CDK_SHRINKWRAP_PATH}/resources/prometheus/prometheus.snap --to 9 ${CDK_SHRINKWRAP_PATH}/charms/prometheus2
	# deploy grafana
	juju deploy -n 1 --to lxd:9 ${CDK_SHRINKWRAP_PATH}/charms/grafana --config ~/cdk-config.yaml
	# deploy telegraf
	juju deploy ${CDK_SHRINKWRAP_PATH}/charms/telegraf --config ~/cdk-config.yaml
	# deploy elasticsearch
	juju deploy -n 1 --to lxd:8 ${CDK_SHRINKWRAP_PATH}/charms/elasticsearch --config ~/cdk-config.yaml
	# deploy filebeat
	juju deploy ${CDK_SHRINKWRAP_PATH}/charms/filebeat --config ~/cdk-config.yaml
	# deploy graylog
	juju deploy -n 1 --resource graylog=${CDK_SHRINKWRAP_PATH}/resources/graylog/graylog.snap --to 10 ${CDK_SHRINKWRAP_PATH}/charms/graylog
	# deploy mongodb
	juju deploy -n 1 --to lxd:10 ${CDK_SHRINKWRAP_PATH}/charms/mongodb
	# deploy apache2
	juju deploy -n 1 --to lxd:10 ${CDK_SHRINKWRAP_PATH}/charms/apache2 --config ~/cdk-config.yaml
	}


	make_config_yaml(){
	tee ~/cdk-config.yaml > /dev/null << EOL
	kubernetes-master:
	channel: $KUBE_VER
	kubernetes-worker:
	kubelet-extra-args: "pod-infra-container-image=$PRIV_REGISTRY_HOST/google_containers/pause-amd64:3.1"
	docker-opts: "--insecure-registry=$PRIV_REGISTRY_HOST"
	nginx-image: "$PRIV_REGISTRY_HOST/nginx-ingress-controller:0.16.1"
	default-backend-image: "$PRIV_REGISTRY_HOST/defaultbackend:1.4"
	channel: $KUBE_VER
	etcd:
	channel: $ETCD_VER
	apache2:
	enable_modules: "headers proxy_html proxy_http"
	grafana:
	install_sources: "deb http://${APT_MIRROR_HOST}/packagecloud.io/grafana/stable/debian/ stretch main"
	elasticsearch:
	apt-key-url: "http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch"
	apt-repository: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
	filebeat:
	logpath: '/var/log/.log /var/log/containers/.log'
	install_sources: "deb http://${APT_MIRROR_HOST}/artifacts.elastic.co/packages/5.x/apt stable main"
	install_keys: $(echo -e "\|\n - \|\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-elasticsearch \| sed -e 's/^/ /')")
	telegraf:
	install_sources: "deb http://${APT_MIRROR_HOST}/ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main"
	install_keys: $(echo -e "\|\n - \|\n$(wget -qO - http://${APT_MIRROR_HOST}/keys/GPG-KEY-telegraf \| sed -e 's/^/ /')")
	EOL
	}


	juju_add_relations(){
	# add relations
	juju relate kubernetes-master:kube-api-endpoint kubeapi-load-balancer:apiserver
	juju relate kubernetes-master:loadbalancer kubeapi-load-balancer:loadbalancer
	juju relate kubernetes-master:kube-control kubernetes-worker:kube-control
	juju relate kubernetes-master:certificates easyrsa:client
	juju relate etcd:certificates easyrsa:client
	juju relate kubernetes-master:etcd etcd:db
	juju relate kubernetes-worker:certificates easyrsa:client
	juju relate kubernetes-worker:kube-api-endpoint kubeapi-load-balancer:website
	juju relate kubeapi-load-balancer:certificates easyrsa:client
	juju relate flannel:etcd etcd:db
	juju relate flannel:cni kubernetes-master:cni
	juju relate flannel:cni kubernetes-worker:cni
	juju relate apache2:reverseproxy graylog:website
	juju relate graylog:elasticsearch elasticsearch:client
	juju relate graylog:mongodb mongodb:database
	juju relate filebeat:beats-host kubernetes-master:juju-info
	juju relate filebeat:beats-host kubernetes-worker:juju-info
	juju relate filebeat:logstash graylog:beats
	juju relate prometheus2:grafana-source grafana:grafana-source
	juju relate telegraf:prometheus-client prometheus2:target
	juju relate kubernetes-master:juju-info telegraf:juju-info
	juju relate kubernetes-worker:juju-info telegraf:juju-info
	juju relate ceph-mon:osd ceph-osd:mon
	juju relate kubernetes-master:ceph-storage ceph-mon:admin
	}

	adjust_kube_master(){
	# kubernetes-master charm doesn't support to change registry address for cdk-addons
	kube_master_machines=$(juju status kubernetes-master --format json \| jq -r '.machines\|length')
	installed_cnt=0
	while [ $installed_cnt -ne $kube_master_machines ]
	do
	sleep 10
	installed_cnt=$(juju run --application kubernetes-master "snap list \| grep cdk-addons" \| grep -c cdk-addons \|\| true)
	done
	juju run --application kubernetes-master -- snap set cdk-addons registry="$PRIV_REGISTRY_HOST"
	}

	# main
	set_vars
	check_prerequisite
	juju_bootstrap
	juju_add_machines
	make_config_yaml
	juju_deploy
	juju_add_relations

	# after deployment
	echo "sleep for 3 minutes..."
	sleep 180
	adjust_kube_master
	# following is required when deploying charms that uses snap resources to lxd container, because core.snap is not installed in lxd containers
	#juju run --machine $id -- "sudo find /var/lib/lxd/containers/ -path "/var/lib/lxd/containers/juju-*/rootfs/home/ubuntu" -type d -exec cp /home/ubuntu/core.snap {} \;"
	#juju run --machine $id -- "sudo find /var/lib/lxd/containers/ -path "/var/lib/lxd/containers/juju-*/rootfs/home/ubuntu" -type d -exec cp /home/ubuntu/core.snap {} \;"
	#juju run --application prometheus2 "sudo snap install --dangerous /home/ubuntu/core.snap"
	#juju run --application graylog "sudo snap install --dangerous /home/ubuntu/core.snap"

	echo "disk devices are still not set in ceph-osd. ex) set devices by,"
	echo "juju config ceph-osd osd-devices=\"/dev/sdb /dev/sdc\""
	#!/usr/bin/env bash

	## Variables
	APT_MIRROR_PATH="/var/spool/apt-mirror/"
	SSTREAM_PATH="/var/spool/sstreams/"
	GPG_KEY_TELEGRAF=C94406F5
	GPG_KEY_ELASTICSEARCH=D88E42B4
	SERIES=xenial
	CERT_PATH="/etc/pki/tls/certs/"
	PRIV_KEY_PATH="/etc/pki/tls/private/"
	CERT_FILE="mirror.crt"
	PRIV_KEY_FILE="mirror.key"

	## Install all necessary packages
	echo "Installing Missing Packages & Repositories"
	sudo apt-add-repository -y ppa:telegraf-devs/ppa
	sudo apt-add-repository -y ppa:juju/stable
	wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch \| sudo apt-key add -
	wget -qO - https://packagecloud.io/gpg.key \| sudo apt-key add -
	sudo apt update && sudo apt install -y apt-mirror docker.io git apache2 python3-pip unzip juju
	sudo pip3 install pyyaml
	sudo pip3 install pyaml
	sudo snap install kubectl --classic
	sudo snap install charm

	## Setup apt-mirror
	echo "Configuring apt-mirror"
	sudo tee /etc/apt/mirror.list > /dev/null <<EOL
	set base_path $APT_MIRROR_PATH
	set nthreads 20
	set _tilde 0

	deb http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
	deb http://archive.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
	deb http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
	deb http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse

	deb-src http://archive.ubuntu.com/ubuntu xenial main restricted universe multiverse
	deb-src http://archive.ubuntu.com/ubuntu xenial-security main restricted universe multiverse
	deb-src http://archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse
	deb-src http://archive.ubuntu.com/ubuntu xenial-backports main restricted universe multiverse

	deb http://ppa.launchpad.net/telegraf-devs/ppa/ubuntu xenial main
	deb https://artifacts.elastic.co/packages/5.x/apt stable main
	deb https://packagecloud.io/grafana/stable/debian/ stretch main

	clean http://archive.ubuntu.com/ubuntu
	clean http://ppa.launchpad.net/telegraf-devs/ppa/ubuntu
	clean https://artifacts.elastic.co/packages/5.x/apt
	clean https://packagecloud.io/grafana/stable/debian/
	EOL
	# save GPG keys
	apt_key_path="$APT_MIRROR_PATH"mirror/keys/
	sudo mkdir -p $apt_key_path
	apt-key export $GPG_KEY_TELEGRAF \| sudo tee "$apt_key_path"GPG-KEY-telegraf
	apt-key export $GPG_KEY_ELASTICSEARCH \| sudo tee "$apt_key_path"GPG-KEY-elasticsearch


	echo "Start syncing Files, this will take few hours"
	# syncing docker images
	unset -e
	sudo docker run -d -p 5000:5000 --restart=always --name registry registry:2
	export REGISTRY="localhost:5000"
	set -e
	sudo docker pull gcr.io/google_containers/pause-amd64:3.1
	sudo docker tag gcr.io/google_containers/pause-amd64:3.1 "$REGISTRY"/google_containers/pause-amd64:3.1
	sudo docker push "$REGISTRY"/google_containers/pause-amd64:3.1

	sudo docker pull quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1
	sudo docker tag quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.1 "$REGISTRY"/nginx-ingress-controller:0.16.1
	sudo docker push "$REGISTRY"/nginx-ingress-controller:0.16.1

	sudo docker pull k8s.gcr.io/defaultbackend:1.4
	sudo docker tag k8s.gcr.io/defaultbackend:1.4 "$REGISTRY"/defaultbackend:1.4
	sudo docker push "$REGISTRY"/defaultbackend:1.4

	sudo docker pull cdkbot/addon-resizer-amd64:1.8.1
	sudo docker tag cdkbot/addon-resizer-amd64:1.8.1 ${REGISTRY}/addon-resizer-amd64:1.8.1
	sudo docker push ${REGISTRY}/addon-resizer-amd64:1.8.1

	sudo docker pull k8s.gcr.io/heapster-amd64:v1.5.3
	sudo docker tag k8s.gcr.io/heapster-amd64:v1.5.3 ${REGISTRY}/heapster-amd64:v1.5.3
	sudo docker push ${REGISTRY}/heapster-amd64:v1.5.3

	sudo docker pull k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
	sudo docker tag k8s.gcr.io/heapster-influxdb-amd64:v1.3.3 ${REGISTRY}/heapster-influxdb-amd64:v1.3.3
	sudo docker push ${REGISTRY}/heapster-influxdb-amd64:v1.3.3

	sudo docker pull k8s.gcr.io/heapster-grafana-amd64:v4.4.3
	sudo docker tag k8s.gcr.io/heapster-grafana-amd64:v4.4.3 ${REGISTRY}/heapster-grafana-amd64:v4.4.3
	sudo docker push ${REGISTRY}/heapster-grafana-amd64:v4.4.3

	sudo docker pull k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10
	sudo docker tag k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10 ${REGISTRY}/k8s-dns-kube-dns-amd64:1.14.10
	sudo docker push ${REGISTRY}/k8s-dns-kube-dns-amd64:1.14.10

	sudo docker pull k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10
	sudo docker tag k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.10 ${REGISTRY}/k8s-dns-dnsmasq-nanny-amd64:1.14.10
	sudo docker push ${REGISTRY}/k8s-dns-dnsmasq-nanny-amd64:1.14.10

	sudo docker pull k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10
	sudo docker tag k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.10 ${REGISTRY}/k8s-dns-sidecar-amd64:1.14.10
	sudo docker push ${REGISTRY}/k8s-dns-sidecar-amd64:1.14.10

	sudo docker pull k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3
	sudo docker tag k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3 ${REGISTRY}/kubernetes-dashboard-amd64:v1.8.3
	sudo docker push ${REGISTRY}/kubernetes-dashboard-amd64:v1.8.3

	sudo docker pull k8s.gcr.io/metrics-server-amd64:v0.2.1
	sudo docker tag k8s.gcr.io/metrics-server-amd64:v0.2.1 ${REGISTRY}/metrics-server-amd64:v0.2.1
	sudo docker push ${REGISTRY}/metrics-server-amd64:v0.2.1
	# pull nexus for a private registry (optional)
	sudo docker pull sonatype/nexus3:latest
	sudo docker tag sonatype/nexus3:latest ${REGISTRY}/nexus3:latest
	sudo docker push ${REGISTRY}/nexus3:latest

	# pull rancher for management (optional)
	sudo docker pull rancher/rancher:latest
	sudo docker tag rancher/rancher:latest ${REGISTRY}/rancher:latest
	sudo docker push ${REGISTRY}/rancher:latest

	# synch apt packages (this will take several hours)
	sudo apt-mirror

	# synch simplestreams metadata
	workdir=${SSTREAM_PATH}juju
	sudo sstream-mirror --no-verify --progress --max=1 --path=streams/v1/index2.sjson https://streams.canonical.com/juju/tools/ $workdir 'arch=amd64' 'release~(xenial\|bionic)' 'version~(2.2\|2.3\|2.4)'
	workdir=${SSTREAM_PATH}lxdkvm
	sudo sstream-mirror --keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg --progress --max=1 --path=streams/v1/index.json https://cloud-images.ubuntu.com/releases/ $workdir/_latest 'arch=amd64' 'release~(trusty\|xenial)' 'ftype~(lxd.tar.xz\|squashfs\|root.tar.xz\|root.tar.gz\|disk1.img\|.json\|.sjson)'

	# Running CDK Shrink Wrap
	if [ -d cdk-shrinkwrap ]; then
	cd cdk-shrinkwrap
	git pull
	else
	git clone https://github.com/juju-solutions/cdk-shrinkwrap.git
	cd cdk-shrinkwrap
	fi
	./shrinkwrap.py canonical-kubernetes --channel stable
	cdk_shrinkwrap_name=$(ls -t canonical-kubernetes-stable-*.tar.gz \| head -1 \| cut -d'.' -f1)
	tar xf ${cdk_shrinkwrap_name}.tar.gz
	cdk_shrinkwrap_path=$HOME/cdk-shrinkwrap/${cdk_shrinkwrap_name}

	# Pull missing charms
	charms="ceph-osd ceph-mon vault prometheus2 prometheus grafana telegraf elasticsearch filebeat graylog mongodb apache2"
	for charm in $charms
	do
	charm pull $SERIES/$charm ${cdk_shrinkwrap_path}/charms/$charm >> downloaded.txt
	done
	container_charms="canal calico"
	for charm in $container_charms
	do
	charm pull cs:~containers/$SERIES/$charm ${cdk_shrinkwrap_path}/charms/$charm >> downloaded.txt
	done

	# Pull missing snaps for graylog and prometheus
	snap download graylog --stable
	#snap download prometheus --stable
	snap download --channel=2/stable prometheus
	mkdir -p ${cdk_shrinkwrap_path}/resources/graylog
	mkdir -p ${cdk_shrinkwrap_path}/resources/prometheus
	rm *.assert
	mv graylog* ${cdk_shrinkwrap_path}/resources/graylog/graylog.snap
	mv prometheus* ${cdk_shrinkwrap_path}/resources/prometheus/prometheus.snap

	# Configure Mirror to Serve Repo to other nodes
	sudo tee /etc/apache2/sites-available/sstreams-mirror.conf > /dev/null <<EOL
	<VirtualHost *:443>
	ServerName sstreams.cdk-juju
	ServerAlias *
	DocumentRoot ${SSTREAM_PATH}
	SSLCACertificatePath /etc/ssl/certs
	SSLCertificateFile /etc/pki/tls/certs/mirror.crt
	SSLEngine On
	SSLCertificateKeyFile /etc/pki/tls/private/mirror.key
	LogLevel info
	ErrorLog /var/log/apache2/mirror-lxdkvm-error.log
	CustomLog /var/log/apache2/mirror-lxdkvm-access.log combined
	<Directory ${SSTREAM_PATH}>
	Options Indexes FollowSymLinks
	AllowOverride None
	Require all granted
	</Directory>
	</VirtualHost>
	EOL

	sudo tee /etc/apache2/sites-available/ubuntu-mirror.conf > /dev/null <<EOL
	<VirtualHost *:80>
	ServerName cdk-juju
	DocumentRoot ${APT_MIRROR_PATH}mirror/

	LogLevel info
	ErrorLog /var/log/apache2/mirror-error.log
	CustomLog /var/log/apache2/mirror-access.log combined
	<Directory $APT_MIRROR_PATH>
	Options Indexes FollowSymLinks
	AllowOverride None
	Require all granted
	</Directory>

	</VirtualHost>
	EOL

	# Generate SSL cert to be used by apache
	sudo mkdir -p $PRIV_KEY_PATH
	sudo mkdir -p $CERT_PATH

	PRIMARYIP=`hostname -i`
	sudo tee /root/$HOSTNAME.conf > /dev/null <<EOL
	[ req ]
	prompt = no
	default_bits = 4096
	distinguished_name = req_distinguished_name
	req_extensions = req_ext
	[ req_distinguished_name ]
	C=GB
	ST=London
	L=London
	O=Canonical
	OU=Canonical
	CN=$HOSTNAME
	[ req_ext ]
	subjectAltName = @alt_names
	[alt_names]
	DNS.1 = $HOSTNAME
	DNS.2 = $PRIMARYIP
	IP.1 = $PRIMARYIP
	EOL

	sudo openssl req \
	-new \
	-newkey rsa:4096 \
	-days 3650 \
	-nodes \
	-x509 \
	-config /root/$HOSTNAME.conf \
	-keyout ${PRIV_KEY_PATH}${PRIV_KEY_FILE} \
	-out ${CERT_PATH}${CERT_FILE}

	sudo a2enmod ssl
	sudo a2ensite sstreams-mirror.conf
	sudo a2ensite ubuntu-mirror.conf
	sudo systemctl restart apache2

	echo "Repo configuration and sync done, exiting...!"
	echo "Ubuntu repo: http://$HOSTNAME/ubuntu/"
	echo "LXD and KVM metadata: https://$HOSTNAME/lxdkvm/_latest"
	echo "Juju metadata: https://$HOSTNAME/juju/"
	echo "You should also be able to use the IP address"