gistfile1.txt

vim /usr/local/bin/ovf_conf
vim /usr/local/bin/kubeadm_setup.py
/etc/kaas/install.env.j2, replace default pod cidr with the following   
	fd02::/112
/etc/kaas/kubeadm.yaml.j2, add the following
	serviceSubnet: fd03::/112

sudo sysctl -w net.ipv6.conf.ens3.accept_ra=2
sudo sysctl -w net.ipv6.conf.all.forwarding=1

untar /root/.helm/percona
vim percona/template/_helper.tpl take out nc and ipcalc block. Set RESTRICT_IP=%
vim percona/template/perconal.yaml take out nc and ipcalc block. Set RESTRICT_IP=%
vim percona/template/change_wsrep_password_job.yaml take out nc block.

save the aes key job file (below)

ovf_conf
sudo service ovf_conf restart

kubeadm_setup (calico)
sudo service kubeadm_setup restart

ccp-startup-tasks
sudo service ccp-startup-tasks restart

setup-k8s-user-access
sudo service setup-k8s-user-access restart

management_install
sudo service management_install restart

use root
cd /root/.helm/
helm delete --purge percona
helm install percon

delete current aes key job
apply the ase key job below

ccp_install

use this aes key job

apiVersion: batch/v1
kind: Job
metadata:
  name: kaas-cx-aes-key-job-chl5o
spec:
  backoffLimit: 10
  activeDeadlineSeconds: 1800
  template:
    spec:
      serviceAccount: ccp-installer
      serviceAccountName: ccp-installer
      automountServiceAccountToken: true
      containers:
        - name: kaas-cx-aes-key-container
          image: registry.ci.ciscolabs.com/cpsg_ccp/ccp:4.1.0-x-14-g74c8eed
          imagePullPolicy: IfNotPresent
          command: [ "bash", "-c"]
          args:
          - |
            set -euo pipefail

            function post_api {
                curl -f --cacert "$CA" -XPOST -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" -d@"$2" "$APISERVER$1" | sed -E 's/"aes-key": "[a-zA-Z0-9\=]+"/"aes-key": "*** sanitized ***"/'
            }

            function get_api {
                curl -f -s --cacert "$CA" -H 'Content-Type: application/json' -H "Authorization: Bearer $TOKEN" "$APISERVER$1" | sed -E 's/"aes-key": "[a-zA-Z0-9\=]+"/"aes-key": "*** sanitized ***"/'
            }

            function aes_key_exists() {
                local retries=15
                local cx_sec

                while true
                do
                    if cx_sec=$(get_api "/api/v1/namespaces/$NS/secrets" | jq '.items[].metadata.name' | grep '"cx-aes-key"' | sed -e 's/"//g') &&
                        [ -n "$cx_sec" ]; then
                            return 0
                    fi

                    if [ $retries -gt 0 ]; then
                        retries=$(( retries - 1))
                    else
                        break
                    fi

                    sleep 2 
                done
                return 1
            }

            APISERVER="https://[$KUBERNETES_SERVICE_HOST]"
            export APISERVER

            if ! TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) ||
                [ -z "$TOKEN" ]; then
                    echo "cant get service account token" >&2
                    exit 1
            fi
            export TOKEN

            if ! NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) ||
                [ -z "$NS" ]; then
                echo "can't get namespace" >&2
                exit 1
            fi
            export NS

            export CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

            if aes_key_exists; then
                echo "find cx-aes-key. Exit."
                exit 0
            fi

            if ! rand_16_bytes="$(openssl rand 16 | openssl base64 -e | openssl base64 -e)" ||
                [ -z "$rand_16_bytes" ]; then
                    echo "get rand bytes failed" >&2
                    exit 1
            fi

            cat > data <<EOF
            {"apiVersion":"v1","kind":"Secret","metadata":{"name":"cx-aes-key"},"type":"Opaque","data":{"aes-key": "$rand_16_bytes"}}
            EOF

            if ! post_api "/api/v1/namespaces/default/secrets" "data"; then
                echo "create secret failed" >&2
                exit 1
            fi

            if ! aes_key_exists; then
                echo "create aes key failed." >&2
                exit 1
            fi
      restartPolicy: OnFailure