v1k0d3n/install.sh

## install.sh
# System preparation
## Update system:
sudo apt-get update && sudo apt-get dist-upgrade -y

## Prepare for Docker-CE install:
sudo apt-get install -y \
  apt-transport-https \
  ca-certificates \
  curl \
  gnupg-agent \
  software-properties-common


## Add GPG key for Docker-CE repo:
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo apt-key fingerprint 0EBFCD88

## Apt repository add for Docker-CE:
sudo add-apt-repository \
  "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) \
  stable"

## Install docker-ce, docker-ce-cli, containerd:
sudo apt-get update && sudo apt-get upgrade -y
sudo apt-get install -y \
  docker-ce \
  docker-ce-cli \
  containerd.io

# Prepare for Kubernetes tools:
## Add GPG key for Kubernetes repo:
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -

## Apt repository add for Kubernetes tools:
sudo bash -c 'cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
EOF'

## Apt update:
sudo apt-get update

## Apt Install Kubernetes tools:
sudo apt-get install -y \
  kubelet \
  kubeadm \
  kubectl \
  ipvsadm \
  jq

# Calico preparation:
## Obtain Calico manifests (the one included in this gist):
## You can either download the Calico maniest in this Gist and edit to your needs, or you can simply apply directly as described below:
# curl https://gist.githubusercontent.com/v1k0d3n/5e6fcc7ff929a827d1edd94df0bdc49d/raw/0438346eddaf2d99d5500b362f2b30f7921c811a/sdn-calico.yaml -O > /home/"${USER}"/calico.yaml

## Replace default POD/CIDR with custom POD/CIDR:
export CALICO_POD_CIDR="192.168.0.0/16"
export POD_CIDR="10.25.0.0/22"
sed -i -e "s?${CALICO_POD_CIDR}?${POD_CIDR}?g" calico.yaml

## FYI: THERE ARE NEW VERSIONS AT THE FOLLOWING URLS:
# curl https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml -O > /home/"${USER}"/calico-rbac-kdd.yaml
# curl https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml -O > /home/"${USER}"/calico.yaml

# Configure Docker:
## Versions of Kubeadm now fail when using `cgroupfs` drivers. You need to configure `systemd` now, and restart docker.
sudo bash -c 'cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF'
sudo systemctl restart docker

# Swapoff:
sudo swapoff -a

# Initialize Kubernetes:
## First option: kubeadm initialization command:
curl https://gist.githubusercontent.com/v1k0d3n/5e6fcc7ff929a827d1edd94df0bdc49d/raw/f1ee3fdabd1905d87ae959285e44b6f6adf69b71/kubeadm-init.yaml -O
kubeadm init --config kubeadm-init.yaml

## FYI: Or if you do not want to use this configuration YAML, you can init by doing:
# sudo kubeadm init --pod-network-cidr="${POD_CIDR}"

## Secondary option: kubeadm initialization command:
# sudo kubeadm init \
#   --apiserver-advertise-address="192.168.4.81" \
#   --apiserver-bind-port="8443" \
#   --kubernetes-version="v1.13.0" \
#   --node-name="kubenode01.flagship.sh" \
#   --pod-network-cidr="10.25.0.0/22" \
#   --service-cidr="10.96.0.0/12" \
#   --token="783bde.3f89s0fje9f38fhf" \
#   --service-dns-domain="cluster.local" \
#   --token-ttl="24h0m0s"


## Allow ${USER} to control Kubernetes:
sudo cp /etc/kubernetes/admin.conf /home/"${USER}"/.kube/config
sudo chown "${USER}":"${USER}" /home/"${USER}"/.kube/config

## Apply Calico Configuration:
# If you choose to edit with custom CIDR ranges, curl and edit the following file:
# kubectl apply -f /home/"${USER}"/calico.yaml
kubectl apply -f https://gist.githubusercontent.com/v1k0d3n/5e6fcc7ff929a827d1edd94df0bdc49d/raw/0438346eddaf2d99d5500b362f2b30f7921c811a/sdn-calico.yaml


## Untaint the node for AIO workloads:
kubectl taint nodes --all node-role.kubernetes.io/master-

# kubeadm self-hosted initialization:
## Move control plane to self-hosted mode:
sudo kubeadm alpha selfhosting pivot

# Remove all Kubernetes componets after testing:
## Reset the cluster:
sudo kubeadm reset --force
sudo ipvsadm --clear
sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X

# Other helpful commands:
kubectl -n kube-system get cm kubeadm-config -oyaml

# For audit logs, create the following directory:
/var/log/kubernetes/apiserver

## kubeadm-init.yaml
# INITIALIZATION CONFIGURATION:
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.4.81
  bindPort: 6443
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: dcebae.9432563132ffacba
  ttl: 1h0m0s
  usages:
  - signing
  - authentication
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: kubenode01.flagship.sh
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
# CLUSTER CONFIGURATION:
# INCLUDED STILL: apiServer, controllerManager, scheduler, etcd, dns, networking
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: v1.13.0
clusterName: kubernetes
imageRepository: k8s.gcr.io
useHyperKubeImage: true
certificatesDir: /etc/kubernetes/pki
controlPlaneEndpoint: ""
# KUBE-APISERVER CONFIGURATION:
apiServer:
  timeoutForControlPlane: 4m0s
  extraArgs:
    allow-privileged: "true"
    anonymous-auth: "true"
    audit-log-path: "/var/log/kubernetes/apiserver/audit.log"
    authorization-mode: "Node,RBAC"
    # NOTES: USED IN FLAGSHIP DEVELOPMENT MACOS-BASED CLUSTERS
    # basic-auth-file: "/srv/kubernetes/secrets/basic_auth.csv"
    profiling: "false"
    advertise-address: "192.168.4.81"
    client-ca-file: "/etc/kubernetes/pki/ca.crt"
    cloud-provider:
    enable-admission-plugins: "NodeRestriction"
    enable-bootstrap-token-auth: "true"
    etcd-cafile: "/etc/kubernetes/pki/etcd/ca.crt"
    etcd-certfile: "/etc/kubernetes/pki/apiserver-etcd-client.crt"
    etcd-keyfile: "/etc/kubernetes/pki/apiserver-etcd-client.key"
    etcd-servers: "https://127.0.0.1:2379"
    insecure-port: "0"
    kubelet-client-certificate: "/etc/kubernetes/pki/apiserver-kubelet-client.crt"
    kubelet-client-key: "/etc/kubernetes/pki/apiserver-kubelet-client.key"
    kubelet-preferred-address-types: "InternalIP,ExternalIP,Hostname"
    proxy-client-cert-file: "/etc/kubernetes/pki/front-proxy-client.crt"
    proxy-client-key-file: "/etc/kubernetes/pki/front-proxy-client.key"
    requestheader-allowed-names: "front-proxy-client"
    requestheader-client-ca-file: "/etc/kubernetes/pki/front-proxy-ca.crt"
    requestheader-extra-headers-prefix: "X-Remote-Extra-"
    requestheader-group-headers: "X-Remote-Group"
    requestheader-username-headers: "X-Remote-User"
    secure-port: "6443"
    service-account-key-file: "/etc/kubernetes/pki/sa.pub"
    service-cluster-ip-range: "10.96.0.0/22"
    service-node-port-range: "10000-32767"
    storage-backend: "etcd3"
    tls-cert-file: "/etc/kubernetes/pki/apiserver.crt"
    tls-private-key-file: "/etc/kubernetes/pki/apiserver.key"
    # NOTES: CIS REASONS FOR CHANGES
    # anonymous-auth: "false" / CIS: 1.1.1 / causes api/scheduler to periodically bounce
    # allow-privileged: "false" / CIS: 1.7.1 / causes daemonsets to not deploy by default
# KUBE-CONTROLLER CONFIGURATION:
controllerManager:
  extraArgs:
    address: "127.0.0.1"
    allocate-node-cidrs: "true"
    authentication-kubeconfig: /etc/kubernetes/controller-manager.conf
    authorization-kubeconfig: /etc/kubernetes/controller-manager.conf
    client-ca-file: /etc/kubernetes/pki/ca.crt
    cloud-provider:
    cluster-cidr: "10.25.0.0/22"
    cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt
    cluster-signing-key-file: /etc/kubernetes/pki/ca.key
    configure-cloud-routes: "false"
    controllers: "*,bootstrapsigner,tokencleaner"
    kubeconfig: /etc/kubernetes/controller-manager.conf
    leader-elect: "true"
    node-cidr-mask-size: "22"
    requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt
    root-ca-file: /etc/kubernetes/pki/ca.crt
    service-account-private-key-file: /etc/kubernetes/pki/sa.key
    service-cluster-ip-range: "10.96.0.0/16"
    use-service-account-credentials: "true"
# KUBE-SCHEDULER CONFIGURATION:
scheduler:
  extraArgs:
    address: "127.0.0.1"
    kubeconfig: /etc/kubernetes/scheduler.conf
    leader-elect: "true"
# KUBE-DNS CONFIGURATION:
dns:
  type: CoreDNS
# KUBE-ETCD CONFIGURATION:
etcd:
  local:
    imageRepository: k8s.gcr.io
    imageTag: 3.2.24
    dataDir: /var/lib/etcd
    # extraArgs:
    #   name: "flagship-etcd"
  # EXTERNAL ETCD SUPPORT:
  # external:
  #   caFile: /etc/kubernetes/pki/etcd/ca.crt
  #   certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
  #   keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
  #   endpoints:
  #   - https://192.168.4.81:2379
  #   - https://127.0.0.1:2379
networking:
  dnsDomain: cluster.local
  podSubnet: 10.25.0.0/22
  serviceSubnet: 10.96.0.0/12
---
# KUBE-PROXY CONFIGURATION:
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 5
clusterCIDR: 10.25.0.0/22
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  syncPeriod: 30s
metricsBindAddress: 127.0.0.1:10249
mode: ipvs
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
resourceContainer: /kube-proxy
udpIdleTimeout: 250ms
---
# KUBELET CONFIGURATION:
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
staticPodPath: /etc/kubernetes/manifests
syncFrequency: 1m0s
fileCheckFrequency: 20s
httpCheckFrequency: 20s
address: 0.0.0.0
port: 10250
# DEFAULTS:
# tlsCertFile: /var/lib/kubelet/pki/kubelet.crt
# tlsPrivateKeyFile: /var/lib/kubelet/pki/kubelet.key
tlsCertFile: /etc/kubernetes/pki/apiserver-kubelet-client.crt
tlsPrivateKeyFile: /etc/kubernetes/pki/apiserver-kubelet-client.key
rotateCertificates: true
authentication:
 x509:
  clientCAFile: /etc/kubernetes/pki/ca.crt
 webhook:
  enabled: true
  cacheTTL: 2m0s
 anonymous:
  enabled: false
authorization:
 mode: Webhook
 webhook:
  cacheAuthorizedTTL: 10m0s
  cacheUnauthorizedTTL: 30s
registryPullQPS: 5
registryBurst: 10
eventRecordQPS: 5
eventBurst: 10
enableDebuggingHandlers: true
healthzPort: 10248
healthzBindAddress: 127.0.0.1
oomScoreAdj: -999
clusterDomain: cluster.local
clusterDNS:
 - 10.96.0.10
streamingConnectionIdleTimeout: 4h0m0s
nodeStatusUpdateFrequency: 10s
nodeStatusReportFrequency: 1m0s
nodeLeaseDurationSeconds: 40
imageMinimumGCAge: 2m0s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
volumeStatsAggPeriod: 1m0s
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
runtimeRequestTimeout: 2m0s
hairpinMode: promiscuous-bridge
maxPods: 110
podPidsLimit: -1
cpuCFSQuota: true
cpuCFSQuotaPeriod: 100ms
maxOpenFiles: 1000000
contentType: application/vnd.kubernetes.protobuf
kubeAPIQPS: 5
kubeAPIBurst: 10
serializeImagePulls: true
evictionHard:
 "imagefs.available": "15%"
 "memory.available": 100Mi
 "nodefs.available": "10%"
 "nodefs.inodesFree": "5%"
evictionPressureTransitionPeriod: 5m0s
enableControllerAttachDetach: true
makeIPTablesUtilChains: true
iptablesMasqueradeBit: 14
iptablesDropBit: 15
failSwapOn: true
containerLogMaxSize: 10Mi
containerLogMaxFiles: 5
configMapAndSecretChangeDetectionStrategy: Watch
enforceNodeAllocatable:
 - pods

## sdn-calico.yaml
# Calico Version v3.5.0
# https://docs.projectcalico.org/v3.5/releases#v3.5.0
# This manifest includes the following component versions:
#   calico/node:v3.5.0
#   calico/cni:v3.5.0

# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
apiVersion: v1
metadata:
  name: calico-config
  namespace: kube-system
data:
  # You must set a non-zero value for Typha replicas below.
  typha_service_name: "calico-typha"
  # Configure the Calico backend to use.
  calico_backend: "bird"

  # Configure the MTU to use
  veth_mtu: "1440"

  # The CNI network configuration to install on each node.  The special
  # values in this config will be automatically populated.
  cni_network_config: |-
    {
      "name": "k8s-pod-network",
      "cniVersion": "0.3.0",
      "plugins": [
        {
          "type": "calico",
          "log_level": "info",
          "datastore_type": "kubernetes",
          "nodename": "__KUBERNETES_NODE_NAME__",
          "mtu": __CNI_MTU__,
          "ipam": {
            "type": "host-local",
            "subnet": "usePodCidr"
          },
          "policy": {
              "type": "k8s"
          },
          "kubernetes": {
              "kubeconfig": "__KUBECONFIG_FILEPATH__"
          }
        },
        {
          "type": "portmap",
          "snat": true,
          "capabilities": {"portMappings": true}
        }
      ]
    }

---

# This manifest creates a Service, which will be backed by Calico's Typha daemon.
# Typha sits in between Felix and the API server, reducing Calico's load on the API server.

apiVersion: v1
kind: Service
metadata:
  name: calico-typha
  namespace: kube-system
  labels:
    k8s-app: calico-typha
spec:
  ports:
    - port: 5473
      protocol: TCP
      targetPort: calico-typha
      name: calico-typha
  selector:
    k8s-app: calico-typha

---

# This manifest creates a Deployment of Typha to back the above service.

apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: calico-typha
  namespace: kube-system
  labels:
    k8s-app: calico-typha
spec:
  # Number of Typha replicas.  To enable Typha, set this to a non-zero value *and* set the
  # typha_service_name variable in the calico-config ConfigMap above.
  #
  # We recommend using Typha if you have more than 50 nodes.  Above 100 nodes it is essential
  # (when using the Kubernetes datastore).  Use one replica for every 100-200 nodes.  In
  # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
  replicas: 1
  revisionHistoryLimit: 2
  template:
    metadata:
      labels:
        k8s-app: calico-typha
      annotations:
        # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical
        # add-on, ensuring it gets priority scheduling and that its resources are reserved
        # if it ever gets evicted.
        scheduler.alpha.kubernetes.io/critical-pod: ''
        cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
    spec:
      nodeSelector:
        beta.kubernetes.io/os: linux
      hostNetwork: true
      tolerations:
        # Mark the pod as a critical add-on for rescheduling.
        - key: CriticalAddonsOnly
          operator: Exists
      # Since Calico can't network a pod until Typha is up, we need to run Typha itself
      # as a host-networked pod.
      serviceAccountName: calico-node
      containers:
      - image: calico/typha:v3.5.0
        name: calico-typha
        ports:
        - containerPort: 5473
          name: calico-typha
          protocol: TCP
        env:
          # Enable "info" logging by default.  Can be set to "debug" to increase verbosity.
          - name: TYPHA_LOGSEVERITYSCREEN
            value: "info"
          # Disable logging to file and syslog since those don't make sense in Kubernetes.
          - name: TYPHA_LOGFILEPATH
            value: "none"
          - name: TYPHA_LOGSEVERITYSYS
            value: "none"
          # Monitor the Kubernetes API to find the number of running instances and rebalance
          # connections.
          - name: TYPHA_CONNECTIONREBALANCINGMODE
            value: "kubernetes"
          - name: TYPHA_DATASTORETYPE
            value: "kubernetes"
          - name: TYPHA_HEALTHENABLED
            value: "true"
          # Uncomment these lines to enable prometheus metrics.  Since Typha is host-networked,
          # this opens a port on the host, which may need to be secured.
          #- name: TYPHA_PROMETHEUSMETRICSENABLED
          #  value: "true"
          #- name: TYPHA_PROMETHEUSMETRICSPORT
          #  value: "9093"
        livenessProbe:
          exec:
            command:
            - calico-typha
            - check
            - liveness
          periodSeconds: 30
          initialDelaySeconds: 30
        readinessProbe:
          exec:
            command:
            - calico-typha
            - check
            - readiness
          periodSeconds: 10

---

# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict

apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: calico-typha
  namespace: kube-system
  labels:
    k8s-app: calico-typha
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: calico-typha

---
# This manifest installs the calico/node container, as well
# as the Calico CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
  name: calico-node
  namespace: kube-system
  labels:
    k8s-app: calico-node
spec:
  selector:
    matchLabels:
      k8s-app: calico-node
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        k8s-app: calico-node
      annotations:
        # This, along with the CriticalAddonsOnly toleration below,
        # marks the pod as a critical add-on, ensuring it gets
        # priority scheduling and that its resources are reserved
        # if it ever gets evicted.
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      nodeSelector:
        beta.kubernetes.io/os: linux
      hostNetwork: true
      tolerations:
        # Make sure calico-node gets scheduled on all nodes.
        - effect: NoSchedule
          operator: Exists
        # Mark the pod as a critical add-on for rescheduling.
        - key: CriticalAddonsOnly
          operator: Exists
        - effect: NoExecute
          operator: Exists
      serviceAccountName: calico-node
      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
      initContainers:
        # This container installs the Calico CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
          image: calico/cni:v3.5.0
          command: ["/install-cni.sh"]
          env:
            # Name of the CNI config file to create.
            - name: CNI_CONF_NAME
              value: "10-calico.conflist"
            # The CNI network config to install on each node.
            - name: CNI_NETWORK_CONFIG
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: cni_network_config
            # Set the hostname based on the k8s node name.
            - name: KUBERNETES_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # CNI MTU Config variable
            - name: CNI_MTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # Prevents the container from sleeping forever.
            - name: SLEEP
              value: "false"
          volumeMounts:
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
            - mountPath: /host/etc/cni/net.d
              name: cni-net-dir
      containers:
        # Runs calico/node container on each Kubernetes node.  This
        # container programs network policy and routes on each
        # host.
        - name: calico-node
          image: calico/node:v3.5.0
          env:
            # Use Kubernetes API as the backing datastore.
            - name: DATASTORE_TYPE
              value: "kubernetes"
            # Typha support: controlled by the ConfigMap.
            - name: FELIX_TYPHAK8SSERVICENAME
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: typha_service_name
            # Wait for the datastore.
            - name: WAIT_FOR_DATASTORE
              value: "true"
            # Set based on the k8s node name.
            - name: NODENAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # Choose the backend to use.
            - name: CALICO_NETWORKING_BACKEND
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: calico_backend
            # Cluster type to identify the deployment type
            - name: CLUSTER_TYPE
              value: "k8s,bgp"
            # Auto-detect the BGP IP address.
            - name: IP
              value: "autodetect"
            # Enable IPIP
            - name: CALICO_IPV4POOL_IPIP
              value: "Always"
            # Set MTU for tunnel device used if ipip is enabled
            - name: FELIX_IPINIPMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            # The default IPv4 pool to create on startup if none exists. Pod IPs will be
            # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within `--cluster-cidr`.
            - name: CALICO_IPV4POOL_CIDR
              value: "10.25.0.0/22"
            # Disable file logging so `kubectl logs` works.
            - name: CALICO_DISABLE_FILE_LOGGING
              value: "true"
            # Set Felix endpoint to host default action to ACCEPT.
            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
              value: "ACCEPT"
            # Disable IPv6 on Kubernetes.
            - name: FELIX_IPV6SUPPORT
              value: "false"
            # Set Felix logging to "info"
            - name: FELIX_LOGSEVERITYSCREEN
              value: "info"
            - name: FELIX_HEALTHENABLED
              value: "true"
          securityContext:
            privileged: true
          resources:
            requests:
              cpu: 250m
          livenessProbe:
            httpGet:
              path: /liveness
              port: 9099
              host: localhost
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
            exec:
              command:
              - /bin/calico-node
              - -bird-ready
              - -felix-ready
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
              name: lib-modules
              readOnly: true
            - mountPath: /run/xtables.lock
              name: xtables-lock
              readOnly: false
            - mountPath: /var/run/calico
              name: var-run-calico
              readOnly: false
            - mountPath: /var/lib/calico
              name: var-lib-calico
              readOnly: false
      volumes:
        # Used by calico/node.
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: var-run-calico
          hostPath:
            path: /var/run/calico
        - name: var-lib-calico
          hostPath:
            path: /var/lib/calico
        - name: xtables-lock
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
        # Used to install CNI.
        - name: cni-bin-dir
          hostPath:
            path: /opt/cni/bin
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-node
  namespace: kube-system

---
# Create all the CustomResourceDefinitions needed for
# Calico policy and networking mode.

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
   name: felixconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: FelixConfiguration
    plural: felixconfigurations
    singular: felixconfiguration
---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgppeers.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPPeer
    plural: bgppeers
    singular: bgppeer

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgpconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPConfiguration
    plural: bgpconfigurations
    singular: bgpconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ippools.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPPool
    plural: ippools
    singular: ippool

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: hostendpoints.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: HostEndpoint
    plural: hostendpoints
    singular: hostendpoint

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: clusterinformations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: ClusterInformation
    plural: clusterinformations
    singular: clusterinformation

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworkpolicies.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkPolicy
    plural: globalnetworkpolicies
    singular: globalnetworkpolicy

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworksets.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkSet
    plural: globalnetworksets
    singular: globalnetworkset

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: networkpolicies.crd.projectcalico.org
spec:
  scope: Namespaced
  group: crd.projectcalico.org
  version: v1
  names:
    kind: NetworkPolicy
    plural: networkpolicies
    singular: networkpolicy
---

# Include a clusterrole for the calico-node DaemonSet,
# and bind it to the calico-node serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: calico-node
rules:
  # The CNI plugin needs to get pods, nodes, and namespaces.
  - apiGroups: [""]
    resources:
      - pods
      - nodes
      - namespaces
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - endpoints
      - services
    verbs:
      # Used to discover service IPs for advertisement.
      - watch
      - list
      # Used to discover Typhas.
      - get
  - apiGroups: [""]
    resources:
      - nodes/status
    verbs:
      # Needed for clearing NodeNetworkUnavailable flag.
      - patch
      # Calico stores some configuration information in node annotations.
      - update
  # Watch for changes to Kubernetes NetworkPolicies.
  - apiGroups: ["networking.k8s.io"]
    resources:
      - networkpolicies
    verbs:
      - watch
      - list
  # Used by Calico for policy information.
  - apiGroups: [""]
    resources:
      - pods
      - namespaces
      - serviceaccounts
    verbs:
      - list
      - watch
  # The CNI plugin patches pods/status.
  - apiGroups: [""]
    resources:
      - pods/status
    verbs:
      - patch
  # Calico monitors various CRDs for config.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - globalfelixconfigs
      - felixconfigurations
      - bgppeers
      - globalbgpconfigs
      - bgpconfigurations
      - ippools
      - globalnetworkpolicies
      - globalnetworksets
      - networkpolicies
      - clusterinformations
      - hostendpoints
    verbs:
      - get
      - list
      - watch
  # Calico must create and update some CRDs on startup.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
      - felixconfigurations
      - clusterinformations
    verbs:
      - create
      - update
  # Calico stores some configuration information on the node.
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - get
      - list
      - watch
  # These permissions are only requried for upgrade from v2.6, and can
  # be removed after upgrade or on fresh installations.
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - bgpconfigurations
      - bgppeers
    verbs:
      - create
      - update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: calico-node
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-node
subjects:
- kind: ServiceAccount
  name: calico-node
  namespace: kube-system
---
	# System preparation
	## Update system:
	sudo apt-get update && sudo apt-get dist-upgrade -y

	## Prepare for Docker-CE install:
	sudo apt-get install -y \
	apt-transport-https \
	ca-certificates \
	curl \
	gnupg-agent \
	software-properties-common


	## Add GPG key for Docker-CE repo:
	curl -fsSL https://download.docker.com/linux/ubuntu/gpg \| sudo apt-key add -
	sudo apt-key fingerprint 0EBFCD88

	## Apt repository add for Docker-CE:
	sudo add-apt-repository \
	"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
	$(lsb_release -cs) \
	stable"

	## Install docker-ce, docker-ce-cli, containerd:
	sudo apt-get update && sudo apt-get upgrade -y
	sudo apt-get install -y \
	docker-ce \
	docker-ce-cli \
	containerd.io

	# Prepare for Kubernetes tools:
	## Add GPG key for Kubernetes repo:
	curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg \| sudo apt-key add -

	## Apt repository add for Kubernetes tools:
	sudo bash -c 'cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
	deb https://apt.kubernetes.io/ kubernetes-xenial main
	EOF'

	## Apt update:
	sudo apt-get update

	## Apt Install Kubernetes tools:
	sudo apt-get install -y \
	kubelet \
	kubeadm \
	kubectl \
	ipvsadm \
	jq

	# Calico preparation:
	## Obtain Calico manifests (the one included in this gist):
	## You can either download the Calico maniest in this Gist and edit to your needs, or you can simply apply directly as described below:
	# curl https://gist.githubusercontent.com/v1k0d3n/5e6fcc7ff929a827d1edd94df0bdc49d/raw/0438346eddaf2d99d5500b362f2b30f7921c811a/sdn-calico.yaml -O > /home/"${USER}"/calico.yaml

	## Replace default POD/CIDR with custom POD/CIDR:
	export CALICO_POD_CIDR="192.168.0.0/16"
	export POD_CIDR="10.25.0.0/22"
	sed -i -e "s?${CALICO_POD_CIDR}?${POD_CIDR}?g" calico.yaml

	## FYI: THERE ARE NEW VERSIONS AT THE FOLLOWING URLS:
	# curl https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml -O > /home/"${USER}"/calico-rbac-kdd.yaml
	# curl https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml -O > /home/"${USER}"/calico.yaml

	# Configure Docker:
	## Versions of Kubeadm now fail when using `cgroupfs` drivers. You need to configure `systemd` now, and restart docker.
	sudo bash -c 'cat > /etc/docker/daemon.json <<EOF
	{
	"exec-opts": ["native.cgroupdriver=systemd"],
	"log-driver": "json-file",
	"log-opts": {
	"max-size": "100m"
	},
	"storage-driver": "overlay2"
	}
	EOF'
	sudo systemctl restart docker

	# Swapoff:
	sudo swapoff -a

	# Initialize Kubernetes:
	## First option: kubeadm initialization command:
	curl https://gist.githubusercontent.com/v1k0d3n/5e6fcc7ff929a827d1edd94df0bdc49d/raw/f1ee3fdabd1905d87ae959285e44b6f6adf69b71/kubeadm-init.yaml -O
	kubeadm init --config kubeadm-init.yaml

	## FYI: Or if you do not want to use this configuration YAML, you can init by doing:
	# sudo kubeadm init --pod-network-cidr="${POD_CIDR}"

	## Secondary option: kubeadm initialization command:
	# sudo kubeadm init \
	# --apiserver-advertise-address="192.168.4.81" \
	# --apiserver-bind-port="8443" \
	# --kubernetes-version="v1.13.0" \
	# --node-name="kubenode01.flagship.sh" \
	# --pod-network-cidr="10.25.0.0/22" \
	# --service-cidr="10.96.0.0/12" \
	# --token="783bde.3f89s0fje9f38fhf" \
	# --service-dns-domain="cluster.local" \
	# --token-ttl="24h0m0s"


	## Allow ${USER} to control Kubernetes:
	sudo cp /etc/kubernetes/admin.conf /home/"${USER}"/.kube/config
	sudo chown "${USER}":"${USER}" /home/"${USER}"/.kube/config

	## Apply Calico Configuration:
	# If you choose to edit with custom CIDR ranges, curl and edit the following file:
	# kubectl apply -f /home/"${USER}"/calico.yaml
	kubectl apply -f https://gist.githubusercontent.com/v1k0d3n/5e6fcc7ff929a827d1edd94df0bdc49d/raw/0438346eddaf2d99d5500b362f2b30f7921c811a/sdn-calico.yaml


	## Untaint the node for AIO workloads:
	kubectl taint nodes --all node-role.kubernetes.io/master-

	# kubeadm self-hosted initialization:
	## Move control plane to self-hosted mode:
	sudo kubeadm alpha selfhosting pivot

	# Remove all Kubernetes componets after testing:
	## Reset the cluster:
	sudo kubeadm reset --force
	sudo ipvsadm --clear
	sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X

	# Other helpful commands:
	kubectl -n kube-system get cm kubeadm-config -oyaml

	# For audit logs, create the following directory:
	/var/log/kubernetes/apiserver
	# INITIALIZATION CONFIGURATION:
	apiVersion: kubeadm.k8s.io/v1beta1
	kind: InitConfiguration
	localAPIEndpoint:
	advertiseAddress: 192.168.4.81
	bindPort: 6443
	bootstrapTokens:
	- groups:
	- system:bootstrappers:kubeadm:default-node-token
	token: dcebae.9432563132ffacba
	ttl: 1h0m0s
	usages:
	- signing
	- authentication
	nodeRegistration:
	criSocket: /var/run/dockershim.sock
	name: kubenode01.flagship.sh
	taints:
	- effect: NoSchedule
	key: node-role.kubernetes.io/master
	---
	# CLUSTER CONFIGURATION:
	# INCLUDED STILL: apiServer, controllerManager, scheduler, etcd, dns, networking
	apiVersion: kubeadm.k8s.io/v1beta1
	kind: ClusterConfiguration
	kubernetesVersion: v1.13.0
	clusterName: kubernetes
	imageRepository: k8s.gcr.io
	useHyperKubeImage: true
	certificatesDir: /etc/kubernetes/pki
	controlPlaneEndpoint: ""
	# KUBE-APISERVER CONFIGURATION:
	apiServer:
	timeoutForControlPlane: 4m0s
	extraArgs:
	allow-privileged: "true"
	anonymous-auth: "true"
	audit-log-path: "/var/log/kubernetes/apiserver/audit.log"
	authorization-mode: "Node,RBAC"
	# NOTES: USED IN FLAGSHIP DEVELOPMENT MACOS-BASED CLUSTERS
	# basic-auth-file: "/srv/kubernetes/secrets/basic_auth.csv"
	profiling: "false"
	advertise-address: "192.168.4.81"
	client-ca-file: "/etc/kubernetes/pki/ca.crt"
	cloud-provider:
	enable-admission-plugins: "NodeRestriction"
	enable-bootstrap-token-auth: "true"
	etcd-cafile: "/etc/kubernetes/pki/etcd/ca.crt"
	etcd-certfile: "/etc/kubernetes/pki/apiserver-etcd-client.crt"
	etcd-keyfile: "/etc/kubernetes/pki/apiserver-etcd-client.key"
	etcd-servers: "https://127.0.0.1:2379"
	insecure-port: "0"
	kubelet-client-certificate: "/etc/kubernetes/pki/apiserver-kubelet-client.crt"
	kubelet-client-key: "/etc/kubernetes/pki/apiserver-kubelet-client.key"
	kubelet-preferred-address-types: "InternalIP,ExternalIP,Hostname"
	proxy-client-cert-file: "/etc/kubernetes/pki/front-proxy-client.crt"
	proxy-client-key-file: "/etc/kubernetes/pki/front-proxy-client.key"
	requestheader-allowed-names: "front-proxy-client"
	requestheader-client-ca-file: "/etc/kubernetes/pki/front-proxy-ca.crt"
	requestheader-extra-headers-prefix: "X-Remote-Extra-"
	requestheader-group-headers: "X-Remote-Group"
	requestheader-username-headers: "X-Remote-User"
	secure-port: "6443"
	service-account-key-file: "/etc/kubernetes/pki/sa.pub"
	service-cluster-ip-range: "10.96.0.0/22"
	service-node-port-range: "10000-32767"
	storage-backend: "etcd3"
	tls-cert-file: "/etc/kubernetes/pki/apiserver.crt"
	tls-private-key-file: "/etc/kubernetes/pki/apiserver.key"
	# NOTES: CIS REASONS FOR CHANGES
	# anonymous-auth: "false" / CIS: 1.1.1 / causes api/scheduler to periodically bounce
	# allow-privileged: "false" / CIS: 1.7.1 / causes daemonsets to not deploy by default
	# KUBE-CONTROLLER CONFIGURATION:
	controllerManager:
	extraArgs:
	address: "127.0.0.1"
	allocate-node-cidrs: "true"
	authentication-kubeconfig: /etc/kubernetes/controller-manager.conf
	authorization-kubeconfig: /etc/kubernetes/controller-manager.conf
	client-ca-file: /etc/kubernetes/pki/ca.crt
	cloud-provider:
	cluster-cidr: "10.25.0.0/22"
	cluster-signing-cert-file: /etc/kubernetes/pki/ca.crt
	cluster-signing-key-file: /etc/kubernetes/pki/ca.key
	configure-cloud-routes: "false"
	controllers: "*,bootstrapsigner,tokencleaner"
	kubeconfig: /etc/kubernetes/controller-manager.conf
	leader-elect: "true"
	node-cidr-mask-size: "22"
	requestheader-client-ca-file: /etc/kubernetes/pki/front-proxy-ca.crt
	root-ca-file: /etc/kubernetes/pki/ca.crt
	service-account-private-key-file: /etc/kubernetes/pki/sa.key
	service-cluster-ip-range: "10.96.0.0/16"
	use-service-account-credentials: "true"
	# KUBE-SCHEDULER CONFIGURATION:
	scheduler:
	extraArgs:
	address: "127.0.0.1"
	kubeconfig: /etc/kubernetes/scheduler.conf
	leader-elect: "true"
	# KUBE-DNS CONFIGURATION:
	dns:
	type: CoreDNS
	# KUBE-ETCD CONFIGURATION:
	etcd:
	local:
	imageRepository: k8s.gcr.io
	imageTag: 3.2.24
	dataDir: /var/lib/etcd
	# extraArgs:
	# name: "flagship-etcd"
	# EXTERNAL ETCD SUPPORT:
	# external:
	# caFile: /etc/kubernetes/pki/etcd/ca.crt
	# certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt
	# keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key
	# endpoints:
	# - https://192.168.4.81:2379
	# - https://127.0.0.1:2379
	networking:
	dnsDomain: cluster.local
	podSubnet: 10.25.0.0/22
	serviceSubnet: 10.96.0.0/12
	---
	# KUBE-PROXY CONFIGURATION:
	apiVersion: kubeproxy.config.k8s.io/v1alpha1
	kind: KubeProxyConfiguration
	bindAddress: 0.0.0.0
	clientConnection:
	acceptContentTypes: ""
	burst: 10
	contentType: application/vnd.kubernetes.protobuf
	kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
	qps: 5
	clusterCIDR: 10.25.0.0/22
	configSyncPeriod: 15m0s
	conntrack:
	max: null
	maxPerCore: 32768
	min: 131072
	tcpCloseWaitTimeout: 1h0m0s
	tcpEstablishedTimeout: 24h0m0s
	enableProfiling: false
	healthzBindAddress: 0.0.0.0:10256
	hostnameOverride: ""
	iptables:
	masqueradeAll: false
	masqueradeBit: 14
	minSyncPeriod: 0s
	syncPeriod: 30s
	ipvs:
	excludeCIDRs: null
	minSyncPeriod: 0s
	scheduler: ""
	syncPeriod: 30s
	metricsBindAddress: 127.0.0.1:10249
	mode: ipvs
	nodePortAddresses: null
	oomScoreAdj: -999
	portRange: ""
	resourceContainer: /kube-proxy
	udpIdleTimeout: 250ms
	---
	# KUBELET CONFIGURATION:
	apiVersion: kubelet.config.k8s.io/v1beta1
	kind: KubeletConfiguration
	staticPodPath: /etc/kubernetes/manifests
	syncFrequency: 1m0s
	fileCheckFrequency: 20s
	httpCheckFrequency: 20s
	address: 0.0.0.0
	port: 10250
	# DEFAULTS:
	# tlsCertFile: /var/lib/kubelet/pki/kubelet.crt
	# tlsPrivateKeyFile: /var/lib/kubelet/pki/kubelet.key
	tlsCertFile: /etc/kubernetes/pki/apiserver-kubelet-client.crt
	tlsPrivateKeyFile: /etc/kubernetes/pki/apiserver-kubelet-client.key
	rotateCertificates: true
	authentication:
	x509:
	clientCAFile: /etc/kubernetes/pki/ca.crt
	webhook:
	enabled: true
	cacheTTL: 2m0s
	anonymous:
	enabled: false
	authorization:
	mode: Webhook
	webhook:
	cacheAuthorizedTTL: 10m0s
	cacheUnauthorizedTTL: 30s
	registryPullQPS: 5
	registryBurst: 10
	eventRecordQPS: 5
	eventBurst: 10
	enableDebuggingHandlers: true
	healthzPort: 10248
	healthzBindAddress: 127.0.0.1
	oomScoreAdj: -999
	clusterDomain: cluster.local
	clusterDNS:
	- 10.96.0.10
	streamingConnectionIdleTimeout: 4h0m0s
	nodeStatusUpdateFrequency: 10s
	nodeStatusReportFrequency: 1m0s
	nodeLeaseDurationSeconds: 40
	imageMinimumGCAge: 2m0s
	imageGCHighThresholdPercent: 85
	imageGCLowThresholdPercent: 80
	volumeStatsAggPeriod: 1m0s
	cpuManagerPolicy: none
	cpuManagerReconcilePeriod: 10s
	runtimeRequestTimeout: 2m0s
	hairpinMode: promiscuous-bridge
	maxPods: 110
	podPidsLimit: -1
	cpuCFSQuota: true
	cpuCFSQuotaPeriod: 100ms
	maxOpenFiles: 1000000
	contentType: application/vnd.kubernetes.protobuf
	kubeAPIQPS: 5
	kubeAPIBurst: 10
	serializeImagePulls: true
	evictionHard:
	"imagefs.available": "15%"
	"memory.available": 100Mi
	"nodefs.available": "10%"
	"nodefs.inodesFree": "5%"
	evictionPressureTransitionPeriod: 5m0s
	enableControllerAttachDetach: true
	makeIPTablesUtilChains: true
	iptablesMasqueradeBit: 14
	iptablesDropBit: 15
	failSwapOn: true
	containerLogMaxSize: 10Mi
	containerLogMaxFiles: 5
	configMapAndSecretChangeDetectionStrategy: Watch
	enforceNodeAllocatable:
	- pods
	# Calico Version v3.5.0
	# https://docs.projectcalico.org/v3.5/releases#v3.5.0
	# This manifest includes the following component versions:
	# calico/node:v3.5.0
	# calico/cni:v3.5.0

	# This ConfigMap is used to configure a self-hosted Calico installation.
	kind: ConfigMap
	apiVersion: v1
	metadata:
	name: calico-config
	namespace: kube-system
	data:
	# You must set a non-zero value for Typha replicas below.
	typha_service_name: "calico-typha"
	# Configure the Calico backend to use.
	calico_backend: "bird"

	# Configure the MTU to use
	veth_mtu: "1440"

	# The CNI network configuration to install on each node. The special
	# values in this config will be automatically populated.
	cni_network_config: \|-
	{
	"name": "k8s-pod-network",
	"cniVersion": "0.3.0",
	"plugins": [
	{
	"type": "calico",
	"log_level": "info",
	"datastore_type": "kubernetes",
	"nodename": "__KUBERNETES_NODE_NAME__",
	"mtu": __CNI_MTU__,
	"ipam": {
	"type": "host-local",
	"subnet": "usePodCidr"
	},
	"policy": {
	"type": "k8s"
	},
	"kubernetes": {
	"kubeconfig": "__KUBECONFIG_FILEPATH__"
	}
	},
	{
	"type": "portmap",
	"snat": true,
	"capabilities": {"portMappings": true}
	}
	]
	}

	---

	# This manifest creates a Service, which will be backed by Calico's Typha daemon.
	# Typha sits in between Felix and the API server, reducing Calico's load on the API server.

	apiVersion: v1
	kind: Service
	metadata:
	name: calico-typha
	namespace: kube-system
	labels:
	k8s-app: calico-typha
	spec:
	ports:
	- port: 5473
	protocol: TCP
	targetPort: calico-typha
	name: calico-typha
	selector:
	k8s-app: calico-typha

	---

	# This manifest creates a Deployment of Typha to back the above service.

	apiVersion: apps/v1beta1
	kind: Deployment
	metadata:
	name: calico-typha
	namespace: kube-system
	labels:
	k8s-app: calico-typha
	spec:
	# Number of Typha replicas. To enable Typha, set this to a non-zero value and set the
	# typha_service_name variable in the calico-config ConfigMap above.
	#
	# We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential
	# (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In
	# production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
	replicas: 1
	revisionHistoryLimit: 2
	template:
	metadata:
	labels:
	k8s-app: calico-typha
	annotations:
	# This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical
	# add-on, ensuring it gets priority scheduling and that its resources are reserved
	# if it ever gets evicted.
	scheduler.alpha.kubernetes.io/critical-pod: ''
	cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
	spec:
	nodeSelector:
	beta.kubernetes.io/os: linux
	hostNetwork: true
	tolerations:
	# Mark the pod as a critical add-on for rescheduling.
	- key: CriticalAddonsOnly
	operator: Exists
	# Since Calico can't network a pod until Typha is up, we need to run Typha itself
	# as a host-networked pod.
	serviceAccountName: calico-node
	containers:
	- image: calico/typha:v3.5.0
	name: calico-typha
	ports:
	- containerPort: 5473
	name: calico-typha
	protocol: TCP
	env:
	# Enable "info" logging by default. Can be set to "debug" to increase verbosity.
	- name: TYPHA_LOGSEVERITYSCREEN
	value: "info"
	# Disable logging to file and syslog since those don't make sense in Kubernetes.
	- name: TYPHA_LOGFILEPATH
	value: "none"
	- name: TYPHA_LOGSEVERITYSYS
	value: "none"
	# Monitor the Kubernetes API to find the number of running instances and rebalance
	# connections.
	- name: TYPHA_CONNECTIONREBALANCINGMODE
	value: "kubernetes"
	- name: TYPHA_DATASTORETYPE
	value: "kubernetes"
	- name: TYPHA_HEALTHENABLED
	value: "true"
	# Uncomment these lines to enable prometheus metrics. Since Typha is host-networked,
	# this opens a port on the host, which may need to be secured.
	#- name: TYPHA_PROMETHEUSMETRICSENABLED
	# value: "true"
	#- name: TYPHA_PROMETHEUSMETRICSPORT
	# value: "9093"
	livenessProbe:
	exec:
	command:
	- calico-typha
	- check
	- liveness
	periodSeconds: 30
	initialDelaySeconds: 30
	readinessProbe:
	exec:
	command:
	- calico-typha
	- check
	- readiness
	periodSeconds: 10

	---

	# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict

	apiVersion: policy/v1beta1
	kind: PodDisruptionBudget
	metadata:
	name: calico-typha
	namespace: kube-system
	labels:
	k8s-app: calico-typha
	spec:
	maxUnavailable: 1
	selector:
	matchLabels:
	k8s-app: calico-typha

	---
	# This manifest installs the calico/node container, as well
	# as the Calico CNI plugins and network config on
	# each master and worker node in a Kubernetes cluster.
	kind: DaemonSet
	apiVersion: extensions/v1beta1
	metadata:
	name: calico-node
	namespace: kube-system
	labels:
	k8s-app: calico-node
	spec:
	selector:
	matchLabels:
	k8s-app: calico-node
	updateStrategy:
	type: RollingUpdate
	rollingUpdate:
	maxUnavailable: 1
	template:
	metadata:
	labels:
	k8s-app: calico-node
	annotations:
	# This, along with the CriticalAddonsOnly toleration below,
	# marks the pod as a critical add-on, ensuring it gets
	# priority scheduling and that its resources are reserved
	# if it ever gets evicted.
	scheduler.alpha.kubernetes.io/critical-pod: ''
	spec:
	nodeSelector:
	beta.kubernetes.io/os: linux
	hostNetwork: true
	tolerations:
	# Make sure calico-node gets scheduled on all nodes.
	- effect: NoSchedule
	operator: Exists
	# Mark the pod as a critical add-on for rescheduling.
	- key: CriticalAddonsOnly
	operator: Exists
	- effect: NoExecute
	operator: Exists
	serviceAccountName: calico-node
	# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
	# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
	terminationGracePeriodSeconds: 0
	initContainers:
	# This container installs the Calico CNI binaries
	# and CNI network config file on each node.
	- name: install-cni
	image: calico/cni:v3.5.0
	command: ["/install-cni.sh"]
	env:
	# Name of the CNI config file to create.
	- name: CNI_CONF_NAME
	value: "10-calico.conflist"
	# The CNI network config to install on each node.
	- name: CNI_NETWORK_CONFIG
	valueFrom:
	configMapKeyRef:
	name: calico-config
	key: cni_network_config
	# Set the hostname based on the k8s node name.
	- name: KUBERNETES_NODE_NAME
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	# CNI MTU Config variable
	- name: CNI_MTU
	valueFrom:
	configMapKeyRef:
	name: calico-config
	key: veth_mtu
	# Prevents the container from sleeping forever.
	- name: SLEEP
	value: "false"
	volumeMounts:
	- mountPath: /host/opt/cni/bin
	name: cni-bin-dir
	- mountPath: /host/etc/cni/net.d
	name: cni-net-dir
	containers:
	# Runs calico/node container on each Kubernetes node. This
	# container programs network policy and routes on each
	# host.
	- name: calico-node
	image: calico/node:v3.5.0
	env:
	# Use Kubernetes API as the backing datastore.
	- name: DATASTORE_TYPE
	value: "kubernetes"
	# Typha support: controlled by the ConfigMap.
	- name: FELIX_TYPHAK8SSERVICENAME
	valueFrom:
	configMapKeyRef:
	name: calico-config
	key: typha_service_name
	# Wait for the datastore.
	- name: WAIT_FOR_DATASTORE
	value: "true"
	# Set based on the k8s node name.
	- name: NODENAME
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	# Choose the backend to use.
	- name: CALICO_NETWORKING_BACKEND
	valueFrom:
	configMapKeyRef:
	name: calico-config
	key: calico_backend
	# Cluster type to identify the deployment type
	- name: CLUSTER_TYPE
	value: "k8s,bgp"
	# Auto-detect the BGP IP address.
	- name: IP
	value: "autodetect"
	# Enable IPIP
	- name: CALICO_IPV4POOL_IPIP
	value: "Always"
	# Set MTU for tunnel device used if ipip is enabled
	- name: FELIX_IPINIPMTU
	valueFrom:
	configMapKeyRef:
	name: calico-config
	key: veth_mtu
	# The default IPv4 pool to create on startup if none exists. Pod IPs will be
	# chosen from this range. Changing this value after installation will have
	# no effect. This should fall within `--cluster-cidr`.
	- name: CALICO_IPV4POOL_CIDR
	value: "10.25.0.0/22"
	# Disable file logging so `kubectl logs` works.
	- name: CALICO_DISABLE_FILE_LOGGING
	value: "true"
	# Set Felix endpoint to host default action to ACCEPT.
	- name: FELIX_DEFAULTENDPOINTTOHOSTACTION
	value: "ACCEPT"
	# Disable IPv6 on Kubernetes.
	- name: FELIX_IPV6SUPPORT
	value: "false"
	# Set Felix logging to "info"
	- name: FELIX_LOGSEVERITYSCREEN
	value: "info"
	- name: FELIX_HEALTHENABLED
	value: "true"
	securityContext:
	privileged: true
	resources:
	requests:
	cpu: 250m
	livenessProbe:
	httpGet:
	path: /liveness
	port: 9099
	host: localhost
	periodSeconds: 10
	initialDelaySeconds: 10
	failureThreshold: 6
	readinessProbe:
	exec:
	command:
	- /bin/calico-node
	- -bird-ready
	- -felix-ready
	periodSeconds: 10
	volumeMounts:
	- mountPath: /lib/modules
	name: lib-modules
	readOnly: true
	- mountPath: /run/xtables.lock
	name: xtables-lock
	readOnly: false
	- mountPath: /var/run/calico
	name: var-run-calico
	readOnly: false
	- mountPath: /var/lib/calico
	name: var-lib-calico
	readOnly: false
	volumes:
	# Used by calico/node.
	- name: lib-modules
	hostPath:
	path: /lib/modules
	- name: var-run-calico
	hostPath:
	path: /var/run/calico
	- name: var-lib-calico
	hostPath:
	path: /var/lib/calico
	- name: xtables-lock
	hostPath:
	path: /run/xtables.lock
	type: FileOrCreate
	# Used to install CNI.
	- name: cni-bin-dir
	hostPath:
	path: /opt/cni/bin
	- name: cni-net-dir
	hostPath:
	path: /etc/cni/net.d
	---

	apiVersion: v1
	kind: ServiceAccount
	metadata:
	name: calico-node
	namespace: kube-system

	---
	# Create all the CustomResourceDefinitions needed for
	# Calico policy and networking mode.

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: felixconfigurations.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: FelixConfiguration
	plural: felixconfigurations
	singular: felixconfiguration
	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: bgppeers.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: BGPPeer
	plural: bgppeers
	singular: bgppeer

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: bgpconfigurations.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: BGPConfiguration
	plural: bgpconfigurations
	singular: bgpconfiguration

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: ippools.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: IPPool
	plural: ippools
	singular: ippool

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: hostendpoints.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: HostEndpoint
	plural: hostendpoints
	singular: hostendpoint

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: clusterinformations.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: ClusterInformation
	plural: clusterinformations
	singular: clusterinformation

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: globalnetworkpolicies.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: GlobalNetworkPolicy
	plural: globalnetworkpolicies
	singular: globalnetworkpolicy

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: globalnetworksets.crd.projectcalico.org
	spec:
	scope: Cluster
	group: crd.projectcalico.org
	version: v1
	names:
	kind: GlobalNetworkSet
	plural: globalnetworksets
	singular: globalnetworkset

	---

	apiVersion: apiextensions.k8s.io/v1beta1
	kind: CustomResourceDefinition
	metadata:
	name: networkpolicies.crd.projectcalico.org
	spec:
	scope: Namespaced
	group: crd.projectcalico.org
	version: v1
	names:
	kind: NetworkPolicy
	plural: networkpolicies
	singular: networkpolicy
	---

	# Include a clusterrole for the calico-node DaemonSet,
	# and bind it to the calico-node serviceaccount.
	kind: ClusterRole
	apiVersion: rbac.authorization.k8s.io/v1beta1
	metadata:
	name: calico-node
	rules:
	# The CNI plugin needs to get pods, nodes, and namespaces.
	- apiGroups: [""]
	resources:
	- pods
	- nodes
	- namespaces
	verbs:
	- get
	- apiGroups: [""]
	resources:
	- endpoints
	- services
	verbs:
	# Used to discover service IPs for advertisement.
	- watch
	- list
	# Used to discover Typhas.
	- get
	- apiGroups: [""]
	resources:
	- nodes/status
	verbs:
	# Needed for clearing NodeNetworkUnavailable flag.
	- patch
	# Calico stores some configuration information in node annotations.
	- update
	# Watch for changes to Kubernetes NetworkPolicies.
	- apiGroups: ["networking.k8s.io"]
	resources:
	- networkpolicies
	verbs:
	- watch
	- list
	# Used by Calico for policy information.
	- apiGroups: [""]
	resources:
	- pods
	- namespaces
	- serviceaccounts
	verbs:
	- list
	- watch
	# The CNI plugin patches pods/status.
	- apiGroups: [""]
	resources:
	- pods/status
	verbs:
	- patch
	# Calico monitors various CRDs for config.
	- apiGroups: ["crd.projectcalico.org"]
	resources:
	- globalfelixconfigs
	- felixconfigurations
	- bgppeers
	- globalbgpconfigs
	- bgpconfigurations
	- ippools
	- globalnetworkpolicies
	- globalnetworksets
	- networkpolicies
	- clusterinformations
	- hostendpoints
	verbs:
	- get
	- list
	- watch
	# Calico must create and update some CRDs on startup.
	- apiGroups: ["crd.projectcalico.org"]
	resources:
	- ippools
	- felixconfigurations
	- clusterinformations
	verbs:
	- create
	- update
	# Calico stores some configuration information on the node.
	- apiGroups: [""]
	resources:
	- nodes
	verbs:
	- get
	- list
	- watch
	# These permissions are only requried for upgrade from v2.6, and can
	# be removed after upgrade or on fresh installations.
	- apiGroups: ["crd.projectcalico.org"]
	resources:
	- bgpconfigurations
	- bgppeers
	verbs:
	- create
	- update
	---
	apiVersion: rbac.authorization.k8s.io/v1beta1
	kind: ClusterRoleBinding
	metadata:
	name: calico-node
	roleRef:
	apiGroup: rbac.authorization.k8s.io
	kind: ClusterRole
	name: calico-node
	subjects:
	- kind: ServiceAccount
	name: calico-node
	namespace: kube-system
	---