100% Offline installation on Ubuntu with kubeadm for Kubernetes 1.15.3

aws.ini

[Global]

[ServiceOverride "ec2"]
URL = "https://ec2.${AWS_ENDPOINT_URL}"
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = ec2

[ServiceOverride "elasticloadbalancing"]
URL = https://elasticloadbalancing.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = elasticloadbalancing

[ServiceOverride "autoscaling"]
URL = https://autoscaling.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = autoscaling

[ServiceOverride "kms"]
URL = https://kms.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = kms

[ServiceOverride "sts"]
URL = https://sts.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = sts

[ServiceOverride "metadata"]
URL = http://169.254.169.254
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = metadata

offline-containerd-install.sh

# As root

## Incomplete, switched to Docker for no good reason I later found out.

# Unpack
tar -xvf ~ubuntu/packages.deb.tar.gz -C /var/cache/apt/archives/
tar -xvf ~ubuntu/packages.containers.tar.gz -C /tmp

cat > /etc/modules-load.d/containerd.conf <<EOF
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

# Setup required sysctl params, these persist across reboots.
cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sysctl --system

apt install -y /var/cache/apt/archives/*.deb

systemctl daemon-reload

containerd config default > /etc/containerd/config.toml

systemctl enable --now containerd kubelet
# kubelet will crash loop until configured, that's fine.

for img in /tmp/*.tar
do
  ctr cri load /tmp/${img}
done

touch /etc/kubernetes/aws.ini
# Do some magic voodoo in this file if necessary, wink wink nudge nudge how do you do governor

cat > /tmp/kubeadm.confg.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
nodeRegistration:
  kubeletExtraArgs:
    cloud-provider: "aws"
    cloud-config: "/etc/kubernetes/aws.ini"
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: 1.15.3
networking:
  podSubnet: 10.244.0.0/16
apiServer:
  extraArgs:
    cloud-provider: "aws"
    cloud-config: "/etc/kubernetes/aws.ini"
  extraVolumes:
  - name: cloud
    hostPath: "/etc/kubernetes/aws.ini"
    mountPath: "/etc/kubernetes/aws.ini"
controllerManager:
  extraArgs:
    cloud-provider: "aws"
    cloud-config: "/etc/kubernetes/aws.ini"
  extraVolumes:
  - name: cloud
    hostPath: "/etc/kubernetes/aws.ini"
    mountPath: "/etc/kubernetes/aws.ini"
EOF

## No AWS for testing purposes
# cat > /tmp/kubeadm.confg.yaml <<EOF
# ---
# apiVersion: kubeadm.k8s.io/v1beta2
# kind: ClusterConfiguration
# kubernetesVersion: 1.15.3
# networking:
#   podSubnet: 10.244.0.0/16
# EOF

kubeadm init --config /tmp/kubeadm.confg.yaml

KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f canal.yaml

## exit root
# exit

# mkdir -p $HOME/.kube
# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
# sudo chown $(id -u):$(id -g) $HOME/.kube/config

# kubectl get nodes

offline-docker-install.sh

# As root

mkdir -p /etc/systemd/system/docker.service.d /etc/docker/

# Copy over CA certs and update
cp ~ubuntu/*.crt /usr/local/share/ca-certificates/
update-ca-certificates

# Get your DNS server for Docker

# Setup daemon.
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "dns": ["$(grep nameserver /run/systemd/resolve/resolv.conf | awk '{print $2}')"]
}
EOF

tar -xvf ~ubuntu/packages.deb.tar.gz -C /
apt install -y /var/cache/apt/archives/*.deb

systemctl daemon-reload

systemctl enable --now docker kubelet

mkdir -p /tmp/containers
tar -xvf ~ubuntu/packages.containers.tar.gz -C /tmp/containers

pushd /tmp/containers
for image in $(ls -1 *.tar)
do
  docker load -i ${image}
done

export AWS_REGION="$(curl -SsL http://169.254.169.254/latest/meta-data/placement/availability-zone/ | rev | cut -c 2- | rev)"
export AWS_ENDPOINT_URL="us-east-1.foo.baz.okay"

## With AWS configured
cat > /etc/kubernetes/aws.ini <<EOF
[Global]

[ServiceOverride "ec2"]
URL = "https://ec2.${AWS_ENDPOINT_URL}"
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = ec2

[ServiceOverride "elasticloadbalancing"]
URL = https://elasticloadbalancing.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = elasticloadbalancing

[ServiceOverride "autoscaling"]
URL = https://autoscaling.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = autoscaling

[ServiceOverride "kms"]
URL = https://kms.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = kms

[ServiceOverride "sts"]
URL = https://sts.${AWS_ENDPOINT_URL}
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = sts

[ServiceOverride "metadata"]
URL = http://169.254.169.254
SigningRegion = "${AWS_REGION}"
Region = "${AWS_REGION}"
SigningMethod = v4
Service = metadata

EOF

# sed -isave -e "s|\${AWS_REGION}|${AWS_REGION}|g" -e "s|\${VPC_ID}|${VPC_ID}|g" -e "s|\${AWS_ENDPOINT_URL}|${AWS_ENDPOINT_URL}|g" 

cat > /tmp/kubeadm.confg.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
nodeRegistration:
  kubeletExtraArgs:
    cloud-provider: "aws"
    cloud-config: "/etc/kubernetes/aws.ini"
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: 1.15.3
networking:
  podSubnet: 10.244.0.0/16
apiServer:
  extraArgs:
    cloud-provider: "aws"
    cloud-config: "/etc/kubernetes/aws.ini"
  extraVolumes:
  - name: cloud
    hostPath: "/etc/kubernetes/aws.ini"
    mountPath: "/etc/kubernetes/aws.ini"
controllerManager:
  extraArgs:
    cloud-provider: "aws"
    cloud-config: "/etc/kubernetes/aws.ini"
    cluster-name: your_cluster_id
  extraVolumes:
  - name: cloud
    hostPath: "/etc/kubernetes/aws.ini"
    mountPath: "/etc/kubernetes/aws.ini"
EOF

## No AWS
# cat > /tmp/kubeadm.confg.yaml <<EOF
# ---
# apiVersion: kubeadm.k8s.io/v1beta2
# kind: ClusterConfiguration
# kubernetesVersion: 1.15.3
# networking:
#   podSubnet: 10.244.0.0/16
# EOF

kubeadm init --config /tmp/kubeadm.confg.yaml --node-name $(curl -SL http://169.254.169.254/latest/meta-data/hostname)

# exit root
# exit

# mkdir -p $HOME/.kube && sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && sudo chown $(id -u):$(id -g) $HOME/.kube/config

# kubectl apply -f canal.yaml

# kubectl taint nodes --all node-role.kubernetes.io/master-

online-prep.sh

# This should be done on an internet accessible machine as root.

apt update

### Install packages to allow apt to use a repository over HTTPS
apt install --download-only --force-reinstall true apt-transport-https ca-certificates curl software-properties-common
apt install -y apt-transport-https ca-certificates curl software-properties-common

# Download containerd
## Set up the repository

### Add Docker’s official GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
### Now Kubernetes
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -

### Add Docker apt repository.
add-apt-repository \
  "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) \
  stable"
    
add-apt-repository \
  "deb [arch=amd64] https://apt.kubernetes.io/ kubernetes-xenial main"

## Install Docker
apt-get update && apt-get install --force-reinstall true --download-only kubelet kubeadm kubectl docker-ce=18.06.2~ce~3-0~ubuntu
## Or containerd
# apt-get update && apt-get install --download-only kubelet kubeadm kubectl containerd.io 

## Files will start building up in here.
# /var/cache/apt/archives/
tar -czvf packages.deb.tar.gz /var/cache/apt/archives/*.deb

# Actually install docker so we can grab images.
apt install -y docker-ce=18.06.2~ce~3-0~ubuntu

# Setup daemon.
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

systemctl daemon-reload

systemctl enable --now docker

cat <<EOF >images.yaml
calico/cni:v3.8.2
calico/node:v3.8.2
calico/pod2daemon-flexvol:v3.8.2
docker.io/calico/cni:v3.8.2
docker.io/calico/node:v3.8.2
docker.io/calico/pod2daemon-flexvol:v3.8.2
k8s.gcr.io/coredns:1.3.1
k8s.gcr.io/etcd:3.3.10
k8s.gcr.io/kube-apiserver:v1.15.3
k8s.gcr.io/kube-controller-manager:v1.15.3
k8s.gcr.io/kube-proxy:v1.15.3
k8s.gcr.io/kube-scheduler:v1.15.3
k8s.gcr.io/pause:3.1
k8s.gcr.io/pause:3.1
quay.io/coreos/flannel:v0.11.0
busybox
nginx
voor/aws-cli
EOF

{
  mkdir -p repository
  for image in $(cat images.yaml)
  do
      tarname="$(echo -n ${image} | sed -e 's/\//-/g' -e 's/:/-/').container.tar"
      docker pull ${image}
      docker save -o ${tarname} ${image}
      echo "${tarname} ${image}" >> tarballs.yaml
  done
}

tar -czvf packages.containers.tar.gz *.yaml *.container.tar

curl -SsL -o canal.yaml https://docs.projectcalico.org/v3.8/manifests/canal.yaml

# Copy it all over, should be 3 files.
scp packages.containers.tar.gz packages.deb.tar.gz canal.yaml ${DESTINATION}:~/

troubleshooting.sh

export AWS_DEFAULT_REGION="someplace"
export AWS_ENDPOINT_URL="youknowhere"

# Get a container with AWS CLI
docker run -it --rm --name=aws-cli --entrypoint /bin/sh -v $PWD/ca:/ca -e AWS_DEFAULT_REGION -e AWS_ENDPOINT_URL -e AWS_CA_BUNDLE=/ca/ca-chain.cert.pem  voor/aws-cli

# Get STS credentials
export AWS_SESSION_TOKEN=$(curl -SsL http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance/ | jq -r -c '.Token')

aws ec2 describe-instances --endpoint-url https://ec2.${AWS_DEFAULT_REGION}.${AWS_ENDPOINT_URL}

aws ec2 create-tags --resources XXX --tags Key=kubernetes.io/cluster/your_cluster_id,Value=shared