.gitignore

pki

cleanup.sh

#!/usr/bin/env bash
set -euxo pipefail
IFS=$'\n\t'

declare -r SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

source "${SCRIPT_DIR}/variables.sh"

docker rm -fv "${REGISTRY_NAME}" || true
kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
rm -rf pki

kind-airgapped-config.yaml

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
featureGates:
  EphemeralContainers: true
nodes:
  - role: control-plane
    extraMounts:
    - hostPath: ./pki/
      containerPath: /etc/kubernetes/pki/kind/
containerdConfigPatches:
  - |-
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
      endpoint = ["https://${REGISTRY_NAME}"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
      endpoint = ["https://${REGISTRY_NAME}/v2/k8s.gcr.io"]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."*"]
      endpoint = ["https://${REGISTRY_NAME}"]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."${REGISTRY_NAME}".tls]
      ca_file = "/etc/kubernetes/pki/kind/${REGISTRY_NAME}.crt"

test-k8s-gcr-mirroring.sh

#!/usr/bin/env bash
set -euxo pipefail
IFS=$'\n\t'

SUDO=''
if ((EUID != 0)); then
  SUDO='sudo'
fi

declare -r SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "${SCRIPT_DIR}/variables.sh"

export KIND_EXPERIMENTAL_DOCKER_NETWORK="${KIND_CLUSTER_NAME}"
if ! docker network inspect "${KIND_EXPERIMENTAL_DOCKER_NETWORK}" &>/dev/null; then
  docker network create --driver bridge --internal "${KIND_EXPERIMENTAL_DOCKER_NETWORK}" --subnet "172.19.0.0/16"
fi

declare -r REGISTRY_IP="172.19.0.3"

# create registry container unless it already exists
if ! docker inspect "${REGISTRY_NAME}" &>/dev/null; then

  mkdir -p "${SCRIPT_DIR}/pki/"
  openssl req \
    -newkey rsa:4096 \
    -days 7 \
    -nodes \
    -x509 \
    -subj "/CN=${REGISTRY_NAME}" \
    -extensions SAN \
    -config <(cat "$([[ -f /System/Library/OpenSSL/openssl.cnf ]] && echo /System/Library/OpenSSL/openssl.cnf || echo /etc/ssl/openssl.cnf)" \
      <(printf "[SAN]\nsubjectAltName='DNS.1:%s, IP.1:'%s" "${REGISTRY_NAME}" "${REGISTRY_IP}")) \
    -keyout "${SCRIPT_DIR}/pki/${REGISTRY_NAME}.key" \
    -out "${SCRIPT_DIR}/pki/${REGISTRY_NAME}.crt"

  ${SUDO} mkdir -p "/etc/docker/certs.d/${REGISTRY_IP}/"
  ${SUDO} cp "${SCRIPT_DIR}/pki/${REGISTRY_NAME}.crt" "/etc/docker/certs.d/${REGISTRY_IP}/ca.crt"
  ${SUDO} ls -latrh /etc/docker/certs.d/*

  docker run \
    -d --restart=always --name "${REGISTRY_NAME}" \
    --network "${KIND_EXPERIMENTAL_DOCKER_NETWORK}" \
    -v "${SCRIPT_DIR}/pki/${REGISTRY_NAME}.key":/certs/tls.key \
    -v "${SCRIPT_DIR}/pki/${REGISTRY_NAME}.crt":/certs/tls.crt \
    -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \
    -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/tls.crt \
    -e REGISTRY_HTTP_TLS_KEY=/certs/tls.key \
    -p 443:443 \
    --ip "${REGISTRY_IP}" \
    registry:2
fi

# The create cluster currently fails on an internal network with `failed to get api server port`. Let's
# ignore the error and assume it started up correctly...
kind get clusters 2>/dev/null | grep -E "^${KIND_CLUSTER_NAME}$" &>/dev/null || \
  envsubst < "${SCRIPT_DIR}/kind-airgapped-config.yaml" | kind create cluster --name "${KIND_CLUSTER_NAME}" --config - 2>/dev/null || true

# admin.conf is only readably by root so changing file mode here is necessary in order to run kubectl as another user with `docker exec --user`
docker exec "${KIND_CLUSTER_NAME}"-control-plane chmod 0644 /etc/kubernetes/admin.conf

declare -r DOCKER_CMD=(docker exec -i "${KIND_CLUSTER_NAME}-control-plane")

# Check that there is no access to docker hub directly from kind cluster
if "${DOCKER_CMD[@]}" curl -fsSL --connect-timeout 2 https://index.docker.io; then
  echo "Should not have been able to connect to Docker Hub from airgapped kind cluster"
  exit 1
fi

# Test pulling docker hub images via mirror is working OK
declare -r BUSYBOX_IMAGE='busybox:latest'
docker pull "${BUSYBOX_IMAGE}"
docker tag busybox:latest "${REGISTRY_IP}/${BUSYBOX_IMAGE}"
docker push "${REGISTRY_IP}/library/${BUSYBOX_IMAGE}"
"${DOCKER_CMD[@]}" crictl pull "${BUSYBOX_IMAGE}"
"${DOCKER_CMD[@]}" crictl inspecti "${BUSYBOX_IMAGE}"

# Check that there is no access to k8s.gcr.io directly from kind cluster
if "${DOCKER_CMD[@]}" curl -fsSL --connect-timeout 2 https://k8s.gcr.io; then
  echo "Should not have been able to connect to Docker Hub from airgapped kind cluster"
  exit 1
fi

# Now test pulling Kubernetes GCR images from mirror works
declare -r APISERVER_IMAGE='k8s.gcr.io/kube-apiserver:v1.22.1'
docker pull "${APISERVER_IMAGE}"
docker tag "${APISERVER_IMAGE}" "${REGISTRY_IP}/${APISERVER_IMAGE}"
docker push "${REGISTRY_IP}/${APISERVER_IMAGE}"
"${DOCKER_CMD[@]}" crictl pull "${APISERVER_IMAGE}"
"${DOCKER_CMD[@]}" crictl inspecti "${APISERVER_IMAGE}"

variables.sh

#!/usr/bin/env bash

declare -r KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-k8s-gcr-docker-mirror-test}"
declare -r REGISTRY_NAME="${REGISTRY_NAME:-"${KIND_CLUSTER_NAME}-registry"}"
export REGISTRY_NAME