k3s for local testing

README.md

K3s for local testing/CI

The following is a docker-compose k3s configuration, with a script to pre-populate images in the cluster, allowing complete tear-down and rebuild very quickly.

This could be useful for spinning up a small cluster during CI jobs, as it allows the images to be cached rather than pulled into the newly built cluster for each and every job.

Results from local testing:

	No preloaded images	Preloaded images
Time to bring cluster up	~3 minutes	~53 seconds

docker-compose.yaml

# to run define K3S_TOKEN, K3S_VERSION is optional, eg:
#   K3S_TOKEN=${RANDOM}${RANDOM}${RANDOM} docker-compose up

version: '3'
services:

  server:
    image: "rancher/k3s:v1.21.5-k3s1"
    command: server
    tmpfs:
    - /run
    - /var/run
    ulimits:
      nproc: 65535
      nofile:
        soft: 65535
        hard: 65535
    privileged: true
    restart: always
    environment:
    - K3S_TOKEN=${K3S_TOKEN:?err}
    - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
    - K3S_KUBECONFIG_MODE=666
    volumes:
    - k3s-server:/var/lib/rancher/k3s
    # This is just so that we get the kubeconfig file out
    - .:/output
    - ./manifests:/var/lib/rancher/k3s/server/manifests
    - ./images:/var/lib/rancher/k3s/agent/images
    ports:
    - 6443:6443  # Kubernetes API Server
    # - 80:80      # Ingress controller port 80
    # - 443:443    # Ingress controller port 443

  agent:
    image: "rancher/k3s:${K3S_VERSION:-latest}"
    tmpfs:
    - /run
    - /var/run
    ulimits:
      nproc: 65535
      nofile:
        soft: 65535
        hard: 65535
    privileged: true
    restart: always
    environment:
    - K3S_URL=https://server:6443
    - K3S_TOKEN=${K3S_TOKEN:?err}
    volumes:
    - ./images:/var/lib/rancher/k3s/agent/images

volumes:
  k3s-server: {}

mount-images.sh

#!/usr/bin/env bash
# Run this script to populate the images directory with saved docker images.  These will then be
# mounted onto the k3s server and agents, skipping image pull steps.
images=(
    rancher/klipper-helm:v0.6.4-build20210813
    rancher/library-traefik:2.4.8
    rancher/coredns-coredns:1.8.3
    rancher/local-path-provisioner:v0.0.19
    rancher/klipper-lb:v0.2.0
    rancher/metrics-server:v0.3.6
)

mkdir -p images

for image_tag in ${images[@]}
do
    docker pull $image_tag
    docker save $image_tag -o images/$(echo $image_tag | sed -e 's/[^A-Za-z0-9._-]/-/g').tar
done

time-up.sh

#!/usr/bin/env bash

# Time how long it takes to bring the cluster to ready state
start=$(date +%s%N)

docker-compose up -d

for i in $(seq 1 20);
do
    [ $i -gt 1 ] && sleep 2; kubectl --kubeconfig kubeconfig.yaml rollout status deployment/traefik -n kube-system && s=0 && break || s=$?
done

end=$(date +%s%N)
runtime=$((end-start))
runtime_seconds=$(echo "scale=2;${runtime}/1000000000" | bc)

echo "Started cluster in ${runtime_seconds}s"

unhealthy_pods=$(kubectl --kubeconfig kubeconfig.yaml get pods --all-namespaces -o json | jq '.items | map(select(.status.phase=="Running" | not)) | map(select(.status.phase=="Succeeded" | not)) | length')

echo "Number of unhealthy pods: ${unhealthy_pods}"

exit $s