Skip to content

Instantly share code, notes, and snippets.

@gustavohenrique

gustavohenrique/k8s-lxd.md

Last active March 8, 2023 13:24
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save gustavohenrique/9c57ab06ba06a60f4caebeba8109bc93 to your computer and use it in GitHub Desktop.
Save gustavohenrique/9c57ab06ba06a60f4caebeba8109bc93 to your computer and use it in GitHub Desktop.
Download ZIP
Configure kubernetes on LXD containers
Raw
k8s-lxd.md

Create LXD containers

cat > k8s-profile.yaml <<EOF
config:
  linux.kernel_modules: xt_conntrack,ip_tables,ip6_tables,netlink_diag,nf_nat,overlay
  raw.lxc: "lxc.apparmor.profile=unconfined\nlxc.cap.drop= \nlxc.cgroup.devices.allow=a\nlxc.mount.auto=proc:rw sys:rw\nlxc.mount.entry=/dev/kmsg dev/kmsg none defaults,bind,create=file"
  security.privileged: "true"
  security.nesting: "true"
EOF
lxc profile create k8s
lxc profile edit k8s < k8s-profile.yaml

lxc launch images:ubuntu/22.04 --profile default k8s-main
lxc config device add k8s-main "kmsg" unix-char source="/dev/kmsg" path="/dev/kmsg"
lxc exec k8s-main reboot

lxc launch images:ubuntu/22.04 --profile default k8s-worker1
lxc config device add k8s-worker1 "kmsg" unix-char source="/dev/kmsg" path="/dev/kmsg"
lxc exec k8s-worker1 reboot

Control-Plane

Inside container:

# lxc exec k8s-main bash
apt update && apt install -y conntrack apt-transport-https ca-certificates curl gnupg2 software-properties-common net-tools

# Add kubernetes repo
curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list

# Add docker repo for containerd.io
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list

# Install components
apt update && apt install -y kubeadm kubelet kubectl kubernetes-cni containerd.io

# Configure containerd instead of Docker
mkdir -p /etc/containerd 2>/dev/null
containerd config default | tee /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
systemctl restart containerd

# Configure kubeadm
export CIDR="10.10.10.0/24"

cat > kubeadm-config.yaml <<EOF
kind: ClusterConfiguration
apiVersion: kubeadm.k8s.io/v1beta3
kubernetesVersion: v1.26.0
networking:
  podSubnet: "$CIDR"
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd
EOF

# echo 'L /dev/kmsg - - - - /dev/null' > /etc/tmpfiles.d/kmsg.conf
echo 'L /dev/kmsg - - - - /dev/console' > /etc/tmpfiles.d/kmsg.conf

# Run kubeadm
kubeadm init --config kubeadm-config.yaml --skip-certificate-key-print  --ignore-preflight-errors=FileContent--proc-sys-net-bridge-bridge-nf-call-iptables --upload-certs | tee kubeadm-init.out

# Configure kubectl
mkdir -p ~/.kube 2>/dev/null
sudo cp -i /etc/kubernetes/admin.conf ~/.kube/config
sudo chown $(id -u):$(id -g) ~/.kube/config

# Configure Flannel
curl -sL0 -o kube-flannel.yml https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
sed -i "s+10.244.0.0\/16+$CIDR+g" kube-flannel.yml
kubectl apply -f kube-flannel.yml

# Allow control plane to run pods
kubectl taint nodes k8s node-role.kubernetes.io/control-plane-

Worker

Inside container:

apt update && apt install -y conntrack apt-transport-https ca-certificates curl gnupg2 software-properties-common net-tools

# Add kubernetes repo
curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list

# Install the components
apt update && apt install -y kubeadm kubelet kubectl kubernetes-cni containerd.io

# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
# echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

apt update && apt install -y kubeadm kubelet containerd.io

mkdir -p /etc/containerd 2>/dev/null
containerd config default | tee /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
systemctl restart containerd

#kubeadm join k8s:6443 --token cm81hx.ubshewralew9rqmh --discovery-token-ca-cert-hash sha256:1d27bebde256e65462f2b7774e556a0a5c73b9aeec27d99dd5901595163e7355 --skip-phases=preflight  | tee kubeadm-init.out

# Copy /etc/kubernetes/admin.conf from master to ~/.kube/config
kubeadm join --discovery-file ~/.kube/config

# Set CIDR on node
kubectl patch node k8s-worker1 -p '{"spec":{"podCIDR":"10.10.10.0/24"}}'

Ingress

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.6.4/deploy/static/provider/cloud/deploy.yaml
kubectl delete -A ValidatingWebhookConfiguration ingress-nginx-admission

Hello World

cat > helloworld.yaml <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hello-world-deployment
  labels:
    app: hello-world
spec:
  selector:
    matchLabels:
      app: hello-world
  replicas: 2
  template:
    metadata:
      labels:
        app: hello-world
    spec:
      containers:
      - name: hello-world
        image: gustavohenrique/helloworld-nodejs
        ports:
        - containerPort: 3000
        resources:
          limits:
            memory: 256Mi
            cpu: "250m"
          requests:
            memory: 128Mi
            cpu: "80m"
---
apiVersion: v1
kind: Service
metadata:
  name: hello-world-node-port
spec:
  selector:
    app: hello-world
  type: NodePort
  ports:
    - protocol: TCP
      port: 80
      targetPort: 3000
      nodePort: 30081
---
apiVersion: v1
kind: Service
metadata:
  name: hello-world
spec:
  selector:
    app: hello-world
  type: ClusterIP
  ports:
    - protocol: TCP
      port: 3000
      targetPort: 3000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: hello-world
spec:
  ingressClassName: nginx
  rules:
  - host: hello-world.mydomain.com
    http:
      paths:
      - pathType: Prefix
        path: /
        backend:
          service:
            name: hello-world
            port:
              number: 3000

EOF

kubectl create ns apps
kubectl -n apps apply -f helloworld.yaml
kubectl -n apps get deployments

curl localhost:30081
# echo "127.0.1.1 hello-world.mydomain.com" >> /etc/hosts
curl hello-world.mydomain.com

Troubleshoot

Service Account token expired:

cat > secret.yaml <<EOF
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
  name: default
  annotations: 
    kubernetes.io/service-account.name: default
EOF
kubectl -n default apply -f secret.yaml
kubectl -n default edit sa default
...
secrets:
- default

Cheatsheet

Remove node

kubectl drain --ignore-daemonsets k8s-worker1
kubectl delete node k8s-worker1

Other commands

kubectl scale --replicas=5
kubectl config set-context --current --namespace=ggckad-s2

kubectl get nodes --show-labels
kubectl label --list nodes node_name
kubectl label nodes k8s-node1 varnish=production
kubectl get svc -l host=varnish1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment