k3s dind rootless devcontainer

docker-compose.yaml

version: '3'
services:
  dev:
    build:
      context: .
      dockerfile: Dockerfile
      args:
        USER_UID: ${USER_UID:-1000}
        USER_GID: ${USER_GID:-1000}
        DOCKER_SOCKET_GIT: ${DOCKER_SOCKET_GID:-966}
        PROJECT_DIR: /workspace
    environment:
      PATH: /home/user/.nix-profile/bin:/home/user/.local/bin:/usr/local/bin:/usr/bin:/bin
      KUBECONFIG: /var/run/k3s-kubeconfig/kubeconfig.yaml
      DOCKER_HOST: tcp://localhost:2375
      MINIKUBE_DRIVER: kvm2
      MINIKUBE_CONTAINER_RUNTIME: containerd
      MINIKUBE_KVM_QEMU_URI: qemu+tcp://127.0.0.1:16509/system
    command: sleep infinity
    volumes:
      - ..:/workspace:cached
      - kubeconfig:/var/run/k3s-kubeconfig
      - pulumi:/home/user/.pulumi
      - nix:/nix
      - direnv-allow:/home/user/.config/direnv/allow
    security_opt:
      - label:disable
    depends_on:
      - k3s
      - docker
    network_mode: "service:k3s"

  docker:
    image: xtruder/dind-rootless:latest
    command: ["--insecure-registry=registry.kube-system.svc.cluster.local:5000"]
    environment:
      DOCKER_TLS_CERTDIR: ""
      DOCKER_DRIVER: fuse-overlayfs
    volumes:
      - ..:/workspace:cached
      - docker:/var/lib/docker
    privileged: yes
    security_opt:
      - label:disable
    depends_on:
      - k3s
    network_mode: "service:k3s"

  teleperesence:
    image: xtruder/telepresence:latest
    tty: true
    environment:
      SCOUT_DISABLE: "1"
      KUBECONFIG: /var/run/k3s-kubeconfig/kubeconfig.yaml
    command: ["--method", "vpn-tcp", "--namespace", "kube-system", "--run", "/bin/sleep", "infinity"]
    cap_add:
      - NET_ADMIN
      - NET_BIND_SERVICE
    volumes:
      - kubeconfig:/var/run/k3s-kubeconfig
    restart: always
    depends_on:
      - k3s
    network_mode: "service:k3s"

  socks5:
    image: serjs/go-socks5-proxy
    restart: always
    depends_on:
      - k3s
    network_mode: "service:k3s"

  k3s:
    image: xtruder/k3s-rootless:latest
    command: server --rootless --snapshotter fuse-overlayfs --data-dir /var/lib/rancher/k3s-rootless
    environment:
    - K3S_NODE_NAME=k3s
    - K3S_TOKEN=${K3S_TOKEN:-29338293525080}
    - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
    - K3S_KUBECONFIG_MODE=666
    volumes:
    - k3s-server:/var/lib/rancher/k3s-rootless
    # This is just so that we get the kubeconfig file out
    - kubeconfig:/output
    - ./registries.yaml:/etc/rancher/k3s/registries.yaml
    privileged: true
    security_opt:
      - label:disable
    sysctls:
      - net.ipv4.ip_forward=1
    network_mode: bridge

volumes:
  nix:
  direnv-allow:
  k3s-server:
  docker:
  kubeconfig:
  pulumi:

Dockerfile

FROM xtruder/debian-nix-devcontainer:flakes

# docker user
ARG DOCKER_GID=966
RUN groupadd -g ${DOCKER_GID} docker && usermod -a -G docker ${USERNAME} 

# create volume for pulumi
RUN sudo -u user mkdir -p /home/${USERNAME}/.pulumi
VOLUME /home/${USERNAME}/.pulumi