dind k3s devcontainers root

docker-compose.yaml

version: '3'
services:
  dev:
    build:
      context: .
      dockerfile: Dockerfile
      args:
        USER_UID: ${USER_UID:-1000}
        USER_GID: ${USER_GID:-1000}
        DOCKER_SOCKET_GIT: ${DOCKER_SOCKET_GID:-966}
        PROJECT_DIR: /workspace
    environment:
      PATH: /home/user/.nix-profile/bin:/home/user/.local/bin:/usr/local/bin:/usr/bin:/bin
      KUBECONFIG: /var/run/k3s-kubeconfig/kubeconfig.yaml
      DOCKER_HOST: tcp://localhost:2375
    command: sleep infinity
    volumes:
      - ..:/workspace:cached
      - kubeconfig:/var/run/k3s-kubeconfig
      - pulumi:/home/user/.pulumi
      - nix:/nix
      - direnv-allow:/home/user/.config/direnv/allow
    security_opt:
      - label:disable
    depends_on:
      - k3s
      - docker
      - dnsmasq
    network_mode: "service:k3s"

  docker:
    build:
      context: .
      dockerfile: Dockerfile-dind
    command: ["--insecure-registry=registry.kube-system.svc.cluster.local:5000"]
    environment:
      DOCKER_TLS_CERTDIR: ""
      DOCKER_DRIVER: fuse-overlayfs
      DOCKERD_ROOTLESS_ROOTLESSKIT_NET: host
      DOCKERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER: "none"
    volumes:
      - ..:/workspace:cached
      - docker:/var/lib/docker
    privileged: yes
    security_opt:
      - label:disable
    depends_on:
      - k3s
      - dnsmasq
    network_mode: "service:k3s"

  dnsmasq:
    image: andyshinn/dnsmasq
    cap_add:
      - NET_ADMIN
    volumes:
    - ./resolv.conf:/run/resolv.conf
    command: --log-facility=- -r /run/resolv.conf -S /cluster.local/10.43.0.10
    security_opt:
      - label:disable
    network_mode: "service:k3s"

  socks5:
    image: serjs/go-socks5-proxy
    restart: always
    network_mode: "service:k3s"

  k3s:
    image: "rancher/k3s:${K3S_VERSION:-latest}"
    command: server
    tmpfs:
    - /run
    - /var/run
    privileged: true
    environment:
    - K3S_NODE_NAME=k3s
    - K3S_TOKEN=${K3S_TOKEN:-29338293525080}
    - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
    - K3S_KUBECONFIG_MODE=666
    - K3S_RESOLV_CONF=/run/k3s/resolv.conf
    volumes:
    - k3s-server:/var/lib/rancher/k3s
    # This is just so that we get the kubeconfig file out
    - kubeconfig:/output
    - ./resolv.conf:/run/k3s/resolv.conf
    - ./registries.yaml:/etc/rancher/k3s/registries.yaml
    security_opt:
      - label:disable
    network_mode: bridge
    dns:
      - 127.0.0.1
    ports:
      - 65233:1080
    dns_search:
      - svc.cluster.local
      - cluster.local

volumes:
  nix:
  direnv-allow:
  k3s-server:
  docker:
  kubeconfig:
  pulumi:

Dockerfile

FROM xtruder/debian-nix-devcontainer:flakes

# docker user
ARG DOCKER_GID=966
RUN groupadd -g ${DOCKER_GID} docker && usermod -a -G docker ${USERNAME} 

# create volume for pulumi
RUN sudo -u user mkdir -p /home/${USERNAME}/.pulumi
VOLUME /home/${USERNAME}/.pulumi

registries.yaml

mirrors:
  "registry.kube-system.svc.cluster.local:5000":
    endpoint:
      - "http://registry.kube-system.svc.cluster.local:5000"

resolv.conf

nameserver 1.1.1.2