k3s-control-plane-nbg1-tmc:/ # du -ah / | sort -n -r | head -n 10
du: cannot access '/proc/5204/task/5204/fd/4': No such file or directory
du: cannot access '/proc/5204/task/5204/fdinfo/4': No such file or directory
du: cannot access '/proc/5204/fd/3': No such file or directory
du: cannot access '/proc/5204/fdinfo/3': No such file or directory
1023M /usr
1020M /.snapshots/2/snapshot
1020M /.snapshots/2
1020K /usr/share/pki/trust/ca-certificates-mozilla.trust.p11-kit
1020K /usr/share/pki/trust
1020K /usr/lib64/python3.11/importlib
1020K /.snapshots/2/snapshot/usr/share/pki/trust/ca-certificates-mozilla.trust.p11-kit
1020K /.snapshots/2/snapshot/usr/share/pki/trust
1020K /.snapshots/2/snapshot/usr/lib64/python3.11/importlib
1020K /.snapshots/1/snapshot/usr/share/pki/trust/ca-certificates-mozilla.trust.p11-kit
k3s-control-plane-nbg1-tmc:/ # df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda2 39G 38G 1.1M 100% /
/dev/sda2 39G 38G 1.1M 100% /root
/dev/sda2 39G 38G 1.1M 100% /var
overlay 39G 38G 1.1M 100% /etc
devtmpfs 4.0M 0 4.0M 0% /dev
tmpfs 1.9G 100K 1.9G 1% /dev/shm
efivarfs 256K 15K 242K 6% /sys/firmware/efi/efivars
tmpfs 764M 68M 697M 9% /run
tmpfs 1.9G 0 1.9G 0% /tmp
/dev/sda2 39G 38G 1.1M 100% /.snapshots
/dev/sda2 39G 38G 1.1M 100% /boot/writable
/dev/sda2 39G 38G 1.1M 100% /home
/dev/sda2 39G 38G 1.1M 100% /opt
/dev/sda2 39G 38G 1.1M 100% /boot/grub2/arm64-efi
/dev/sda2 39G 38G 1.1M 100% /usr/local
/dev/sda2 39G 38G 1.1M 100% /srv
/dev/sda1 20M 4.0M 16M 20% /boot/efi
tmpfs 382M 8.0K 382M 1% /run/user/0
● k3s.service - Lightweight Kubernetes
Loaded: loaded (/etc/systemd/system/k3s.service; enabled; preset: disabled)
Active: activating (start) since Fri 2023-11-03 09:12:14 UTC; 6min ago
Docs: https://k3s.io
Process: 5167 ExecStartPre=/bin/sh -xc ! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service (code=exited, status=0/SUCCESS)
Process: 5169 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS)
Process: 5171 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
Main PID: 5174 (k3s-server)
Tasks: 18
CPU: 7min 6.121s
CGroup: /system.slice/k3s.service
├─5174 "/usr/local/bin/k3s server"
└─5187 "containerd "
Nov 03 09:19:01 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"warn","ts":"2023-11-03T09:19:01.496737Z","caller":"rafthttp/http.go:271","msg":"failed to save incoming database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"error":"write /var/lib/rancher/k3s/server/db/etcd/member/snap/tmp3874139576: no space left on device"}
Nov 03 09:19:01 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"info","ts":"2023-11-03T09:19:01.929443Z","caller":"rafthttp/http.go:257","msg":"receiving database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"incoming-snapshot-message-size-bytes":434067,"incoming-snapshot-message-size":"434 kB"}
Nov 03 09:19:01 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"warn","ts":"2023-11-03T09:19:01.947487Z","caller":"rafthttp/http.go:271","msg":"failed to save incoming database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"error":"write /var/lib/rancher/k3s/server/db/etcd/member/snap/tmp4254258424: no space left on device"}
Nov 03 09:19:02 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"info","ts":"2023-11-03T09:19:02.422559Z","caller":"rafthttp/http.go:257","msg":"receiving database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"incoming-snapshot-message-size-bytes":434067,"incoming-snapshot-message-size":"434 kB"}
Nov 03 09:19:02 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"warn","ts":"2023-11-03T09:19:02.441463Z","caller":"rafthttp/http.go:271","msg":"failed to save incoming database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"error":"write /var/lib/rancher/k3s/server/db/etcd/member/snap/tmp1254702228: no space left on device"}
Nov 03 09:19:02 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"info","ts":"2023-11-03T09:19:02.960905Z","caller":"rafthttp/http.go:257","msg":"receiving database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"incoming-snapshot-message-size-bytes":434067,"incoming-snapshot-message-size":"434 kB"}
Nov 03 09:19:02 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"warn","ts":"2023-11-03T09:19:02.984177Z","caller":"rafthttp/http.go:271","msg":"failed to save incoming database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"error":"write /var/lib/rancher/k3s/server/db/etcd/member/snap/tmp48750555: no space left on device"}
Nov 03 09:19:03 k3s-control-plane-nbg1-tmc k3s[5174]: time="2023-11-03T09:19:02Z" level=info msg="Waiting to retrieve kube-proxy configuration; server is not ready: https://127.0.0.1:6443/v1-k3s/readyz: 500 Internal Server Error"
Nov 03 09:19:03 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"info","ts":"2023-11-03T09:19:03.428766Z","caller":"rafthttp/http.go:257","msg":"receiving database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"incoming-snapshot-message-size-bytes":434067,"incoming-snapshot-message-size":"434 kB"}
Nov 03 09:19:03 k3s-control-plane-nbg1-tmc k3s[5174]: {"level":"warn","ts":"2023-11-03T09:19:03.449117Z","caller":"rafthttp/http.go:271","msg":"failed to save incoming database snapshot","local-member-id":"66d8021377906a72","remote-snapshot-sender-id":"6862d5ef6fbce322","incoming-snapshot-index":105409,"error":"write /var/lib/rancher/k3s/server/db/etcd/member/snap/tmp3303052739: no space left on device"}
