Skip to content

Instantly share code, notes, and snippets.

@oskarm93
Last active November 3, 2023 09:14
Show Gist options
  • Save oskarm93/891056c540a8b9949af332d760d0a3ae to your computer and use it in GitHub Desktop.
Save oskarm93/891056c540a8b9949af332d760d0a3ae to your computer and use it in GitHub Desktop.
k3s-control-plane-fsn1-xit:~ # du -ah / | sort -n -r | head -n 10
du: cannot access '/proc/4495/task/4495/fd/4': No such file or directory
du: cannot access '/proc/4495/task/4495/fdinfo/4': No such file or directory
du: cannot access '/proc/4495/fd/3': No such file or directory
du: cannot access '/proc/4495/fdinfo/3': No such file or directory
1023M   /usr
1020M   /.snapshots/2/snapshot
1020M   /.snapshots/2
1020K   /usr/share/pki/trust/ca-certificates-mozilla.trust.p11-kit
1020K   /usr/share/pki/trust
1020K   /usr/lib64/python3.11/importlib
1020K   /.snapshots/4/snapshot/usr/share/pki/trust/ca-certificates-mozilla.trust.p11-kit
1020K   /.snapshots/4/snapshot/usr/share/pki/trust
1020K   /.snapshots/4/snapshot/usr/lib64/python3.11/importlib
1020K   /.snapshots/2/snapshot/usr/share/pki/trust/ca-certificates-mozilla.trust.p11-kit
k3s-control-plane-fsn1-xit:~ # df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/sda2        39G   12G   27G  31% /
/dev/sda2        39G   12G   27G  31% /root
/dev/sda2        39G   12G   27G  31% /var
overlay          39G   12G   27G  31% /etc
devtmpfs        4.0M  8.0K  4.0M   1% /dev
tmpfs           1.9G  100K  1.9G   1% /dev/shm
efivarfs        256K   16K  241K   6% /sys/firmware/efi/efivars
tmpfs           764M  2.4M  761M   1% /run
tmpfs           1.9G     0  1.9G   0% /tmp
/dev/sda1        20M   11M  9.8M  52% /boot/efi
/dev/sda2        39G   12G   27G  31% /.snapshots
/dev/sda2        39G   12G   27G  31% /home
/dev/sda2        39G   12G   27G  31% /boot/grub2/arm64-efi
/dev/sda2        39G   12G   27G  31% /opt
/dev/sda2        39G   12G   27G  31% /boot/writable
/dev/sda2        39G   12G   27G  31% /usr/local
/dev/sda2        39G   12G   27G  31% /srv
tmpfs           400M   12K  400M   1% /var/lib/kubelet/pods/f78ba07c-6d88-4f54-8adf-2b30fe27632f/volumes/kubernetes.io~projected/kube-api-access-fqc97
tmpfs           170M   12K  170M   1% /var/lib/kubelet/pods/090da089-db0a-4dc8-add4-f4ffab74c64b/volumes/kubernetes.io~projected/kube-api-access-xlpq5
tmpfs           3.8G   12K  3.8G   1% /var/lib/kubelet/pods/03a5d724-3795-4b3d-8905-c4bc18085a1e/volumes/kubernetes.io~projected/kube-api-access-2s26l
tmpfs           3.8G   12K  3.8G   1% /var/lib/kubelet/pods/d00a9317-adf3-4dcf-ad13-3ce685437201/volumes/kubernetes.io~projected/kube-api-access-vnm9p
tmpfs           3.8G   12K  3.8G   1% /var/lib/kubelet/pods/895f848d-0294-41a3-bfae-7b153671f199/volumes/kubernetes.io~projected/kube-api-access-mf4wx
tmpfs           3.8G   12K  3.8G   1% /var/lib/kubelet/pods/7df5a253-8e17-4db4-90b0-e4ee9f8f7bd7/volumes/kubernetes.io~projected/kube-api-access-9p52v
tmpfs           3.8G   12K  3.8G   1% /var/lib/kubelet/pods/9fd66fd6-5a70-4abc-9bb9-b17c8567ec57/volumes/kubernetes.io~projected/kube-api-access-jkvpg
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/5dca03d2e360f2aa271afb473a71042f814f75329ae1eddfd150aa0d5577010d/shm
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/5dca03d2e360f2aa271afb473a71042f814f75329ae1eddfd150aa0d5577010d/rootfs
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/634bce2bd2a0527a3385079d7325ae21f4787a401d495b55634a28dc491b6970/shm
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/634bce2bd2a0527a3385079d7325ae21f4787a401d495b55634a28dc491b6970/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/1bad0c5a1d3088bb71d1f6c733b820cb55147c43102df97fd9a28ca6d28b5ef8/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/cb8d7d75bbf01c936796fdb03c6dc8c369d09b5345cda867e72f3db5424dfad3/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/8c7f8c4160ab74011ccfd28383c55256f6ae32cbb41da18c21c3724157fa95a8/rootfs
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/af63f4757552fde6cee6d08f2d1a2762353a2f0fd595c5d905dd4c1ba8338b57/shm
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/74a542174cde1030f5540b8428d9f9778973da0433e22c7854ae6e472f2f8d9b/shm
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/11ce850282358cac8f6fd5a8be82907227aabc608ebc6341dd9d1a712fe1f731/shm
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/c880942b93826eab4a76c49f9409130cfa37b9b29be50df90c12bbe07db56450/shm
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/af63f4757552fde6cee6d08f2d1a2762353a2f0fd595c5d905dd4c1ba8338b57/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/74a542174cde1030f5540b8428d9f9778973da0433e22c7854ae6e472f2f8d9b/rootfs
shm              64M     0   64M   0% /run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/648044eb84c4fc870e129993df804a5df179dd6170ceb34dc4b600e2cedc05b2/shm
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/23dc6e06963f8cfeb96afd43232308225516828dccada4303502c39dc41e9012/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/11ce850282358cac8f6fd5a8be82907227aabc608ebc6341dd9d1a712fe1f731/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/c880942b93826eab4a76c49f9409130cfa37b9b29be50df90c12bbe07db56450/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/648044eb84c4fc870e129993df804a5df179dd6170ceb34dc4b600e2cedc05b2/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/d2a4aad7a860741a8ae9845b11b0b666fbde561f38e84ddc82c22af2b2a2c182/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/dff1e499dd7a08eb68226666c24edecd26270027894e0083b11a54e9a9dd1d20/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/eb2050ae20545980775796088154b4c0fd18c3a0d373a76d0c3b10e8c0968768/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/b73017e493d51237717d47fc2781cadaeb17d162ebeb4e1ea94f1c7eaa7723e4/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/5f13dd17c29d78902134c7beb61ff2ba0ab3de32b7eb3378d23a763fd814acd2/rootfs
overlay          39G   12G   27G  31% /run/k3s/containerd/io.containerd.runtime.v2.task/k8s.io/74489baae612fa1290fde48a925c4ca0307f4508ba5d05e7a5604d9b8c2b6994/rootfs
tmpfs           382M  8.0K  382M   1% /run/user/0
k3s-control-plane-fsn1-xit:~ # systemctl status k3s.service
● k3s.service - Lightweight Kubernetes
     Loaded: loaded (/etc/systemd/system/k3s.service; enabled; preset: disabled)
     Active: activating (start) since Fri 2023-11-03 09:05:31 UTC; 52s ago
       Docs: https://k3s.io
    Process: 4329 ExecStartPre=/bin/sh -xc ! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service (code=exited, status=0/SUCCESS)
    Process: 4331 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS)
    Process: 4333 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
   Main PID: 4335 (k3s-server)
      Tasks: 111
        CPU: 44.819s
     CGroup: /system.slice/k3s.service
             ├─3004 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5dca03d2e360f2aa271afb473a71042f814f75329ae1eddfd150aa0d5577010d -address /run/k3s/containerd/containerd.sock
             ├─3163 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id 634bce2bd2a0527a3385079d7325ae21f4787a401d495b55634a28dc491b6970 -address /run/k3s/containerd/containerd.sock
             ├─4335 "/usr/local/bin/k3s server"
             ├─4350 "containerd "
             ├─4683 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id af63f4757552fde6cee6d08f2d1a2762353a2f0fd595c5d905dd4c1ba8338b57 -address /run/k3s/containerd/containerd.sock
             ├─4715 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id 74a542174cde1030f5540b8428d9f9778973da0433e22c7854ae6e472f2f8d9b -address /run/k3s/containerd/containerd.sock
             ├─4914 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id 11ce850282358cac8f6fd5a8be82907227aabc608ebc6341dd9d1a712fe1f731 -address /run/k3s/containerd/containerd.sock
             ├─4939 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id c880942b93826eab4a76c49f9409130cfa37b9b29be50df90c12bbe07db56450 -address /run/k3s/containerd/containerd.sock
             └─4984 /var/lib/rancher/k3s/data/b5d8cff4895c7bb703dbf350edf80a728221f852717ad65504f0e7a929c7dd38/bin/containerd-shim-runc-v2 -namespace k8s.io -id 648044eb84c4fc870e129993df804a5df179dd6170ceb34dc4b600e2cedc05b2 -address /run/k3s/containerd/containerd.sock

Nov 03 09:06:22 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"info","ts":"2023-11-03T09:06:22.905027Z","caller":"rafthttp/snapshot_sender.go:84","msg":"sending database snapshot","snapshot-index":105409,"remote-peer-id":"66d8021377906a72","bytes":23457683,"size":"24 MB"}
Nov 03 09:06:22 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"warn","ts":"2023-11-03T09:06:22.952016Z","caller":"etcdserver/snapshot_merge.go:72","msg":"failed to send database snapshot to writer","size":"5.2 MB","error":"io: read/write on closed pipe"}
Nov 03 09:06:22 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"warn","ts":"2023-11-03T09:06:22.95246Z","caller":"rafthttp/snapshot_sender.go:102","msg":"failed to send database snapshot","snapshot-index":105409,"remote-peer-id":"66d8021377906a72","bytes":23457683,"size":"24 MB","error":"unexpected http status Internal Server Error whil>
Nov 03 09:06:22 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"info","ts":"2023-11-03T09:06:22.952655Z","caller":"etcdserver/server.go:2128","msg":"sent merged snapshot","from":"6862d5ef6fbce322","to":"66d8021377906a72","bytes":23457683,"size":"24 MB","took":"48.001353ms"}
Nov 03 09:06:23 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"info","ts":"2023-11-03T09:06:23.400903Z","caller":"etcdserver/server.go:2110","msg":"sending merged snapshot","from":"6862d5ef6fbce322","to":"66d8021377906a72","bytes":23457683,"size":"24 MB"}
Nov 03 09:06:23 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"info","ts":"2023-11-03T09:06:23.401358Z","caller":"rafthttp/snapshot_sender.go:84","msg":"sending database snapshot","snapshot-index":105409,"remote-peer-id":"66d8021377906a72","bytes":23457683,"size":"24 MB"}
Nov 03 09:06:23 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"warn","ts":"2023-11-03T09:06:23.452408Z","caller":"etcdserver/snapshot_merge.go:72","msg":"failed to send database snapshot to writer","size":"6.8 MB","error":"io: read/write on closed pipe"}
Nov 03 09:06:23 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"warn","ts":"2023-11-03T09:06:23.453089Z","caller":"rafthttp/snapshot_sender.go:102","msg":"failed to send database snapshot","snapshot-index":105409,"remote-peer-id":"66d8021377906a72","bytes":23457683,"size":"24 MB","error":"unexpected http status Internal Server Error whi>
Nov 03 09:06:23 k3s-control-plane-fsn1-xit k3s[4335]: {"level":"info","ts":"2023-11-03T09:06:23.453194Z","caller":"etcdserver/server.go:2128","msg":"sent merged snapshot","from":"6862d5ef6fbce322","to":"66d8021377906a72","bytes":23457683,"size":"24 MB","took":"52.294574ms"}
Nov 03 09:06:23 k3s-control-plane-fsn1-xit k3s[4335]: time="2023-11-03T09:06:23Z" level=info msg="Waiting to retrieve kube-proxy configuration; server is not ready: https://127.0.0.1:6443/v1-k3s/readyz: 500 Internal Server Error"
lines 1-31/31 (END)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment