advanced-docker-cleaner.py

advanced-docker-cleaner.py

import re
from glob import glob

import docker
import os

from subprocess import Popen, PIPE

try:
    from config import BALENA_HOST
except ImportError:
    # BALENA_HOST = os.getenv("DOCKER_HOST", "unix:///var/run/balena.sock")
    BALENA_HOST = os.getenv("DOCKER_HOST", "unix:///var/run/docker.sock")

VAR_LIB_DOCKER = "/var/lib/docker/"


# VAR_LIB_DOCKER = "/data/balenadataroot/"

class AdvancedDockerCleaner(object):
    def __init__(self, docker_host=BALENA_HOST):
        self.client = docker.DockerClient(base_url=docker_host, )

    def _get_images_from_folders(self, folder_list):
        out = set()
        full_string = ":".join(folder_list)
        m = re.findall('overlay2/([a-zA-z0-9\-]+)/?', full_string)
        [out.add(i) for i in m]
        return out

    def get_used_graphdriver_folder(self, image):
        return self._get_images_from_folders([v for _, v in image.attrs["GraphDriver"]["Data"].items()])

    def get_used_layerdb(self, image):
        out = set([x.replace("sha256:", "") for x in image.attrs["RootFS"]["Layers"]])
        return out

    def get_mountpoints(self):
        child = Popen("mount",
                      stdout=PIPE,
                      stderr=PIPE, shell=True)

        out, err = child.communicate()
        return out.decode()

    def delete_overlay_layer(self, layer_name):
        BASE_OVERLAY = VAR_LIB_DOCKER + "/overlay2/"
        layer_to_del = BASE_OVERLAY + layer_name
        link_file = layer_to_del + "/link"

        if not os.path.isfile(link_file):
            return

        linked_fn = ""
        with open(link_file) as f:
            link_file = f.read()

        link_file_full_path = BASE_OVERLAY + "l/%s" % link_file
        if os.path.exists(link_file_full_path):
            os.unlink(link_file_full_path)

        print("--> Removing Overlay %s" % layer_to_del)
        child = Popen("rm -rf %s" % (layer_to_del),
                      stdout=PIPE,
                      stderr=PIPE, shell=True)

        out, err = child.communicate()

        if child.returncode != 0:
            print("[ out: %s - err: %s ]" % (out.decode(), err.decode()))
        else:
            print("--> Removed")

    def clean(self):
        used_overlays = set()
        used_layers = set()

        all_images = list(self.client.images.list())
        for i in all_images:
            used_overlays = used_overlays | self.get_used_graphdriver_folder(i)
            used_layers = used_layers | self.get_used_layerdb(i)

        fs_overlays = self._get_images_from_folders([x for x in glob(VAR_LIB_DOCKER + "overlay2/*/")])
        fs_overlays = {x for x in fs_overlays if len(x) > 3}
        print("Overlays -------")
        print("Folder #: %s" % len(fs_overlays))
        print("Used   #: %s" % len(used_overlays))

        overlay_diff = fs_overlays - used_overlays

        fs_layers = {x.split("/")[-1] for x in glob(VAR_LIB_DOCKER + "image/overlay2/layerdb/sha256/*/")}

        print("Layers -------")
        print("Folder #: %s" % len(fs_layers))
        print("Used   #: %s" % len(used_layers))

        layers_diff = fs_overlays - used_overlays

        print("\n".join([x for x in overlay_diff]))

        mounted = self.get_mountpoints()

        for f in overlay_diff:
            self.delete_overlay_layer(f)

        for l in layers_diff:
            self.delete_layer(l)

            #
            # if mounted.find(f):
            #     print("%s is mounted! --- skipping" % f)
            #     continue

    def delete_layer(self, f):
        to_del = VAR_LIB_DOCKER + "image/overlay2/layerdb/sha256/%s" % f
        print("--> Removing LayerDb %s" % to_del)
        child = Popen("rm -rf %s" % (to_del),
                      stdout=PIPE,
                      stderr=PIPE, shell=True)

        out, err = child.communicate()

        if child.returncode != 0:
            print("[ out: %s - err: %s ]" % (out.decode(), err.decode()))
        else:
            print("--> Removed")


AdvancedDockerCleaner().clean()