docker registry v2 cleanup script

cleanreg.py

#!/usr/bin/env python3

from requests import get
import sys
from json import dumps
from os import listdir,remove,rmdir
from shutil import rmtree

REGISTRY_URL = "http://127.0.0.1:5000/v2/"
REGISTRY_DATA_PATH = "/data/registry/"
LIB_IMAGES = ["mktp/drupal", "mktp/elastic", "mktp/python", "mktp/symfony", "mktp/varnish"]
KEEP_LAST = 30

def get_json(path):
    return get(REGISTRY_URL+path).json()

def list_tags(repo):
    return get_json("%s/tags/list" % repo)["tags"]


def get_layers(image, tag):
    layers = []
    fslayers = get_json("%s/manifests/%s" % (image, tag))["fsLayers"]
    for item in fslayers:
        layers.append(item["blobSum"])
    return layers

def sort_tags(tagslist):
    # Sort by build number (rel-2015-1-1-353 -> [rel, 2015, 1, 1, 353], sort by last number). Special case for "base" - base should always be kept so it's moved to the beginning
    return sorted(tagslist, key=lambda tag: int(tag.split("-")[-1]) if tag != "base" else 99999)

def add_if_unique(list, item):
    if not item in list:
        list.append(item)

if __name__ == "__main__":
    all_layers_unique = []
    tags_kept = {}
    
    # Get list of all layers belonging to images we want to keep
    for name in LIB_IMAGES:
        unique_layers = []
        tags_kept[name]=[]
        # Get list of tags for current image
        # Get recent 15 images + base
        image_tags = sort_tags(list_tags(name))
        image_tags.reverse()
        image_tags = image_tags[0:KEEP_LAST]
        
        
        # For each tag, record unique layers
        for tag in image_tags:
            tags_kept[name].append(tag)
            layers = get_layers(name, tag)
            for layer in layers:
                layer = layer[7:] # Strip sha256: heading from value
                add_if_unique(unique_layers,     layer)
                add_if_unique(all_layers_unique, layer)
            # Get tag revison hash
            with open(REGISTRY_DATA_PATH+'docker/registry/v2/repositories/'+name+'/_manifests/tags/'+tag+'/current/link', 'r') as link:
                revHash = link.read()[7:] # Strip sha256: heading from value
            
            add_if_unique(unique_layers,     revHash)
            add_if_unique(all_layers_unique, revHash)
            
            # Get revision signature
            signatures = listdir(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/signatures/sha256/" % (name, revHash))
            for s in signatures:
                add_if_unique(unique_layers,     s)
                add_if_unique(all_layers_unique, s)
            
        #print("%s - %s unique layers" % (name, len(unique_layers)))
        unique_layers.sort()
        #print(unique_layers)
    #print("ALL - %s unique layers" % len(all_layers_unique))
    
    # List of unique layers from tags we want to keep has been generated 
    # Remove all blobs that are not in this list
    all_blobs = []
    for subdir in listdir(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/"):
        for blob in listdir(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/"+subdir):
            all_blobs.append(blob)
    
    #print("All blobs: %s" % all_blobs)
    
    orphan_blobs = []
    
    for blob in all_blobs:
        if not blob in all_layers_unique:
            orphan_blobs.append(blob)
    
    print("Orphaned blobs (%s): %s" % (len(orphan_blobs), orphan_blobs))
    for blob in orphan_blobs:
        print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s/data" % (blob[0:2], blob))
        print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s" % (blob[0:2], blob))
        remove(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s/data" % (blob[0:2], blob))
        rmdir(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s" % (blob[0:2], blob))
        pass
    
    # Remove tags we didn't whitelist
    for name in LIB_IMAGES:
        print("Tags we're keeping: %s" % tags_kept[name])
        tags_to_delete = list(set(list_tags(name)) - set(tags_kept[name]))
        print("Tags we're deleting: %s" % tags_to_delete)
        # List tags for image
        for tag in tags_to_delete:
            with open(REGISTRY_DATA_PATH+'docker/registry/v2/repositories/'+name+'/_manifests/tags/'+tag+'/current/link', 'r') as link:
                revHash = link.read()[7:] # Strip sha256: heading from value
            
            print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s" % (name, tag))
            rmtree(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s/current" % (name, tag))
            rmtree(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s/index" % (name, tag))
            rmdir(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s" % (name, tag))
            
            
            # Remove signatures for this tag
            print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s" % (name, revHash))
            remove(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/link" % (name, revHash))
            rmtree(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/signatures" % (name, revHash))
            rmdir(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s" % (name, revHash))