Skip to content

Instantly share code, notes, and snippets.

@dpedu
Created June 5, 2016 20:15
Show Gist options
  • Save dpedu/d8edf93301795183ebe144c7fc15e125 to your computer and use it in GitHub Desktop.
Save dpedu/d8edf93301795183ebe144c7fc15e125 to your computer and use it in GitHub Desktop.
docker registry v2 cleanup script
#!/usr/bin/env python3
from requests import get
import sys
from json import dumps
from os import listdir,remove,rmdir
from shutil import rmtree
REGISTRY_URL = "http://127.0.0.1:5000/v2/"
REGISTRY_DATA_PATH = "/data/registry/"
LIB_IMAGES = ["mktp/drupal", "mktp/elastic", "mktp/python", "mktp/symfony", "mktp/varnish"]
KEEP_LAST = 30
def get_json(path):
return get(REGISTRY_URL+path).json()
def list_tags(repo):
return get_json("%s/tags/list" % repo)["tags"]
def get_layers(image, tag):
layers = []
fslayers = get_json("%s/manifests/%s" % (image, tag))["fsLayers"]
for item in fslayers:
layers.append(item["blobSum"])
return layers
def sort_tags(tagslist):
# Sort by build number (rel-2015-1-1-353 -> [rel, 2015, 1, 1, 353], sort by last number). Special case for "base" - base should always be kept so it's moved to the beginning
return sorted(tagslist, key=lambda tag: int(tag.split("-")[-1]) if tag != "base" else 99999)
def add_if_unique(list, item):
if not item in list:
list.append(item)
if __name__ == "__main__":
all_layers_unique = []
tags_kept = {}
# Get list of all layers belonging to images we want to keep
for name in LIB_IMAGES:
unique_layers = []
tags_kept[name]=[]
# Get list of tags for current image
# Get recent 15 images + base
image_tags = sort_tags(list_tags(name))
image_tags.reverse()
image_tags = image_tags[0:KEEP_LAST]
# For each tag, record unique layers
for tag in image_tags:
tags_kept[name].append(tag)
layers = get_layers(name, tag)
for layer in layers:
layer = layer[7:] # Strip sha256: heading from value
add_if_unique(unique_layers, layer)
add_if_unique(all_layers_unique, layer)
# Get tag revison hash
with open(REGISTRY_DATA_PATH+'docker/registry/v2/repositories/'+name+'/_manifests/tags/'+tag+'/current/link', 'r') as link:
revHash = link.read()[7:] # Strip sha256: heading from value
add_if_unique(unique_layers, revHash)
add_if_unique(all_layers_unique, revHash)
# Get revision signature
signatures = listdir(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/signatures/sha256/" % (name, revHash))
for s in signatures:
add_if_unique(unique_layers, s)
add_if_unique(all_layers_unique, s)
#print("%s - %s unique layers" % (name, len(unique_layers)))
unique_layers.sort()
#print(unique_layers)
#print("ALL - %s unique layers" % len(all_layers_unique))
# List of unique layers from tags we want to keep has been generated
# Remove all blobs that are not in this list
all_blobs = []
for subdir in listdir(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/"):
for blob in listdir(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/"+subdir):
all_blobs.append(blob)
#print("All blobs: %s" % all_blobs)
orphan_blobs = []
for blob in all_blobs:
if not blob in all_layers_unique:
orphan_blobs.append(blob)
print("Orphaned blobs (%s): %s" % (len(orphan_blobs), orphan_blobs))
for blob in orphan_blobs:
print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s/data" % (blob[0:2], blob))
print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s" % (blob[0:2], blob))
remove(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s/data" % (blob[0:2], blob))
rmdir(REGISTRY_DATA_PATH+"docker/registry/v2/blobs/sha256/%s/%s" % (blob[0:2], blob))
pass
# Remove tags we didn't whitelist
for name in LIB_IMAGES:
print("Tags we're keeping: %s" % tags_kept[name])
tags_to_delete = list(set(list_tags(name)) - set(tags_kept[name]))
print("Tags we're deleting: %s" % tags_to_delete)
# List tags for image
for tag in tags_to_delete:
with open(REGISTRY_DATA_PATH+'docker/registry/v2/repositories/'+name+'/_manifests/tags/'+tag+'/current/link', 'r') as link:
revHash = link.read()[7:] # Strip sha256: heading from value
print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s" % (name, tag))
rmtree(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s/current" % (name, tag))
rmtree(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s/index" % (name, tag))
rmdir(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/tags/%s" % (name, tag))
# Remove signatures for this tag
print("delete %s" % REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s" % (name, revHash))
remove(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/link" % (name, revHash))
rmtree(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s/signatures" % (name, revHash))
rmdir(REGISTRY_DATA_PATH+"docker/registry/v2/repositories/%s/_manifests/revisions/sha256/%s" % (name, revHash))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment