Skip to content

Instantly share code, notes, and snippets.

@virtuman
Created January 29, 2018 22:28
Show Gist options
  • Save virtuman/b371fdc4b4f520092ee269110c5c04f3 to your computer and use it in GitHub Desktop.
Save virtuman/b371fdc4b4f520092ee269110c5c04f3 to your computer and use it in GitHub Desktop.
gitlab docker registry cleanup through API + screen scraping
<?php
# Accepts command line arguments:
# -url=https://gitlab.example.com/
# -token=GITLAB_TOKEN
# -debug=false | true
# -dryrun=false | true
# -keep=2 day
$args = arguments($argv);
$BASE_URL = $args['url'];
$GITLAB_CI_TOKEN = $args['token'];
$DEBUG = $args['debug'] ?? false;
$DRY_RUN = $args['dryrun'] ?? false;
$AGE_OF_CONTAINERS_TO_KEEP = $args['keep'] ?? '2 day'; // using php strtotime function
$TAGS_TO_WHITELIST = ['dev', 'devel', 'develop', 'master', 'latest'];
$header = [];
function arguments ( $args )
{
array_shift( $args );
$args = join( $args, ' ' );
preg_match_all('/ (--\w+ (?:[= ] [^-]+ [^\s-] )? ) | (-\w+) | (\w+) /x', $args, $match );
$args = array_shift( $match );
foreach ( $args as $arg ) {
$value = preg_split( '/[= ]/', $arg, 2 );
$com = substr( array_shift($value), 2 );
$value = join($value);
$ret[$com] = !empty($value) ? $value : true;
}
return $ret;
}
// Setup header that is needed to do delete tag calls.
function setupProperHeaders()
{
global $GITLAB_CI_TOKEN, $header, $FETCH_HEADER_URL;
$ch = curl_init();
$options = [
CURLOPT_URL => $FETCH_HEADER_URL,
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_HEADER => 1,
CURLOPT_HTTPHEADER => ['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]
];
curl_setopt_array($ch, $options);
$response = curl_exec($ch);
$header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
$header_text = substr($response, 0, $header_size);
$body_text = substr($response, $header_size);
curl_close($ch);
$headers = [];
foreach (explode("\r\n", $header_text) as $i => $line) {
if ($i === 0) {
$headers['http_code'] = $line;
} else {
$data = explode(': ', $line);
if (count($data) == 2) {
list ($key, $value) = $data;
$headers[$key] = $value;
}
}
};
preg_match_all('/<meta *.*csrf-token*.* content="(.*|\n*?)" * \/>/'
, $body_text
, $regexOut
, PREG_PATTERN_ORDER
, 0);
$X_CSRF_TOKEN = $regexOut[1][0];
$COOKIE = $headers['Set-Cookie'];
$header = [
'X-CSRF-Token: ' . $X_CSRF_TOKEN,
'Accept: application/json, text/plain, */*',
'Cookie: ' . $COOKIE,
'X-Requested-With: XMLHttpRequest',
'PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN
];
}
// Wrapper for curl requests
function curlWrapper($url, $header = [], $asDeleteRequest = false, $printOnly = false)
{
global $DEBUG;
$ch = curl_init();
$options = [
CURLOPT_URL => $url,
CURLOPT_USERAGENT => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)',
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_CONNECTTIMEOUT => 15,
CURLOPT_TIMEOUT => 5,
];
curl_setopt_array($ch, $options);
if ($asDeleteRequest) {
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "DELETE");
}
if (!empty($header)) {
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
}
if (!empty($username)) {
curl_setopt($ch, CURLOPT_USERPWD, $username . ':' . $password);
}
if (!$printOnly) {
$data = curl_exec($ch);
$httpcode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (empty($data) && $DEBUG) {
$error = curl_error($ch);
$errorno = curl_errno($ch);
echo PHP_EOL . 'ERROR - ' . $error . ' --- ' . $errorno;
}
curl_close($ch);
} else {
echo PHP_EOL . $url . PHP_EOL;
$httpcode = 200;
$data = '';
}
if ($httpcode >= 200 && $httpcode < 300) {
return $data;
}
return false;
}
// taken from http://jeffreysambells.com/2012/10/25/human-readable-filesize-php
function human_filesize($bytes, $decimals = 2)
{
$size = array('B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB');
$factor = floor((strlen($bytes) - 1) / 3);
return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$size[$factor];
}
// --- Script startes here --- //
echo PHP_EOL . "Retrieving projects";
$i=0;
while (true) {
$i++;
$retrieved_projects = json_decode(curlWrapper($BASE_URL . 'api/v4/projects?per_page=100&page='. $i,
['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]));
if (empty($retrieved_projects)) {
break;
}
foreach ($retrieved_projects as $project) {
if (!$project->container_registry_enabled) continue;
$projects[] = $project->path_with_namespace;
}
}
if (empty($projects)) {
echo PHP_EOL . "No projects where found where registry is enabled";
exit;
}
$all_tags = [];
# Use first matched project as URL to establish proper headers for curl delete requests
$FETCH_HEADER_URL = $BASE_URL . $projects[0] . '/container_registry';
foreach ($projects as $project) {
$FETCH_REGISTRIES_URL = $BASE_URL . $project . '/container_registry.json';
echo PHP_EOL . "Retrieving registries for - " . $project;
$retrieved_registries = json_decode(curlWrapper($FETCH_REGISTRIES_URL,
['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]));
foreach($retrieved_registries as $registry) {
$i = 0;
echo PHP_EOL . "Retrieving tags";
while (true || $i > 100) {
$i += 1;
$retrieved_tags = json_decode(curlWrapper($BASE_URL . $registry->tags_path . '&page=' . $i,
['PRIVATE-TOKEN: ' . $GITLAB_CI_TOKEN]));
if (empty($retrieved_tags)) {
break;
}
#print_r($retrieved_tags);exit;
echo '.';
$all_tags = array_merge($all_tags, $retrieved_tags);
}
}
}
print_r($all_tags);
if (!empty($all_tags)) {
echo PHP_EOL . "Registry list fetched " . count($all_tags) . ' found';
}
if (count($all_tags) == 0) {
echo " - Nothing found" . PHP_EOL;
return;
}
setupProperHeaders();
$newest_timestamp_to_keep = strtotime('-' . $AGE_OF_CONTAINERS_TO_KEEP);
$deletedCount = 0;
$total_size = 0;
foreach ($all_tags as $tag) {
if (empty($tag->name)) {
continue;
}
$tag_name_from_uri = "";
# if multi-level image names are used - assume that last part of URI is the tag name
# registry.example.com/group/project:some-tag
# registry.example.com/group/project/image:latest
# registry.example.com/group/project/my/image:rc1
# registry.example.com/group/project/my/master:latest
# registry.example.com/group/project/my/master:a234234werwe234
if (substr_count($tag->location, '/') > 2) {
$tag_name_from_uri = (string)array_pop(explode('/', $tag->location));
}
if (in_array($tag->name, $TAGS_TO_WHITELIST) ||
stripos($tag->name, '-stable') !== false ||
in_array($tag_name_from_uri, $TAGS_TO_WHITELIST) ||
stripos($tag_name_from_uri, '-stable') !== false
) {
continue;
}
$tag_date = date_create_from_format('Y-m-d\TH:i:s.u+T', $tag->created_at);
$test = empty($tag->created_at) ? '' : $tag_date->getTimestamp();
if (!empty($tag->created_at) && $newest_timestamp_to_keep < $tag_date->getTimestamp()) {
continue;
}
echo PHP_EOL . "Trying to delete " . $tag->name . ' - created at ' . $tag->created_at;
$call = curlWrapper($BASE_URL . $tag->destroy_path, $header, true, $DRY_RUN);
// returns empty call if sucess, null if failed
if ($call == '') {
echo " - deleted ";
$deletedCount++;
$total_size += $tag->total_size;
}
}
if ($deletedCount == 0) {
echo PHP_EOL . PHP_EOL . "None of the tags were set to be deleted" . PHP_EOL;
} else {
echo PHP_EOL . PHP_EOL . "You deleted: " . $deletedCount . " - Cleaned up: " . human_filesize($total_size) . PHP_EOL;
}
// end for script
?>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment