Skip to content

Instantly share code, notes, and snippets.

@tianon
Last active May 11, 2019 14:40
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save tianon/e22655f5753abfdaa2fc3bff6a9669e7 to your computer and use it in GitHub Desktop.
Save tianon/e22655f5753abfdaa2fc3bff6a9669e7 to your computer and use it in GitHub Desktop.
The original Bash-based prototype for https://github.com/docker-library/official-images/pull/5897 to prove out the concept \m/
#!/usr/bin/env bash
set -Eeuo pipefail
# for real pushes, this would be "library"
targetOrg='trollin'
# https://github.com/tianon/dockerhub-public-proxy
publicProxy="$DOCKERHUB_PUBLIC_PROXY"
_curl() {
curl -fsSL --retry 3 "$@"
}
split_img() {
local img="$1"; shift
local repoVar="$1"; shift
local tagVar="$1"; shift
local repo="${img%%:*}"
local tag="${img#$repo:}"
[ "$tag" != "$repo" ] || tag='latest'
local printfEval
printfEval="$(printf "declare -g $repoVar=%q $tagVar=%q" "$repo" "$tag")"
eval "$printfEval"
}
arch_to_ns() {
local arch="$1"; shift
case "$arch" in
windows-*) echo -n "win${arch#windows-}" ;;
*) echo -n "$arch" ;;
esac
}
fetch_digest() {
local repo="$1"; shift
local man="$1"; shift
local curl
curl="$(_curl --head "$publicProxy/v2/$repo/manifests/$man")" || return 1
curl="$(tr -d '\r' <<<"$curl")" || return 1
local digest
digest="$(sed --quiet --regexp-extended -e '/^docker-content-digest:[[:space:]]+/ s/// p' <<<"${curl,,}")" || return 1
[ -n "$digest" ] || return 1
[[ "$digest" == sha256:* ]] || return 1
echo -n "$digest"
}
fetch_manifest() {
local repo="$1"; shift
local digest="$1"; shift
[[ "$digest" == sha256:* ]] || return 1
_curl "$publicProxy/v2/$repo/manifests/$digest" || return 1
}
fetch_blob() {
local repo="$1"; shift
local digest="$1"; shift
[[ "$digest" == sha256:* ]] || return 1
_curl "$publicProxy/v2/$repo/blobs/$digest" || return 1
}
arch_to_platform() {
local arch="$1"; shift
local goos="${arch%-*}"
[ "$goos" != "$arch" ] || goos='linux'
local goarch="${arch#$goos-}" variant=
case "$goarch" in
i386) goarch='386' ;;
arm32v*) variant="${goarch#arm32}"; goarch='arm' ;;
arm64v*) variant="${goarch#arm64}"; goarch='arm64' ;;
esac
jq -n --arg os "$goos" --arg arch "$goarch" --arg variant "$variant" '{
"architecture": $arch,
"os": $os
} + if ($variant | length) > 0 then { "variant": $variant } else null end'
}
manifest_list_items() {
local arch="$1"; shift
local repo="$1"; shift
local digest="$1"; shift
local manifest schemaVersion
manifest="$(fetch_manifest "$repo" "$digest")" || return 1
schemaVersion="$(jq -r '.schemaVersion' <<<"$manifest")" || return 1
local platform
platform="$(arch_to_platform "$arch")" || return 1
local mediaType
case "$schemaVersion" in
2)
mediaType="$(jq -r '.mediaType' <<<"$manifest")" || return 1
if [ "$mediaType" = 'application/vnd.docker.distribution.manifest.list.v2+json' ]; then
# if we're already a manifest list, we've scored -- just return the list items directly!
jq '.manifests' <<<"$manifest" || return 1
return 0
fi
if [ "$mediaType" != 'application/vnd.docker.distribution.manifest.v2+json' ]; then
echo >&2 "error: unknown schemaVersion 2 'mediaType' value: '$mediaType' (while parsing '$repo@$digest' for '$arch')"
exit 1
fi
local os; os="$(jq -r '.os' <<<"$platform")"
if [ "$os" = 'windows' ]; then
# if this is a Windows image, we need to fetch "os.version" from the image config object for the "platform" output
local config configDigest
configDigest="$(jq -r '.config.digest' <<<"$manifest")"
config="$(fetch_blob "$repo" "$configDigest")"
local osVersion
osVersion="$(jq -r '."os.version"' <<<"$config")"
if [ -n "$osVersion" ]; then
platform="$(jq -n --argjson 'platform' "$platform" --arg 'osVersion' "$osVersion" '$platform + {
"os.version": $osVersion
}')"
fi
fi
;;
1)
mediaType='application/vnd.docker.distribution.manifest.v1+json'
# TODO ??? (especially "os.version" for a Windows-based image?)
;;
*)
echo >&2 "error: unknown 'schemaVersion' value: '$schemaVersion' (while parsing '$repo@$digest' for '$arch')"
exit 1
;;
esac
jq -n --arg 'mediaType' "$mediaType" --argjson 'size' "${#manifest}" --arg 'digest' "$digest" --argjson 'platform' "$platform" '[
{
"mediaType": $mediaType,
"size": $size,
"digest": $digest,
"platform": $platform
}
]'
}
for img; do
split_img "$img" 'repo' 'tag'
img="$repo:$tag"
echo
echo "$targetOrg/$img"
arches="$(bashbrew cat --format '{{ range .Entries }}{{ .Architectures | join "\n" }}{{ "\n" }}{{ end }}' "$img")"
arches="$(sort -u <<<"$arches")"
arches="$(xargs <<<"$arches")"
manifests='[]'
for arch in $arches; do
ns="$(arch_to_ns "$arch")"
digest="$(fetch_digest "$ns/$repo" "$tag")" || continue # if the tag doesn't exist in this repo, skip it
echo " - $arch: $digest"
manifestListItems="$(manifest_list_items "$arch" "$ns/$repo" "$digest")"
manifests="$(jq -n --argjson 'manifests' "$manifests" --argjson new "$manifestListItems" '$manifests + $new')"
done
newManifestList="$(jq -n --argjson 'manifests' "$manifests" '{
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
"manifests": $manifests
}')"
oldManifestList="$(oldDigest="$(fetch_digest "$targetOrg/$repo" "$tag")" && fetch_manifest "$targetOrg/$repo" "$oldDigest")" || :
diff -u <(jq --tab . <<<"$oldManifestList") <(jq --tab . <<<"$newManifestList") || :
# TODO opportunistic blob mounts + manifest list push
#echo "==> $(jq -c . <<<"$newManifestList")"
done
# TODO rewrite this all in Perl using promises so it can parallelize like crazy
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment