Last active
March 7, 2018 05:25
-
-
Save vsoch/63a0f3821cfeb8437f11f37d169d2927 to your computer and use it in GitHub Desktop.
A working, complete python flow to get the various manifests for the gcr.io DeepVariant image
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"config":{ | |
"AttachStdin":false, | |
"Cmd":[ | |
"/bin/bash" | |
], | |
"Volumes":null, | |
"Image":"sha256:9d30e8ce67321c61be656426c99dc22c6a8be8f1f03a3adcf8e0dd7d08281ed3", | |
"Tty":false, | |
"StdinOnce":false, | |
"Domainname":"", | |
"AttachStdout":false, | |
"OnBuild":[ | |
], | |
"OpenStdin":false, | |
"Labels":null, | |
"ArgsEscaped":true, | |
"Entrypoint":null, | |
"Hostname":"", | |
"AttachStderr":false, | |
"Env":[ | |
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", | |
"DV_GPU_BUILD=0" | |
], | |
"User":"", | |
"WorkingDir":"" | |
}, | |
"architecture":"amd64", | |
"container":"c0ddcb49aa663b7042497275d25c39221db72cffea017c21d45942feadaf052e", | |
"created":"2018-01-29T20:37:52.869007338Z", | |
"docker_version":"17.06.1-ce", | |
"history":[ | |
{ | |
"created":"2018-01-25T18:23:30.610528428Z", | |
"created_by":"/bin/sh -c #(nop) ADD file:a3344b835ea6fdc5692df23826c970403656c6947342e117b2ac1a05956679af in / " | |
}, | |
{ | |
"created":"2018-01-25T18:23:39.075632267Z", | |
"created_by":"/bin/sh -c set -xe \t\t&& echo '#!/bin/sh' > /usr/sbin/policy-rc.d \t&& echo 'exit 101' >> /usr/sbin/policy-rc.d \t&& chmod +x /usr/sbin/policy-rc.d \t\t&& dpkg-divert --local --rename --add /sbin/initctl \t&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \t&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \t\t&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \t\t&& echo 'DPkg::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' > /etc/apt/apt.conf.d/docker-clean \t&& echo 'APT::Update::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' >> /etc/apt/apt.conf.d/docker-clean \t&& echo 'Dir::Cache::pkgcache \"\"; Dir::Cache::srcpkgcache \"\";' >> /etc/apt/apt.conf.d/docker-clean \t\t&& echo 'Acquire::Languages \"none\";' > /etc/apt/apt.conf.d/docker-no-languages \t\t&& echo 'Acquire::GzipIndexes \"true\"; Acquire::CompressionTypes::Order:: \"gz\";' > /etc/apt/apt.conf.d/docker-gzip-indexes \t\t&& echo 'Apt::AutoRemove::SuggestsImportant \"false\";' > /etc/apt/apt.conf.d/docker-autoremove-suggests" | |
}, | |
{ | |
"created":"2018-01-25T18:23:39.843140353Z", | |
"created_by":"/bin/sh -c rm -rf /var/lib/apt/lists/*" | |
}, | |
{ | |
"created":"2018-01-25T18:23:50.734158349Z", | |
"created_by":"/bin/sh -c sed -i 's/^#\\s*\\(deb.*universe\\)$/\\1/g' /etc/apt/sources.list" | |
}, | |
{ | |
"created":"2018-01-25T18:23:51.48088772Z", | |
"created_by":"/bin/sh -c mkdir -p /run/systemd && echo 'docker' > /run/systemd/container" | |
}, | |
{ | |
"created":"2018-01-25T18:23:51.755470218Z", | |
"created_by":"/bin/sh -c #(nop) CMD [\"/bin/bash\"]", | |
"empty_layer":true | |
}, | |
{ | |
"created":"2018-01-29T20:35:11.157197346Z", | |
"created_by":"/bin/sh -c #(nop) ARG DV_GPU_BUILD=0", | |
"empty_layer":true | |
}, | |
{ | |
"created":"2018-01-29T20:35:11.250378819Z", | |
"created_by":"/bin/sh -c #(nop) ENV DV_GPU_BUILD=0", | |
"empty_layer":true | |
}, | |
{ | |
"created":"2018-01-29T20:35:11.441069427Z", | |
"created_by":"/bin/sh -c #(nop) ADD multi:d2d432f1bc663fa828de7f8dca6a5a020f216fd2573801fa1a70a115de62c14f in / " | |
}, | |
{ | |
"created":"2018-01-29T20:35:11.861361117Z", | |
"created_by":"/bin/sh -c mkdir -p /tmp/deepvariant_docker && mkdir -p /opt/deepvariant/bin" | |
}, | |
{ | |
"created":"2018-01-29T20:35:12.010205356Z", | |
"created_by":"/bin/sh -c #(nop) ADD multi:9fc5b56c2afa8867a337221d5da986135c7a996a7616c14b4dcc10b53bccaf12 in /tmp/deepvariant_docker/ " | |
}, | |
{ | |
"created":"2018-01-29T20:37:51.450960553Z", | |
"created_by":"/bin/sh -c cd /tmp/deepvariant_docker && ./run-prereq.sh && rm -rf /tmp/deepvariant_docker/ && apt-get install -y parallel" | |
}, | |
{ | |
"created":"2018-01-29T20:37:52.426548845Z", | |
"created_by":"/bin/sh -c #(nop) ADD multi:f613ad7fb56a9bd4e7bc1d9de6395472cdea26f9b2c49e423a3c4e2fa27bd39c in /opt/deepvariant/bin/ " | |
}, | |
{ | |
"created":"2018-01-29T20:37:52.869007338Z", | |
"created_by":"/bin/sh -c BASH_HEADER='#!/bin/bash' && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/make_examples.zip \"$@\"' > /opt/deepvariant/bin/make_examples && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/call_variants.zip \"$@\"' > /opt/deepvariant/bin/call_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/postprocess_variants.zip \"$@\"' > /opt/deepvariant/bin/postprocess_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_train.zip \"$@\"' > /opt/deepvariant/bin/model_train && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_eval.zip \"$@\"' > /opt/deepvariant/bin/model_eval && chmod +x /opt/deepvariant/bin/make_examples /opt/deepvariant/bin/call_variants /opt/deepvariant/bin/postprocess_variants /opt/deepvariant/bin/model_train /opt/deepvariant/bin/model_eval" | |
} | |
], | |
"rootfs":{ | |
"diff_ids":[ | |
"sha256:ff986b10a018b48074e6d3a68b39aad8ccc002cdad912d4148c0f92b3729323e", | |
"sha256:9c7183e0ea88b265d83708dfe5b9189c4e12f9a1d8c3e5bce7f286417653f9b7", | |
"sha256:c98ef191df4b42c3fd5155d23385e75ee59707c6a448dfc6c8e4e9c005a3df11", | |
"sha256:92914665e7f61f8f19b56bf7983a2b3758cb617bef498b37adb80899e8b86e32", | |
"sha256:6f4ce6b888495c7c9bd4a0ac124b039d986a3b18250fa873d11d13b42f6a79f4", | |
"sha256:d981835cfeee9f8e1b8e68fde5e29ca342e554f41065429ff993c77c99262121", | |
"sha256:959f030108b71cedde109ac857d1bb4d760c1e18caa3cdbe89c714e968e759bf", | |
"sha256:9edfd6bd057d9c5da040a314efd9a7c257ef8158629c5990ea74fc3b745ac051", | |
"sha256:051edfa0dcb187fa742d00482e05f0c7e32e1b6665fa492a031054abb88e655e", | |
"sha256:51c8d662a1b2d33f15b969bd7bdd70e5a12130ca7a7793d63c944ed31dd5e169", | |
"sha256:d3bc210757dacfbe3b2bf90a1938d68df319841d9eef59fa44da3bb64f1bbb26" | |
], | |
"type":"layers" | |
}, | |
"container_config":{ | |
"AttachStdin":false, | |
"Cmd":[ | |
"/bin/sh", | |
"-c", | |
"BASH_HEADER='#!/bin/bash' && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/make_examples.zip \"$@\"' > /opt/deepvariant/bin/make_examples && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/call_variants.zip \"$@\"' > /opt/deepvariant/bin/call_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/postprocess_variants.zip \"$@\"' > /opt/deepvariant/bin/postprocess_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_train.zip \"$@\"' > /opt/deepvariant/bin/model_train && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_eval.zip \"$@\"' > /opt/deepvariant/bin/model_eval && chmod +x /opt/deepvariant/bin/make_examples /opt/deepvariant/bin/call_variants /opt/deepvariant/bin/postprocess_variants /opt/deepvariant/bin/model_train /opt/deepvariant/bin/model_eval" | |
], | |
"Volumes":null, | |
"Image":"sha256:9d30e8ce67321c61be656426c99dc22c6a8be8f1f03a3adcf8e0dd7d08281ed3", | |
"Tty":false, | |
"StdinOnce":false, | |
"Domainname":"", | |
"AttachStdout":false, | |
"OnBuild":[ | |
], | |
"OpenStdin":false, | |
"Labels":null, | |
"ArgsEscaped":true, | |
"Entrypoint":null, | |
"Hostname":"", | |
"AttachStderr":false, | |
"Env":[ | |
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", | |
"DV_GPU_BUILD=0" | |
], | |
"User":"", | |
"WorkingDir":"" | |
}, | |
"os":"linux" | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# library for http requests | |
import requests | |
# URL: This is the url for the manifest | |
url = 'https://gcr.io/v2/deepvariant-docker/deepvariant/manifests/0.5.0' | |
# The urls themselves are pretty useless, it's all driven by the accept headers we | |
# provide. This is the next thing that we set: | |
# ACCEPT HEADER: This means a version 2.0 manifest, this has layers and sizes but not metadata like environment, entrypoint | |
# We used to always get from version 1 manifest, and now some registries have a config, but gcr.io just has version 2.0 | |
headers = {'Accept': 'application/vnd.docker.distribution.manifest.v2+json'} | |
# Make the call, view the status code, and the manifest | |
response = requests.get(url, headers=headers) | |
# This should be 200 | |
response.status_code | |
# Here is the manifest that the layers come from. So this is fixed, but we need the config | |
manifest = response.json() | |
print(manifest) | |
digest = manifest['config']['digest'] | |
mediaType = manifest['config']['mediaType'] | |
headers = {'Accept': mediaType} | |
blob_url = "https://gcr.io/v2/deepvariant-docker/deepvariant/blobs/%s" %digest | |
config = requests.get(blob_url, headers=headers) | |
print(config.json()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Note this is with current branch fix/gcr.io of singularityhub/sregistry-cli | |
# will be released as pypi 0.0.73 soon :) | |
$ sregistry pull docker://gcr.io/deepvariant-docker/deepvariant:0.5.0 | |
[client|docker] [database|sqlite:////home/vanessa/.singularity/sregistry.db] | |
Exploding /usr/local/libexec/singularity/bootstrap-scripts/environment.tar | |
Exploding /home/vanessa/.singularity/docker/sha256:1be7f2b886e89a58e59c4e685fcc5905a26ddef3201f290b96f1eff7d778e122.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:6fbc4a21b806838b63b774b338c6ad19d696a9e655f50b4e358cc4006c3baa79.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:c71a6f8e13782fed125f2247931c3eb20cc0e6428a5d79edb546f1f1405f0e49.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:4be3072e5a37392e32f632bb234c0b461ff5675ab7e362afad6359fbd36884af.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:06c6d2f5970057aef3aef6da60f0fde280db1c077f0cd88ca33ec1a70a9c7b58.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:c4040a2294230c510b5a00dbc78b22075337b3a9574871f7626a8569c81b3cb4.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:5c2bd73e48dc78f72ebffdf3a8cf3ce71b4f268598ea0943b16de5d4c0e877ae.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:8d8439570e8f78644e334a738bfd1ba926fe3f96c6afd2715a3640d2d45bbf4d.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:5a1f6d79bff2cad8269d67407c273b2c6e3367281605f39b2094182c2a1b5e2b.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:7f3fe796e447ca9fab4debea6894350348a7f62d0041ff99e79e3fc2a6730b8a.tar.gz | |
Exploding /home/vanessa/.singularity/docker/sha256:045deebf3a2e52b364829d82a64d23897cdc6d7ce5232dfb5f785fde3d2f162f.tar.gz | |
Exploding /home/vanessa/.singularity/docker/metadata/sha256:deac2c14535690c8d8cca1e1ea987366d65e6a5e2a30bb4108caf2250f728faf.tar.gz | |
Building into existing container: /home/vanessa/.singularity/shub/deepvariant-docker-deepvariant:0.5.0.simg | |
WARNING: Building container as an unprivileged user. If you run this container as root | |
WARNING: it may be missing some functionality. | |
Building FS image from sandbox: /tmp/tmp2zlyvp3n | |
Building Singularity FS image... | |
Building Singularity SIF container image... | |
Singularity container built: /home/vanessa/.singularity/shub/deepvariant-docker-deepvariant:0.5.0.simg | |
Cleaning up... | |
[container][new] deepvariant-docker/deepvariant:0.5.0 | |
Success! /home/vanessa/.singularity/shub/deepvariant-docker-deepvariant:0.5.0.simg | |
singularity shell $(sregistry get deepvariant-docker/deepvariant:0.5.0) | |
Singularity: Invoking an interactive shell within container... | |
Singularity deepvariant-docker-deepvariant:0.5.0.simg:~/Documents/Dropbox/Code/sregistry/sregistry-cli> cat /.singularity.d/env/10-docker.sh | |
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" | |
export DV_GPU_BUILD="0" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment