Skip to content

Instantly share code, notes, and snippets.

@vsoch
Last active March 7, 2018 05:25
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save vsoch/63a0f3821cfeb8437f11f37d169d2927 to your computer and use it in GitHub Desktop.
Save vsoch/63a0f3821cfeb8437f11f37d169d2927 to your computer and use it in GitHub Desktop.
A working, complete python flow to get the various manifests for the gcr.io DeepVariant image
{
"config":{
"AttachStdin":false,
"Cmd":[
"/bin/bash"
],
"Volumes":null,
"Image":"sha256:9d30e8ce67321c61be656426c99dc22c6a8be8f1f03a3adcf8e0dd7d08281ed3",
"Tty":false,
"StdinOnce":false,
"Domainname":"",
"AttachStdout":false,
"OnBuild":[
],
"OpenStdin":false,
"Labels":null,
"ArgsEscaped":true,
"Entrypoint":null,
"Hostname":"",
"AttachStderr":false,
"Env":[
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"DV_GPU_BUILD=0"
],
"User":"",
"WorkingDir":""
},
"architecture":"amd64",
"container":"c0ddcb49aa663b7042497275d25c39221db72cffea017c21d45942feadaf052e",
"created":"2018-01-29T20:37:52.869007338Z",
"docker_version":"17.06.1-ce",
"history":[
{
"created":"2018-01-25T18:23:30.610528428Z",
"created_by":"/bin/sh -c #(nop) ADD file:a3344b835ea6fdc5692df23826c970403656c6947342e117b2ac1a05956679af in / "
},
{
"created":"2018-01-25T18:23:39.075632267Z",
"created_by":"/bin/sh -c set -xe \t\t&& echo '#!/bin/sh' > /usr/sbin/policy-rc.d \t&& echo 'exit 101' >> /usr/sbin/policy-rc.d \t&& chmod +x /usr/sbin/policy-rc.d \t\t&& dpkg-divert --local --rename --add /sbin/initctl \t&& cp -a /usr/sbin/policy-rc.d /sbin/initctl \t&& sed -i 's/^exit.*/exit 0/' /sbin/initctl \t\t&& echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \t\t&& echo 'DPkg::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' > /etc/apt/apt.conf.d/docker-clean \t&& echo 'APT::Update::Post-Invoke { \"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true\"; };' >> /etc/apt/apt.conf.d/docker-clean \t&& echo 'Dir::Cache::pkgcache \"\"; Dir::Cache::srcpkgcache \"\";' >> /etc/apt/apt.conf.d/docker-clean \t\t&& echo 'Acquire::Languages \"none\";' > /etc/apt/apt.conf.d/docker-no-languages \t\t&& echo 'Acquire::GzipIndexes \"true\"; Acquire::CompressionTypes::Order:: \"gz\";' > /etc/apt/apt.conf.d/docker-gzip-indexes \t\t&& echo 'Apt::AutoRemove::SuggestsImportant \"false\";' > /etc/apt/apt.conf.d/docker-autoremove-suggests"
},
{
"created":"2018-01-25T18:23:39.843140353Z",
"created_by":"/bin/sh -c rm -rf /var/lib/apt/lists/*"
},
{
"created":"2018-01-25T18:23:50.734158349Z",
"created_by":"/bin/sh -c sed -i 's/^#\\s*\\(deb.*universe\\)$/\\1/g' /etc/apt/sources.list"
},
{
"created":"2018-01-25T18:23:51.48088772Z",
"created_by":"/bin/sh -c mkdir -p /run/systemd && echo 'docker' > /run/systemd/container"
},
{
"created":"2018-01-25T18:23:51.755470218Z",
"created_by":"/bin/sh -c #(nop) CMD [\"/bin/bash\"]",
"empty_layer":true
},
{
"created":"2018-01-29T20:35:11.157197346Z",
"created_by":"/bin/sh -c #(nop) ARG DV_GPU_BUILD=0",
"empty_layer":true
},
{
"created":"2018-01-29T20:35:11.250378819Z",
"created_by":"/bin/sh -c #(nop) ENV DV_GPU_BUILD=0",
"empty_layer":true
},
{
"created":"2018-01-29T20:35:11.441069427Z",
"created_by":"/bin/sh -c #(nop) ADD multi:d2d432f1bc663fa828de7f8dca6a5a020f216fd2573801fa1a70a115de62c14f in / "
},
{
"created":"2018-01-29T20:35:11.861361117Z",
"created_by":"/bin/sh -c mkdir -p /tmp/deepvariant_docker && mkdir -p /opt/deepvariant/bin"
},
{
"created":"2018-01-29T20:35:12.010205356Z",
"created_by":"/bin/sh -c #(nop) ADD multi:9fc5b56c2afa8867a337221d5da986135c7a996a7616c14b4dcc10b53bccaf12 in /tmp/deepvariant_docker/ "
},
{
"created":"2018-01-29T20:37:51.450960553Z",
"created_by":"/bin/sh -c cd /tmp/deepvariant_docker && ./run-prereq.sh && rm -rf /tmp/deepvariant_docker/ && apt-get install -y parallel"
},
{
"created":"2018-01-29T20:37:52.426548845Z",
"created_by":"/bin/sh -c #(nop) ADD multi:f613ad7fb56a9bd4e7bc1d9de6395472cdea26f9b2c49e423a3c4e2fa27bd39c in /opt/deepvariant/bin/ "
},
{
"created":"2018-01-29T20:37:52.869007338Z",
"created_by":"/bin/sh -c BASH_HEADER='#!/bin/bash' && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/make_examples.zip \"$@\"' > /opt/deepvariant/bin/make_examples && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/call_variants.zip \"$@\"' > /opt/deepvariant/bin/call_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/postprocess_variants.zip \"$@\"' > /opt/deepvariant/bin/postprocess_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_train.zip \"$@\"' > /opt/deepvariant/bin/model_train && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_eval.zip \"$@\"' > /opt/deepvariant/bin/model_eval && chmod +x /opt/deepvariant/bin/make_examples /opt/deepvariant/bin/call_variants /opt/deepvariant/bin/postprocess_variants /opt/deepvariant/bin/model_train /opt/deepvariant/bin/model_eval"
}
],
"rootfs":{
"diff_ids":[
"sha256:ff986b10a018b48074e6d3a68b39aad8ccc002cdad912d4148c0f92b3729323e",
"sha256:9c7183e0ea88b265d83708dfe5b9189c4e12f9a1d8c3e5bce7f286417653f9b7",
"sha256:c98ef191df4b42c3fd5155d23385e75ee59707c6a448dfc6c8e4e9c005a3df11",
"sha256:92914665e7f61f8f19b56bf7983a2b3758cb617bef498b37adb80899e8b86e32",
"sha256:6f4ce6b888495c7c9bd4a0ac124b039d986a3b18250fa873d11d13b42f6a79f4",
"sha256:d981835cfeee9f8e1b8e68fde5e29ca342e554f41065429ff993c77c99262121",
"sha256:959f030108b71cedde109ac857d1bb4d760c1e18caa3cdbe89c714e968e759bf",
"sha256:9edfd6bd057d9c5da040a314efd9a7c257ef8158629c5990ea74fc3b745ac051",
"sha256:051edfa0dcb187fa742d00482e05f0c7e32e1b6665fa492a031054abb88e655e",
"sha256:51c8d662a1b2d33f15b969bd7bdd70e5a12130ca7a7793d63c944ed31dd5e169",
"sha256:d3bc210757dacfbe3b2bf90a1938d68df319841d9eef59fa44da3bb64f1bbb26"
],
"type":"layers"
},
"container_config":{
"AttachStdin":false,
"Cmd":[
"/bin/sh",
"-c",
"BASH_HEADER='#!/bin/bash' && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/make_examples.zip \"$@\"' > /opt/deepvariant/bin/make_examples && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/call_variants.zip \"$@\"' > /opt/deepvariant/bin/call_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/postprocess_variants.zip \"$@\"' > /opt/deepvariant/bin/postprocess_variants && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_train.zip \"$@\"' > /opt/deepvariant/bin/model_train && printf \"%s\\n%s\\n\" \"${BASH_HEADER}\" 'python /opt/deepvariant/bin/model_eval.zip \"$@\"' > /opt/deepvariant/bin/model_eval && chmod +x /opt/deepvariant/bin/make_examples /opt/deepvariant/bin/call_variants /opt/deepvariant/bin/postprocess_variants /opt/deepvariant/bin/model_train /opt/deepvariant/bin/model_eval"
],
"Volumes":null,
"Image":"sha256:9d30e8ce67321c61be656426c99dc22c6a8be8f1f03a3adcf8e0dd7d08281ed3",
"Tty":false,
"StdinOnce":false,
"Domainname":"",
"AttachStdout":false,
"OnBuild":[
],
"OpenStdin":false,
"Labels":null,
"ArgsEscaped":true,
"Entrypoint":null,
"Hostname":"",
"AttachStderr":false,
"Env":[
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"DV_GPU_BUILD=0"
],
"User":"",
"WorkingDir":""
},
"os":"linux"
}
# library for http requests
import requests
# URL: This is the url for the manifest
url = 'https://gcr.io/v2/deepvariant-docker/deepvariant/manifests/0.5.0'
# The urls themselves are pretty useless, it's all driven by the accept headers we
# provide. This is the next thing that we set:
# ACCEPT HEADER: This means a version 2.0 manifest, this has layers and sizes but not metadata like environment, entrypoint
# We used to always get from version 1 manifest, and now some registries have a config, but gcr.io just has version 2.0
headers = {'Accept': 'application/vnd.docker.distribution.manifest.v2+json'}
# Make the call, view the status code, and the manifest
response = requests.get(url, headers=headers)
# This should be 200
response.status_code
# Here is the manifest that the layers come from. So this is fixed, but we need the config
manifest = response.json()
print(manifest)
digest = manifest['config']['digest']
mediaType = manifest['config']['mediaType']
headers = {'Accept': mediaType}
blob_url = "https://gcr.io/v2/deepvariant-docker/deepvariant/blobs/%s" %digest
config = requests.get(blob_url, headers=headers)
print(config.json())
# Note this is with current branch fix/gcr.io of singularityhub/sregistry-cli
# will be released as pypi 0.0.73 soon :)
$ sregistry pull docker://gcr.io/deepvariant-docker/deepvariant:0.5.0
[client|docker] [database|sqlite:////home/vanessa/.singularity/sregistry.db]
Exploding /usr/local/libexec/singularity/bootstrap-scripts/environment.tar
Exploding /home/vanessa/.singularity/docker/sha256:1be7f2b886e89a58e59c4e685fcc5905a26ddef3201f290b96f1eff7d778e122.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:6fbc4a21b806838b63b774b338c6ad19d696a9e655f50b4e358cc4006c3baa79.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:c71a6f8e13782fed125f2247931c3eb20cc0e6428a5d79edb546f1f1405f0e49.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:4be3072e5a37392e32f632bb234c0b461ff5675ab7e362afad6359fbd36884af.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:06c6d2f5970057aef3aef6da60f0fde280db1c077f0cd88ca33ec1a70a9c7b58.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:c4040a2294230c510b5a00dbc78b22075337b3a9574871f7626a8569c81b3cb4.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:5c2bd73e48dc78f72ebffdf3a8cf3ce71b4f268598ea0943b16de5d4c0e877ae.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:8d8439570e8f78644e334a738bfd1ba926fe3f96c6afd2715a3640d2d45bbf4d.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:5a1f6d79bff2cad8269d67407c273b2c6e3367281605f39b2094182c2a1b5e2b.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:7f3fe796e447ca9fab4debea6894350348a7f62d0041ff99e79e3fc2a6730b8a.tar.gz
Exploding /home/vanessa/.singularity/docker/sha256:045deebf3a2e52b364829d82a64d23897cdc6d7ce5232dfb5f785fde3d2f162f.tar.gz
Exploding /home/vanessa/.singularity/docker/metadata/sha256:deac2c14535690c8d8cca1e1ea987366d65e6a5e2a30bb4108caf2250f728faf.tar.gz
Building into existing container: /home/vanessa/.singularity/shub/deepvariant-docker-deepvariant:0.5.0.simg
WARNING: Building container as an unprivileged user. If you run this container as root
WARNING: it may be missing some functionality.
Building FS image from sandbox: /tmp/tmp2zlyvp3n
Building Singularity FS image...
Building Singularity SIF container image...
Singularity container built: /home/vanessa/.singularity/shub/deepvariant-docker-deepvariant:0.5.0.simg
Cleaning up...
[container][new] deepvariant-docker/deepvariant:0.5.0
Success! /home/vanessa/.singularity/shub/deepvariant-docker-deepvariant:0.5.0.simg
singularity shell $(sregistry get deepvariant-docker/deepvariant:0.5.0)
Singularity: Invoking an interactive shell within container...
Singularity deepvariant-docker-deepvariant:0.5.0.simg:~/Documents/Dropbox/Code/sregistry/sregistry-cli> cat /.singularity.d/env/10-docker.sh
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export DV_GPU_BUILD="0"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment