reprounzip-graph from whole-tale/provenance-examples

ajps.md

AJPS example

Real-world single-step Rscript with inputs and output

reprozip trace Rscript Stokes_AJPS_Replication.R
reprozip pack ajps
reprounzip graph --processes process --packages drop --otherfiles io --regex-filter ^/etc graphfile.dot ajps.rpz
dot -Tsvg graphfile.dot -o graph.svg

ajps.svg

multistep.md

Multistep example

Trace three separate very simple scripts (bash, bash, python) using the --continue flag

reprozip trace ./step1.sh
reprozip trace --continue ./step2.sh
reprozip trace --continue ./plot.py
reprozip pack multistep

reprounzip graph --processes process --packages drop --otherfiles io --regex-filter ^/etc graphfile.dot multistep.rpz
dot -Tsvg graphfile.dot -o graph.svg

multistep.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.38.0 (20140413.2041)
 -->
<!-- Title: G Pages: 1 -->
<svg width="1638pt" height="248pt"
 viewBox="0.00 0.00 1637.54 247.87" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 243.87)">
<title>G</title>
<polygon fill="white" stroke="none" points="-4,4 -4,-243.87 1633.54,-243.87 1633.54,4 -4,4"/>
<g id="clust1" class="cluster"><title>cluster_run0</title>
<polygon fill="none" stroke="black" points="17.566,-156.87 17.566,-231.87 389.132,-231.87 389.132,-156.87 17.566,-156.87"/>
<text text-anchor="middle" x="203.349" y="-216.67" font-family="Times,serif" font-size="14.00">run0</text>
</g>
<g id="clust2" class="cluster"><title>cluster_run1</title>
<polygon fill="none" stroke="black" points="440.648,-72.8701 440.648,-147.87 817.164,-147.87 817.164,-72.8701 440.648,-72.8701"/>
<text text-anchor="middle" x="628.906" y="-132.67" font-family="Times,serif" font-size="14.00">run1</text>
</g>
<g id="clust3" class="cluster"><title>cluster_run2</title>
<polygon fill="none" stroke="black" points="1214.57,-12.8701 1214.57,-87.8701 1385.57,-87.8701 1385.57,-12.8701 1214.57,-12.8701"/>
<text text-anchor="middle" x="1300.07" y="-72.6701" font-family="Times,serif" font-size="14.00">run2</text>
</g>
<!-- prog0 -->
<g id="node1" class="node"><title>prog0</title>
<path fill="black" stroke="black" d="M174.566,-200.87C174.566,-200.87 37.566,-200.87 37.566,-200.87 31.566,-200.87 25.566,-194.87 25.566,-188.87 25.566,-188.87 25.566,-176.87 25.566,-176.87 25.566,-170.87 31.566,-164.87 37.566,-164.87 37.566,-164.87 174.566,-164.87 174.566,-164.87 180.566,-164.87 186.566,-170.87 186.566,-176.87 186.566,-176.87 186.566,-188.87 186.566,-188.87 186.566,-194.87 180.566,-200.87 174.566,-200.87"/>
<text text-anchor="middle" x="106.066" y="-179.17" font-family="Times,serif" font-size="14.00" fill="white">/test/multistep/step1.sh (1)</text>
</g>
<!-- prog1 -->
<g id="node2" class="node"><title>prog1</title>
<path fill="black" stroke="black" d="M369.132,-200.87C369.132,-200.87 316.132,-200.87 316.132,-200.87 310.132,-200.87 304.132,-194.87 304.132,-188.87 304.132,-188.87 304.132,-176.87 304.132,-176.87 304.132,-170.87 310.132,-164.87 316.132,-164.87 316.132,-164.87 369.132,-164.87 369.132,-164.87 375.132,-164.87 381.132,-170.87 381.132,-176.87 381.132,-176.87 381.132,-188.87 381.132,-188.87 381.132,-194.87 375.132,-200.87 369.132,-200.87"/>
<text text-anchor="middle" x="342.632" y="-179.17" font-family="Times,serif" font-size="14.00" fill="white">/bin/cat (2)</text>
</g>
<!-- prog0&#45;&gt;prog1 -->
<g id="edge1" class="edge"><title>prog0&#45;&gt;prog1</title>
<path fill="none" stroke="black" d="M186.807,-182.87C222.366,-182.87 263.072,-182.87 293.819,-182.87"/>
<polygon fill="black" stroke="black" points="293.842,-186.37 303.842,-182.87 293.842,-179.37 293.842,-186.37"/>
<text text-anchor="middle" x="258.132" y="-186.67" font-family="Times,serif" font-size="14.00">fork+exec</text>
</g>
<!-- /test/multistep/output1.txt -->
<g id="node10" class="node"><title>/test/multistep/output1.txt</title>
<ellipse fill="#a3b4e0" stroke="black" cx="529.148" cy="-182.87" rx="111.032" ry="26.7407"/>
<text text-anchor="middle" x="529.148" y="-186.67" font-family="Times,serif" font-size="14.00" fill="#131c39">output1.txt</text>
<text text-anchor="middle" x="529.148" y="-171.67" font-family="Times,serif" font-size="14.00" fill="#131c39">/test/multistep/output1.txt</text>
</g>
<!-- prog1&#45;&gt;/test/multistep/output1.txt -->
<g id="edge3" class="edge"><title>prog1&#45;&gt;/test/multistep/output1.txt</title>
<path fill="none" stroke="#000088" d="M381.417,-182.87C389.469,-182.87 398.385,-182.87 407.726,-182.87"/>
<polygon fill="#000088" stroke="#000088" points="408.009,-186.37 418.009,-182.87 408.009,-179.37 408.009,-186.37"/>
</g>
<!-- prog2 -->
<g id="node3" class="node"><title>prog2</title>
<path fill="black" stroke="black" d="M597.648,-116.87C597.648,-116.87 460.648,-116.87 460.648,-116.87 454.648,-116.87 448.648,-110.87 448.648,-104.87 448.648,-104.87 448.648,-92.8701 448.648,-92.8701 448.648,-86.8701 454.648,-80.8701 460.648,-80.8701 460.648,-80.8701 597.648,-80.8701 597.648,-80.8701 603.648,-80.8701 609.648,-86.8701 609.648,-92.8701 609.648,-92.8701 609.648,-104.87 609.648,-104.87 609.648,-110.87 603.648,-116.87 597.648,-116.87"/>
<text text-anchor="middle" x="529.148" y="-95.1701" font-family="Times,serif" font-size="14.00" fill="white">/test/multistep/step2.sh (3)</text>
</g>
<!-- prog3 -->
<g id="node4" class="node"><title>prog3</title>
<path fill="black" stroke="black" d="M797.164,-116.87C797.164,-116.87 744.164,-116.87 744.164,-116.87 738.164,-116.87 732.164,-110.87 732.164,-104.87 732.164,-104.87 732.164,-92.8701 732.164,-92.8701 732.164,-86.8701 738.164,-80.8701 744.164,-80.8701 744.164,-80.8701 797.164,-80.8701 797.164,-80.8701 803.164,-80.8701 809.164,-86.8701 809.164,-92.8701 809.164,-92.8701 809.164,-104.87 809.164,-104.87 809.164,-110.87 803.164,-116.87 797.164,-116.87"/>
<text text-anchor="middle" x="770.664" y="-95.1701" font-family="Times,serif" font-size="14.00" fill="white">/bin/cat (4)</text>
</g>
<!-- prog2&#45;&gt;prog3 -->
<g id="edge2" class="edge"><title>prog2&#45;&gt;prog3</title>
<path fill="none" stroke="black" d="M609.903,-98.8701C646.949,-98.8701 689.765,-98.8701 721.742,-98.8701"/>
<polygon fill="black" stroke="black" points="722.147,-102.37 732.147,-98.8701 722.147,-95.3702 722.147,-102.37"/>
<text text-anchor="middle" x="686.164" y="-102.67" font-family="Times,serif" font-size="14.00">fork+exec</text>
</g>
<!-- /test/multistep/output2.txt -->
<g id="node11" class="node"><title>/test/multistep/output2.txt</title>
<ellipse fill="#a3b4e0" stroke="black" cx="1015.87" cy="-98.8701" rx="111.032" ry="26.7407"/>
<text text-anchor="middle" x="1015.87" y="-102.67" font-family="Times,serif" font-size="14.00" fill="#131c39">output2.txt</text>
<text text-anchor="middle" x="1015.87" y="-87.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">/test/multistep/output2.txt</text>
</g>
<!-- prog3&#45;&gt;/test/multistep/output2.txt -->
<g id="edge5" class="edge"><title>prog3&#45;&gt;/test/multistep/output2.txt</title>
<path fill="none" stroke="#000088" d="M809.398,-98.8701C832.573,-98.8701 863.673,-98.8701 894.653,-98.8701"/>
<polygon fill="#000088" stroke="#000088" points="894.697,-102.37 904.697,-98.8701 894.696,-95.3702 894.697,-102.37"/>
</g>
<!-- prog4 -->
<g id="node5" class="node"><title>prog4</title>
<path fill="black" stroke="black" d="M1365.57,-56.8701C1365.57,-56.8701 1234.57,-56.8701 1234.57,-56.8701 1228.57,-56.8701 1222.57,-50.8701 1222.57,-44.8701 1222.57,-44.8701 1222.57,-32.8701 1222.57,-32.8701 1222.57,-26.8701 1228.57,-20.8701 1234.57,-20.8701 1234.57,-20.8701 1365.57,-20.8701 1365.57,-20.8701 1371.57,-20.8701 1377.57,-26.8701 1377.57,-32.8701 1377.57,-32.8701 1377.57,-44.8701 1377.57,-44.8701 1377.57,-50.8701 1371.57,-56.8701 1365.57,-56.8701"/>
<text text-anchor="middle" x="1300.07" y="-35.1701" font-family="Times,serif" font-size="14.00" fill="white">/test/multistep/plot.py (5)</text>
</g>
<!-- /test/multistep/figure.pdf -->
<g id="node7" class="node"><title>/test/multistep/figure.pdf</title>
<ellipse fill="#a3b4e0" stroke="black" cx="1522.06" cy="-38.8701" rx="107.46" ry="26.7407"/>
<text text-anchor="middle" x="1522.06" y="-42.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">figure.pdf</text>
<text text-anchor="middle" x="1522.06" y="-27.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">/test/multistep/figure.pdf</text>
</g>
<!-- prog4&#45;&gt;/test/multistep/figure.pdf -->
<g id="edge10" class="edge"><title>prog4&#45;&gt;/test/multistep/figure.pdf</title>
<path fill="none" stroke="#000088" d="M1377.7,-38.8701C1386.35,-38.8701 1395.31,-38.8701 1404.32,-38.8701"/>
<polygon fill="#000088" stroke="#000088" points="1404.51,-42.3702 1414.51,-38.8701 1404.51,-35.3702 1404.51,-42.3702"/>
</g>
<!-- /root/.cache/matplotlib/fontlist&#45;v300.json -->
<g id="node6" class="node"><title>/root/.cache/matplotlib/fontlist&#45;v300.json</title>
<ellipse fill="#a3b4e0" stroke="black" cx="1015.87" cy="-26.8701" rx="169.912" ry="26.7407"/>
<text text-anchor="middle" x="1015.87" y="-30.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">fontlist&#45;v300.json</text>
<text text-anchor="middle" x="1015.87" y="-15.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">/root/.cache/matplotlib/fontlist&#45;v300.json</text>
</g>
<!-- /root/.cache/matplotlib/fontlist&#45;v300.json&#45;&gt;prog4 -->
<g id="edge8" class="edge"><title>/root/.cache/matplotlib/fontlist&#45;v300.json&#45;&gt;prog4</title>
<path fill="none" stroke="#8888cc" d="M1180.21,-33.8158C1191.18,-34.2822 1201.97,-34.7409 1212.3,-35.1801"/>
<polygon fill="#8888cc" stroke="#8888cc" points="1212.32,-38.6842 1222.46,-35.6123 1212.62,-31.6905 1212.32,-38.6842"/>
</g>
<!-- /test/multistep/input1.txt -->
<g id="node8" class="node"><title>/test/multistep/input1.txt</title>
<ellipse fill="#a3b4e0" stroke="black" cx="106.066" cy="-119.87" rx="106.132" ry="26.7407"/>
<text text-anchor="middle" x="106.066" y="-123.67" font-family="Times,serif" font-size="14.00" fill="#131c39">input1.txt</text>
<text text-anchor="middle" x="106.066" y="-108.67" font-family="Times,serif" font-size="14.00" fill="#131c39">/test/multistep/input1.txt</text>
</g>
<!-- /test/multistep/input1.txt&#45;&gt;prog1 -->
<g id="edge4" class="edge"><title>/test/multistep/input1.txt&#45;&gt;prog1</title>
<path fill="none" stroke="#8888cc" d="M179.678,-139.372C216.99,-149.394 261.395,-161.32 294.263,-170.148"/>
<polygon fill="#8888cc" stroke="#8888cc" points="293.506,-173.568 304.072,-172.782 295.322,-166.808 293.506,-173.568"/>
</g>
<!-- /test/multistep/input2.txt -->
<g id="node9" class="node"><title>/test/multistep/input2.txt</title>
<ellipse fill="#a3b4e0" stroke="black" cx="529.148" cy="-35.8701" rx="106.132" ry="26.7407"/>
<text text-anchor="middle" x="529.148" y="-39.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">input2.txt</text>
<text text-anchor="middle" x="529.148" y="-24.6701" font-family="Times,serif" font-size="14.00" fill="#131c39">/test/multistep/input2.txt</text>
</g>
<!-- /test/multistep/input2.txt&#45;&gt;prog3 -->
<g id="edge7" class="edge"><title>/test/multistep/input2.txt&#45;&gt;prog3</title>
<path fill="none" stroke="#8888cc" d="M603.312,-55.1145C641.837,-65.2476 687.982,-77.3854 721.889,-86.3038"/>
<polygon fill="#8888cc" stroke="#8888cc" points="721.435,-89.8034 731.996,-88.9624 723.216,-83.0337 721.435,-89.8034"/>
</g>
<!-- /test/multistep/output1.txt&#45;&gt;prog3 -->
<g id="edge6" class="edge"><title>/test/multistep/output1.txt&#45;&gt;prog3</title>
<path fill="none" stroke="#8888cc" d="M604.774,-163.083C616.681,-159.57 628.812,-155.778 640.164,-151.87 667.944,-142.306 698.38,-129.949 722.761,-119.573"/>
<polygon fill="#8888cc" stroke="#8888cc" points="724.318,-122.714 732.13,-115.557 721.56,-116.28 724.318,-122.714"/>
</g>
<!-- /test/multistep/output2.txt&#45;&gt;prog4 -->
<g id="edge9" class="edge"><title>/test/multistep/output2.txt&#45;&gt;prog4</title>
<path fill="none" stroke="#8888cc" d="M1100.01,-81.1926C1135.38,-73.6726 1176.63,-64.9022 1212.38,-57.3017"/>
<polygon fill="#8888cc" stroke="#8888cc" points="1213.26,-60.6942 1222.31,-55.191 1211.8,-53.8472 1213.26,-60.6942"/>
</g>
</g>
</svg>

onestep.md

Onestep example

Runs three separate very simple scripts (bash, bash, python) via a master script.

reprozip trace ./run.sh
reprozip pack onestep

reprounzip graph --processes process --packages drop --otherfiles io --regex-filter ^/etc graphfile.dot onestep.rpz
dot -Tsvg graphfile.dot -o graph.svg

onestep.svg

water-tale.md

Water tale

Trace execution of nbconvert on a notebook.

reprozip trace  jupyter nbconvert   --ExecutePreprocessor.allow_errors=True   --ExecutePreprocessor.timeout=-1   --FilesWriter.build_directory=./results   --execute workspace/wt_quickstart.ipynb

reprozip pack watertale

reprounzip graph --processes process --packages drop --otherfiles io --regex-filter ^/etc graphfile.dot watertale.rpzk

water-tale.svg