Skip to content

Instantly share code, notes, and snippets.

@dovahcrow

dovahcrow/spec.json

Created August 28, 2020 05:05
Show Gist options
  • Save dovahcrow/9dad96ea7eb75bcc62aa6aefe69b8f55 to your computer and use it in GitHub Desktop.
Save dovahcrow/9dad96ea7eb75bcc62aa6aefe69b8f55 to your computer and use it in GitHub Desktop.
Download ZIP
plot_correlation(df)
Raw
spec.json
{"config": {"view": {"continuousWidth": 400, "continuousHeight": 300}}, "data": {"name": "data-936f2516f58d18e216efa62e17d2ca93"}, "mark": {"type": "line", "point": true}, "encoding": {"color": {"type": "nominal", "field": "name"}, "column": {"type": "nominal", "field": "format", "title": "Data Format"}, "row": {"type": "nominal", "field": "reader", "title": "Data Reader"}, "tooltip": [{"type": "nominal", "field": "name"}, {"type": "quantitative", "field": "elapsed", "format": ".0s"}, {"type": "nominal", "field": "MachineMem"}, {"type": "nominal", "field": "DatasetMemSize"}], "x": {"type": "quantitative", "field": "DVM", "title": "Dataset Size / Memory Size"}, "y": {"type": "quantitative", "field": "elapsed", "title": "Elapsed (s)"}}, "selection": {"selector020": {"type": "interval", "bind": "scales", "encodings": ["x", "y"]}}, "title": "plot_correlation(df) Comparison", "$schema": "https://vega.github.io/schema/vega-lite/v4.8.1.json", "datasets": {"data-936f2516f58d18e216efa62e17d2ca93": [{"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.131422758102417, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.40795087814331, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 3.18289041519165, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 4.67631983757019, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 53.46384119987488, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 394.1534848213196, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.1017813682556152, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.302122592926025, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 3.237552404403686, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 4.690297365188599, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 12.990097761154175, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 108.98597288131714, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.065635919570923, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.479875087738037, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 3.268184423446655, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 4.8982720375061035, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 12.968538522720337, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 20.99100399017334, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.146351814270019, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.600064992904663, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 2.7515735626220703, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 3.385938882827759, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 39.08425235748291, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 26.565727472305298, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.106266498565674, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.673638105392456, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 2.723254203796386, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 3.278810977935791, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 7.050476789474487, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 10.210280179977417, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.126279354095459, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.636481523513794, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 2.853739738464355, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 3.452131032943725, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 6.919025897979736, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 10.16187572479248, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 2.859402894973755, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 3.08121919631958, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 5.17824125289917, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 6.808420181274414, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 841.8001124858856, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 1300.1377506256104, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 2.947719097137451, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 3.072242021560669, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 5.478638648986816, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 6.9808759689331055, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 248.2077248096466, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 276.2861773967743, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 2.8642492294311523, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 3.171290874481201, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 5.504069089889526, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 7.099094867706299, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 49.51304578781128, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 45.36088466644287, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 2.9847865104675293, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 3.6141364574432373, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 4.650147438049316, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 5.431140899658203, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 416.8697440624237, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 166.59323120117188, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 2.761967182159424, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 3.419968843460083, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 4.6529576778411865, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 5.436981201171875, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 93.51000499725342, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 54.17302989959717, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 2.744046926498413, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 3.357399940490722, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 4.612894058227539, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 5.302222490310669, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 28.186084032058716, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 27.041035413742065, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationPandas", "elapsed": 2.138987779617309, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationPandas", "elapsed": 5.05703330039978, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationPandas", "elapsed": 74.62030220031738, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationPandas", "elapsed": 2.111429452896118, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationPandas", "elapsed": 5.081582069396973, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationPandas", "elapsed": 48.93668746948242, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationPandas", "elapsed": 2.123546361923217, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationPandas", "elapsed": 5.185461759567261, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationPandas", "elapsed": 49.34950041770935, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationPandas", "elapsed": 2.148594141006469, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationPandas", "elapsed": 3.795788049697876, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationPandas", "elapsed": 54.98441958427429, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationPandas", "elapsed": 2.00830602645874, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationPandas", "elapsed": 3.780331134796142, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationPandas", "elapsed": 29.405353784561157, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationPandas", "elapsed": 2.121194362640381, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationPandas", "elapsed": 3.787897586822509, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationPandas", "elapsed": 29.549097299575806, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "pandas", "mem_size": 3585393148, "name": "PlotCorrelationPandas", "elapsed": 213.94335913658142, "DVM": 3.3391575776040554, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "dask", "mem_size": 3585393148, "name": "PlotCorrelationPandas", "elapsed": 371.58917927742004, "DVM": 3.3391575776040554, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "pandas", "mem_size": 3585387433, "name": "PlotCorrelationPandas", "elapsed": 108.92083024978638, "DVM": 3.3391522550955415, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-pandas.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "dask", "mem_size": 3585387433, "name": "PlotCorrelationPandas", "elapsed": 235.94760656356812, "DVM": 3.3391522550955415, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "pandas", "mem_size": 3585393148, "name": "PlotCorrelationV2", "elapsed": 1354.3520877361298, "DVM": 3.3391575776040554, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "dask", "mem_size": 3585393148, "name": "PlotCorrelationV2", "elapsed": 2002.254415512085, "DVM": 3.3391575776040554, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "pandas", "mem_size": 3585387433, "name": "PlotCorrelationV2", "elapsed": 589.7265813350677, "DVM": 3.3391522550955415, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "dask", "mem_size": 3585387433, "name": "PlotCorrelationV2", "elapsed": 548.0799398422241, "DVM": 3.3391522550955415, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "pandas", "mem_size": 3585393148, "name": "PlotCorrelationV1", "elapsed": 2014.1933243274689, "DVM": 3.3391575776040554, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "dask", "mem_size": 3585393148, "name": "PlotCorrelationV1", "elapsed": 2706.238584756851, "DVM": 3.3391575776040554, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "pandas", "mem_size": 3585387433, "name": "PlotCorrelationV1", "elapsed": 1068.52019906044, "DVM": 3.3391522550955415, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 7000000, "partition": 16, "reader": "dask", "mem_size": 3585387433, "name": "PlotCorrelationV1", "elapsed": 731.9462985992432, "DVM": 3.3391522550955415, "MachineMem": "1.0G", "DatasetMemSize": "3.3G"}]}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment