Skip to content

Instantly share code, notes, and snippets.

@dovahcrow

dovahcrow/spec.json

Created August 27, 2020 23:14
Show Gist options
  • Save dovahcrow/17e26f19d5182774e010d660bf8538e3 to your computer and use it in GitHub Desktop.
Save dovahcrow/17e26f19d5182774e010d660bf8538e3 to your computer and use it in GitHub Desktop.
Download ZIP
plot_correlation(df)
Raw
spec.json
{"config": {"view": {"continuousWidth": 400, "continuousHeight": 300}}, "data": {"name": "data-7b5c73cc8d7200157814bec3790eaaf6"}, "mark": {"type": "line", "point": true}, "encoding": {"color": {"type": "nominal", "field": "name"}, "column": {"type": "nominal", "field": "format", "title": "Data Format"}, "row": {"type": "nominal", "field": "reader", "title": "Data Reader"}, "tooltip": [{"type": "quantitative", "field": "elapsed", "format": ".0s"}, {"type": "nominal", "field": "MachineMem"}, {"type": "nominal", "field": "DatasetMemSize"}], "x": {"type": "quantitative", "field": "DVM", "title": "Dataset Size / Memory Size"}, "y": {"type": "quantitative", "field": "elapsed", "title": "Elapsed (s)"}}, "selection": {"selector004": {"type": "interval", "bind": "scales", "encodings": ["x", "y"]}}, "title": "plot_correlation(df) Comparison", "$schema": "https://vega.github.io/schema/vega-lite/v4.8.1.json", "datasets": {"data-7b5c73cc8d7200157814bec3790eaaf6": [{"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.131422758102417, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.40795087814331, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 3.18289041519165, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 4.67631983757019, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 53.46384119987488, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 394.1534848213196, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.1017813682556152, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.302122592926025, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 3.237552404403686, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 4.690297365188599, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 12.990097761154175, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 108.98597288131714, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.065635919570923, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV2", "elapsed": 2.479875087738037, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 3.268184423446655, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV2", "elapsed": 4.8982720375061035, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 12.968538522720337, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV2", "elapsed": 20.99100399017334, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.146351814270019, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.600064992904663, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 2.7515735626220703, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 3.385938882827759, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 39.08425235748291, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 26.565727472305298, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.106266498565674, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.673638105392456, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 2.723254203796386, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 3.278810977935791, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 7.050476789474487, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 10.210280179977417, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.126279354095459, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV2", "elapsed": 2.636481523513794, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 2.853739738464355, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV2", "elapsed": 3.452131032943725, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 6.919025897979736, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-new.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV2", "elapsed": 10.16187572479248, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 2.859402894973755, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 3.08121919631958, "DVM": 0.024269841611385345, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 5.17824125289917, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 6.808420181274414, "DVM": 0.24266795720905066, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 841.8001124858856, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 1300.1377506256104, "DVM": 2.4266405291855335, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 2.947719097137451, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 3.072242021560669, "DVM": 0.012134920805692673, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 5.478638648986816, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 6.9808759689331055, "DVM": 0.12133397860452533, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 248.2077248096466, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 276.2861773967743, "DVM": 1.2133202645927668, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 2.8642492294311523, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 26059544, "name": "PlotCorrelationV1", "elapsed": 3.171290874481201, "DVM": 0.006067460402846336, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 5.504069089889526, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 260562735, "name": "PlotCorrelationV1", "elapsed": 7.099094867706299, "DVM": 0.060666989302262664, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 49.51304578781128, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "csv", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2605585428, "name": "PlotCorrelationV1", "elapsed": 45.36088466644287, "DVM": 0.6066601322963834, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 2.9847865104675293, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 3.6141364574432373, "DVM": 0.023661579005420208, "MachineMem": "1.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 4.650147438049316, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 5.431140899658203, "DVM": 0.23664921056479216, "MachineMem": "1.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 416.8697440624237, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 1073741824, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 166.59323120117188, "DVM": 2.36647779494524, "MachineMem": "1.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 2.761967182159424, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 3.419968843460083, "DVM": 0.011830789502710104, "MachineMem": "2.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 4.6529576778411865, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 5.436981201171875, "DVM": 0.11832460528239608, "MachineMem": "2.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 93.51000499725342, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 2147483648, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 54.17302989959717, "DVM": 1.18323889747262, "MachineMem": "2.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "pandas", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 2.744046926498413, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 50000, "partition": 16, "reader": "dask", "mem_size": 25406427, "name": "PlotCorrelationV1", "elapsed": 3.357399940490722, "DVM": 0.005915394751355052, "MachineMem": "4.0G", "DatasetMemSize": "0.0G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "pandas", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 4.612894058227539, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 500000, "partition": 16, "reader": "dask", "mem_size": 254100155, "name": "PlotCorrelationV1", "elapsed": 5.302222490310669, "DVM": 0.05916230264119804, "MachineMem": "4.0G", "DatasetMemSize": "0.2G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "pandas", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 28.186084032058716, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}, {"cpu": 8, "dataset": "adult", "fname": "plot-old.py", "format": "parquet", "memory": 4294967296, "ncol": 12, "nrow": 5000000, "partition": 16, "reader": "dask", "mem_size": 2540986184, "name": "PlotCorrelationV1", "elapsed": 27.041035413742065, "DVM": 0.59161944873631, "MachineMem": "4.0G", "DatasetMemSize": "2.4G"}]}}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment