cswinter

Results of comparing pco and xor compression at https://github.com/cswinter/LocustDB/commit/46edbce88f2aceb3f5c12da2fcc123486390ff58

$ cargo run --release --example gorilla_time
Compression ratio of 3.17 for trace/rollout/forward/action_heads (max_regret=100)
Compression ratio of 5.68 for actions/act_right_up (max_regret=100)
Compression ratio of 2.36 for episode_length.mean (max_regret=100)
Compression ratio of 11.95 for reward.max (max_regret=100)
Compression ratio of 2.37 for gradnorm (max_regret=100)
Compression ratio of 13.52 for sps (max_regret=100)
Compression ratio of 2.20 for approxkl (max_regret=100)

cswinter

mod arc_buffer {
    use lazy_static::lazy_static;
    use pyo3::ffi;
    use pyo3::prelude::*;
    use pyo3::AsPyPointer;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::sync::Arc;

    lazy_static! {
        static ref COUNTER: AtomicUsize = AtomicUsize::default();

cswinter

root@managed-worker-3jq5:/# mpirun --allow-run-as-root -H 10.73.0.209:1,10.73.0.244:1 -np 2 -mca btl_tcp_if_include ens12 -x NCCL_IB_DISABLE=1 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_DEBUG=VERSION -x NCCL_NSOCKS_PERTHREAD=1 -x NCCL_SOCKET_NTHREADS=4 /nccl-tests/build/all_reduce_perf -b 1M -e 1G -f 2 -g 1 -c 0
nThread 1 nGpus 1 minBytes 1048576 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
NCCL version 2.4.7ms0+cuda10.0
# NCCL Tests compiled with NCCL 2.4
# Using devices
#   Rank  0 on managed-worker-3jq5 device  0 [0x00] Tesla V100-SXM2-16GB

#                                                 out-of-place                    in-place
#      bytes             N    type      op     time  algbw  busbw      res     time  algbw  busbw      res
#   Rank  1 on managed-worker-0khr device  0 [0x00] Tesla V100-SXM2-16GB

cswinter

root@managed-worker-sqfj:/# for nrings in 1 2 4 8 16 32; do mpirun --allow-run-as-root -H 10.73.0.19:8,10.73.0.95:8 -np 16 -mca btl_tcp_if_include ens12 -x NCCL_IB_DISABLE=1 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_DEBUG=INFO -x NCCL_MIN_NRINGS=$nrings -x NCCL_MAX_NRINGS=$nrings /nccl-tests/build/all_reduce_perf -b 16M -e 1G -f 2 -g 1 -c 0; done
# nThread 1 nGpus 1 minBytes 16777216 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
#
# Using devices
#   Rank  0 Pid   3071 on managed-worker-sqfj device  0 [0x00] Tesla V100-SXM2-16GB
#   Rank  1 Pid   3072 on managed-worker-sqfj device  1 [0x00] Tesla V100-SXM2-16GB
#   Rank  2 Pid   3073 on managed-worker-sqfj device  2 [0x00] Tesla V100-SXM2-16GB
#   Rank  3 Pid   3074 on managed-worker-sqfj device  3 [0x00] Tesla V100-SXM2-16GB
#   Rank  4 Pid   3076 on managed-worker-sqfj device  4 [0x00] Tesla V100-SXM2-16GB
#   Rank  5 Pid   3077 on managed-worker-sqfj device  5 [0x00] Tesla V100-SXM2-16GB

cswinter

root@managed-worker-l83z:/# mpirun --allow-run-as-root -H 10.73.0.52:1,10.73.0.15:1 -np 2 -mca btl_tcp_if_include ens12 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_MIN_NRINGS=1 -x NCCL_MAX_NRINGS=1 -x NCCL_DEBUG=TRACE /nccl-tests/build/all_reduce_perf -b 1G -e 1G -f 2 -g 1 -c 0
# nThread 1 nGpus 1 minBytes 1073741824 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
#
# Using devices
#   Rank  0 Pid   7938 on managed-worker-l83z device  0 [0x00] Tesla V100-SXM2-16GB
#   Rank  1 Pid   8021 on managed-worker-jbk7 device  0 [0x00] Tesla V100-SXM2-16GB
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/Socket : Using [0]ens12:10.73.0.52<0>
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so).
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/IB : No device found.
managed-worker-jbk7:8021:8021 [0] NCCL INFO NET/Socket : Using [0]ens12:10.73.0.15<0>

cswinter

root@managed-worker-l83z:/nccl-tests# mpirun --allow-run-as-root -H 10.73.0.52:1,10.73.0.15:1 -np 2 -mca btl_tcp_if_include ens12 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_MIN_NRINGS=1 -x NCCL_MAX_NRINGS=1 -x NCCL_DEBUG=TRACE /nccl-tests/build/all_reduce_perf -b 1G -e 1G -f 2 -g 1 -c 0
nThread 1 nGpus 1 minBytes 1073741824 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
managed-worker-l83z:16157:16157 [0] NCCL INFO NET : Using interface ens12:10.73.0.52<0>
managed-worker-l83z:16157:16157 [0] NCCL INFO NET/IB : Using interface ens12 for sideband communication
managed-worker-l83z:16157:16157 [0] NCCL INFO Using internal Network Socket
managed-worker-l83z:16157:16157 [0] NCCL INFO NET : Using interface ens12:10.73.0.52<0>
managed-worker-l83z:16157:16157 [0] NCCL INFO NET/Socket : 1 interfaces found
NCCL version 2.3.7+cuda10.0
managed-worker-l83z:16157:16157 [0] NCCL INFO rank 0 nranks 2
managed-worker-l83z:16157:16163 [0] NCCL INFO comm 0x7f11600566a0 rank 0 nranks 2

cswinter

➜  ~ cd src/LocustDB   
➜  LocustDB git:(master) git fetch origin
^C
➜  LocustDB git:(master) git fetch && rebase
Enter passphrase for key '/home/clemens/.ssh/id_rsa': 

➜  LocustDB git:(master) git fetch && git rebase
Enter passphrase for key '/home/clemens/.ssh/id_rsa': 
remote: Counting objects: 21, done.
remote: Compressing objects: 100% (3/3), done.

cswinter

➜  nyc-clickhouse clickhouse-client --max_threads=12 --max_memory_usage=60000000000
ClickHouse client version 1.1.54388.
Connecting to localhost:9000 as user default.
Connected to ClickHouse server version 1.1.54388.

Archon :) show tables

SHOW TABLES

┌─name──┐

cswinter

import sys


for line in sys.stdin:
        print ','.join([item if len(item.strip()) else '\N'
                                for item in line.strip().split(',')])

cswinter

time (for filename in /home/clemens/Dropbox/nyc-taxi-data-denormalized/trips_x*.csv.gz; do
            gunzip -c $filename | \
                python trans.py | \
                clickhouse-client \
                    --query="INSERT INTO trips FORMAT CSV"
        done)