Clemens Winter cswinter

## nccl 2.4.7ms0+cuda10.0 with NCCL_SOCKET_NTHREADS=4 on 32Gbps GCE machine
root@managed-worker-3jq5:/# mpirun --allow-run-as-root -H 10.73.0.209:1,10.73.0.244:1 -np 2 -mca btl_tcp_if_include ens12 -x NCCL_IB_DISABLE=1 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_DEBUG=VERSION -x NCCL_NSOCKS_PERTHREAD=1 -x NCCL_SOCKET_NTHREADS=4 /nccl-tests/build/all_reduce_perf -b 1M -e 1G -f 2 -g 1 -c 0
nThread 1 nGpus 1 minBytes 1048576 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
NCCL version 2.4.7ms0+cuda10.0
# NCCL Tests compiled with NCCL 2.4
# Using devices
#   Rank  0 on managed-worker-3jq5 device  0 [0x00] Tesla V100-SXM2-16GB

#                                                 out-of-place                    in-place
#      bytes             N    type      op     time  algbw  busbw      res     time  algbw  busbw      res
#   Rank  1 on managed-worker-0khr device  0 [0x00] Tesla V100-SXM2-16GB

## nccl bw test v2.3.7 node-2 gpu-16 nrings-1-8 ms-16MB-1GB
root@managed-worker-sqfj:/# for nrings in 1 2 4 8 16 32; do mpirun --allow-run-as-root -H 10.73.0.19:8,10.73.0.95:8 -np 16 -mca btl_tcp_if_include ens12 -x NCCL_IB_DISABLE=1 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_DEBUG=INFO -x NCCL_MIN_NRINGS=$nrings -x NCCL_MAX_NRINGS=$nrings /nccl-tests/build/all_reduce_perf -b 16M -e 1G -f 2 -g 1 -c 0; done
# nThread 1 nGpus 1 minBytes 16777216 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
#
# Using devices
#   Rank  0 Pid   3071 on managed-worker-sqfj device  0 [0x00] Tesla V100-SXM2-16GB
#   Rank  1 Pid   3072 on managed-worker-sqfj device  1 [0x00] Tesla V100-SXM2-16GB
#   Rank  2 Pid   3073 on managed-worker-sqfj device  2 [0x00] Tesla V100-SXM2-16GB
#   Rank  3 Pid   3074 on managed-worker-sqfj device  3 [0x00] Tesla V100-SXM2-16GB
#   Rank  4 Pid   3076 on managed-worker-sqfj device  4 [0x00] Tesla V100-SXM2-16GB
#   Rank  5 Pid   3077 on managed-worker-sqfj device  5 [0x00] Tesla V100-SXM2-16GB

## nccl 2.4.2 bw test
root@managed-worker-l83z:/# mpirun --allow-run-as-root -H 10.73.0.52:1,10.73.0.15:1 -np 2 -mca btl_tcp_if_include ens12 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_MIN_NRINGS=1 -x NCCL_MAX_NRINGS=1 -x NCCL_DEBUG=TRACE /nccl-tests/build/all_reduce_perf -b 1G -e 1G -f 2 -g 1 -c 0
# nThread 1 nGpus 1 minBytes 1073741824 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
#
# Using devices
#   Rank  0 Pid   7938 on managed-worker-l83z device  0 [0x00] Tesla V100-SXM2-16GB
#   Rank  1 Pid   8021 on managed-worker-jbk7 device  0 [0x00] Tesla V100-SXM2-16GB
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/Socket : Using [0]ens12:10.73.0.52<0>
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so).
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/IB : No device found.
managed-worker-jbk7:8021:8021 [0] NCCL INFO NET/Socket : Using [0]ens12:10.73.0.15<0>

## nccl 2.3.7 bw test
root@managed-worker-l83z:/nccl-tests# mpirun --allow-run-as-root -H 10.73.0.52:1,10.73.0.15:1 -np 2 -mca btl_tcp_if_include ens12 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_MIN_NRINGS=1 -x NCCL_MAX_NRINGS=1 -x NCCL_DEBUG=TRACE /nccl-tests/build/all_reduce_perf -b 1G -e 1G -f 2 -g 1 -c 0
nThread 1 nGpus 1 minBytes 1073741824 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0
managed-worker-l83z:16157:16157 [0] NCCL INFO NET : Using interface ens12:10.73.0.52<0>
managed-worker-l83z:16157:16157 [0] NCCL INFO NET/IB : Using interface ens12 for sideband communication
managed-worker-l83z:16157:16157 [0] NCCL INFO Using internal Network Socket
managed-worker-l83z:16157:16157 [0] NCCL INFO NET : Using interface ens12:10.73.0.52<0>
managed-worker-l83z:16157:16157 [0] NCCL INFO NET/Socket : 1 interfaces found
NCCL version 2.3.7+cuda10.0
managed-worker-l83z:16157:16157 [0] NCCL INFO rank 0 nranks 2
managed-worker-l83z:16157:16163 [0] NCCL INFO comm 0x7f11600566a0 rank 0 nranks 2

## locustdb-mem-breakdown
➜  ~ cd src/LocustDB
➜  LocustDB git:(master) git fetch origin
^C
➜  LocustDB git:(master) git fetch && rebase
Enter passphrase for key '/home/clemens/.ssh/id_rsa':

➜  LocustDB git:(master) git fetch && git rebase
Enter passphrase for key '/home/clemens/.ssh/id_rsa':
remote: Counting objects: 21, done.
remote: Compressing objects: 100% (3/3), done.

## clickhouse-ingestion-ssd
➜  nyc-clickhouse clickhouse-client --max_threads=12 --max_memory_usage=60000000000
ClickHouse client version 1.1.54388.
Connecting to localhost:9000 as user default.
Connected to ClickHouse server version 1.1.54388.

Archon :) show tables

SHOW TABLES

┌─name──┐

## trans.py
import sys


for line in sys.stdin:
        print ','.join([item if len(item.strip()) else '\N'
                                for item in line.strip().split(',')])

## clickhouse-log-import
time (for filename in /home/clemens/Dropbox/nyc-taxi-data-denormalized/trips_x*.csv.gz; do
            gunzip -c $filename | \
                python trans.py | \
                clickhouse-client \
                    --query="INSERT INTO trips FORMAT CSV"
        done)

## clickhouse-cold-query-hdd-1-thread
➜  ~ uname -a
Linux Archon 4.4.0-66-generic #87-Ubuntu SMP Fri Mar 3 15:29:05 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
➜  ~ cd nyc-benchmarks/nyc-clickhouse
➜  nyc-clickhouse sudo lshw -class disk -class storage
[sudo] password for clemens:
  *-storage
       description: SATA controller
       product: ASM1062 Serial ATA Controller
       vendor: ASMedia Technology Inc.

## clickhouse-cold-query-ssd
➜  ~ uname -a
Linux Archon 4.4.0-66-generic #87-Ubuntu SMP Fri Mar 3 15:29:05 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
➜  ~ sudo lshw -class disk -class storage
[sudo] password for clemens:
  *-storage
       description: SATA controller
       product: ASM1062 Serial ATA Controller
       vendor: ASMedia Technology Inc.
       physical id: 0