View arc_vec.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
mod arc_buffer { | |
use lazy_static::lazy_static; | |
use pyo3::ffi; | |
use pyo3::prelude::*; | |
use pyo3::AsPyPointer; | |
use std::sync::atomic::{AtomicUsize, Ordering}; | |
use std::sync::Arc; | |
lazy_static! { | |
static ref COUNTER: AtomicUsize = AtomicUsize::default(); |
View nccl 2.4.7ms0+cuda10.0 with NCCL_SOCKET_NTHREADS=4 on 32Gbps GCE machine
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
root@managed-worker-3jq5:/# mpirun --allow-run-as-root -H 10.73.0.209:1,10.73.0.244:1 -np 2 -mca btl_tcp_if_include ens12 -x NCCL_IB_DISABLE=1 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_DEBUG=VERSION -x NCCL_NSOCKS_PERTHREAD=1 -x NCCL_SOCKET_NTHREADS=4 /nccl-tests/build/all_reduce_perf -b 1M -e 1G -f 2 -g 1 -c 0 | |
nThread 1 nGpus 1 minBytes 1048576 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0 | |
NCCL version 2.4.7ms0+cuda10.0 | |
# NCCL Tests compiled with NCCL 2.4 | |
# Using devices | |
# Rank 0 on managed-worker-3jq5 device 0 [0x00] Tesla V100-SXM2-16GB | |
# out-of-place in-place | |
# bytes N type op time algbw busbw res time algbw busbw res | |
# Rank 1 on managed-worker-0khr device 0 [0x00] Tesla V100-SXM2-16GB |
View nccl bw test v2.3.7 node-2 gpu-16 nrings-1-8 ms-16MB-1GB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
root@managed-worker-sqfj:/# for nrings in 1 2 4 8 16 32; do mpirun --allow-run-as-root -H 10.73.0.19:8,10.73.0.95:8 -np 16 -mca btl_tcp_if_include ens12 -x NCCL_IB_DISABLE=1 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_DEBUG=INFO -x NCCL_MIN_NRINGS=$nrings -x NCCL_MAX_NRINGS=$nrings /nccl-tests/build/all_reduce_perf -b 16M -e 1G -f 2 -g 1 -c 0; done | |
# nThread 1 nGpus 1 minBytes 16777216 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0 | |
# | |
# Using devices | |
# Rank 0 Pid 3071 on managed-worker-sqfj device 0 [0x00] Tesla V100-SXM2-16GB | |
# Rank 1 Pid 3072 on managed-worker-sqfj device 1 [0x00] Tesla V100-SXM2-16GB | |
# Rank 2 Pid 3073 on managed-worker-sqfj device 2 [0x00] Tesla V100-SXM2-16GB | |
# Rank 3 Pid 3074 on managed-worker-sqfj device 3 [0x00] Tesla V100-SXM2-16GB | |
# Rank 4 Pid 3076 on managed-worker-sqfj device 4 [0x00] Tesla V100-SXM2-16GB | |
# Rank 5 Pid 3077 on managed-worker-sqfj device 5 [0x00] Tesla V100-SXM2-16GB |
View nccl 2.4.2 bw test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
root@managed-worker-l83z:/# mpirun --allow-run-as-root -H 10.73.0.52:1,10.73.0.15:1 -np 2 -mca btl_tcp_if_include ens12 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_MIN_NRINGS=1 -x NCCL_MAX_NRINGS=1 -x NCCL_DEBUG=TRACE /nccl-tests/build/all_reduce_perf -b 1G -e 1G -f 2 -g 1 -c 0 | |
# nThread 1 nGpus 1 minBytes 1073741824 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0 | |
# | |
# Using devices | |
# Rank 0 Pid 7938 on managed-worker-l83z device 0 [0x00] Tesla V100-SXM2-16GB | |
# Rank 1 Pid 8021 on managed-worker-jbk7 device 0 [0x00] Tesla V100-SXM2-16GB | |
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/Socket : Using [0]ens12:10.73.0.52<0> | |
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so). | |
managed-worker-l83z:7938:7938 [0] NCCL INFO NET/IB : No device found. | |
managed-worker-jbk7:8021:8021 [0] NCCL INFO NET/Socket : Using [0]ens12:10.73.0.15<0> |
View nccl 2.3.7 bw test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
root@managed-worker-l83z:/nccl-tests# mpirun --allow-run-as-root -H 10.73.0.52:1,10.73.0.15:1 -np 2 -mca btl_tcp_if_include ens12 -x LD_LIBRARY_PATH -x NCCL_SOCKET_IFNAME=ens12 -x NCCL_MIN_NRINGS=1 -x NCCL_MAX_NRINGS=1 -x NCCL_DEBUG=TRACE /nccl-tests/build/all_reduce_perf -b 1G -e 1G -f 2 -g 1 -c 0 | |
nThread 1 nGpus 1 minBytes 1073741824 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 20 validation: 0 | |
managed-worker-l83z:16157:16157 [0] NCCL INFO NET : Using interface ens12:10.73.0.52<0> | |
managed-worker-l83z:16157:16157 [0] NCCL INFO NET/IB : Using interface ens12 for sideband communication | |
managed-worker-l83z:16157:16157 [0] NCCL INFO Using internal Network Socket | |
managed-worker-l83z:16157:16157 [0] NCCL INFO NET : Using interface ens12:10.73.0.52<0> | |
managed-worker-l83z:16157:16157 [0] NCCL INFO NET/Socket : 1 interfaces found | |
NCCL version 2.3.7+cuda10.0 | |
managed-worker-l83z:16157:16157 [0] NCCL INFO rank 0 nranks 2 | |
managed-worker-l83z:16157:16163 [0] NCCL INFO comm 0x7f11600566a0 rank 0 nranks 2 |
View locustdb-mem-breakdown
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
➜ ~ cd src/LocustDB | |
➜ LocustDB git:(master) git fetch origin | |
^C | |
➜ LocustDB git:(master) git fetch && rebase | |
Enter passphrase for key '/home/clemens/.ssh/id_rsa': | |
➜ LocustDB git:(master) git fetch && git rebase | |
Enter passphrase for key '/home/clemens/.ssh/id_rsa': | |
remote: Counting objects: 21, done. | |
remote: Compressing objects: 100% (3/3), done. |
View clickhouse-ingestion-ssd
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
➜ nyc-clickhouse clickhouse-client --max_threads=12 --max_memory_usage=60000000000 | |
ClickHouse client version 1.1.54388. | |
Connecting to localhost:9000 as user default. | |
Connected to ClickHouse server version 1.1.54388. | |
Archon :) show tables | |
SHOW TABLES | |
┌─name──┐ |
View trans.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys | |
for line in sys.stdin: | |
print ','.join([item if len(item.strip()) else '\N' | |
for item in line.strip().split(',')]) |
View clickhouse-log-import
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
time (for filename in /home/clemens/Dropbox/nyc-taxi-data-denormalized/trips_x*.csv.gz; do | |
gunzip -c $filename | \ | |
python trans.py | \ | |
clickhouse-client \ | |
--query="INSERT INTO trips FORMAT CSV" | |
done) |
View clickhouse-cold-query-hdd-1-thread
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
➜ ~ uname -a | |
Linux Archon 4.4.0-66-generic #87-Ubuntu SMP Fri Mar 3 15:29:05 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux | |
➜ ~ cd nyc-benchmarks/nyc-clickhouse | |
➜ nyc-clickhouse sudo lshw -class disk -class storage | |
[sudo] password for clemens: | |
*-storage | |
description: SATA controller | |
product: ASM1062 Serial ATA Controller | |
vendor: ASMedia Technology Inc. | |
physical id: 0 |
NewerOlder