Skip to content

Instantly share code, notes, and snippets.

View blackrez's full-sized avatar

Nabil Servais blackrez

View GitHub Profile
import duckdb
import connectorx as cx
con = duckdb.connect(database=':memory:')
sqlite_db = "sqlite://station.db"
query_sqlite = "SELECT * FROM stations"
arr = cx.read_sql(sqlite_db, query_sqlite, return_type='arrow')
r = con.execute('SELECT ville, count(1) FROM arr GROUP BY ville')
r.fetchall()
# [('Marseille', 63), ('Paris', 54), ('Toulouse', 34), ('Nice', 29), ('Limoges', 22), ('Montpellier', 20), ('Le Mans', 18), ('Reims', 18), ('Lyon', 17), ('Nantes', 17)]
@blackrez
blackrez / example_thread.py
Created February 6, 2022 19:39
duckdb with thread
from multiprocessing import connection
from time import sleep, perf_counter
from threading import Thread
import duckdb
con = duckdb.connect(database=':memory:')
con.execute("CREATE TABLE items(item VARCHAR, value DECIMAL(10,2), count INTEGER)")
con.execute("INSERT INTO items VALUES ('jeans', 20.0, 42), ('hammer', 42.2, 4242)")
import boto3
s3 = boto3.client('s3')
resp = s3.select_object_content(
Bucket='carto-1000x',
Key='data/yellow_tripdata_2016–01.csv',
ExpressionType='SQL',
Expression="SELECT avg(cast(tip_amount as float)) , count(1) FROM s3object s",
InputSerialization = {'CSV': {"FileHeaderInfo": "Use", 'FieldDelimiter': ',','RecordDelimiter': '\n'}, 'CompressionType': 'NONE'},
OutputSerialization = {'CSV': {}},
)
resp = s3.select_object_content(
Bucket='carto-1000x',
Key='data/yellow_tripdata_2016–01.csv'',
ExpressionType='SQL',
Expression="SELECT tip_amount FROM s3object s",
InputSerialization = {'CSV': {"FileHeaderInfo": "Use",'FieldDelimiter': ',','RecordDelimiter': '\n'}, 'CompressionType': 'NONE'},
OutputSerialization = {'CSV': {}},
)
import csv
import urllib.request
import codecs
url = "https://s3.amazonaws.com/carto-1000x/data/yellow_tripdata_2016-01.csv"
stream = urllib.request.urlopen(url)
csvfile = csv.DictReader(codecs.iterdecode(stream, 'utf-8'))
count = 0
z = 0.0
for line in csvfile:
-- Build files have been written to: /home/ubuntu/scylla/build/release/seastar
[1/3] cd /home/ubuntu/scylla/build/release/seastar/_cooking/ingredient/dpdk/build && /usr/bin/cmake -E chdir /home/ubuntu/scylla/seastar/dpdk make -j 2 "EXTRA_CFLAGS=-Wno-error -fcommon" O=/home/ubuntu/scylla/build/release/seastar/_cooking/ingredient/dpdk/build DESTDIR=/home/ubuntu/scylla/build/release/seastar/_cooking/stow/dpdk T=arm64-armv8a-linuxapp-gcc install && /usr/bin/cmake -E touch /home/ubuntu/scylla/build/release/seastar/_cooking/ingredient/dpdk/stamp/ingredient_dpdk-install
FAILED: _cooking/ingredient/dpdk/stamp/ingredient_dpdk-install
cd /home/ubuntu/scylla/build/release/seastar/_cooking/ingredient/dpdk/build && /usr/bin/cmake -E chdir /home/ubuntu/scylla/seastar/dpdk make -j 2 "EXTRA_CFLAGS=-Wno-error -fcommon" O=/home/ubuntu/scylla/build/release/seastar/_cooking/ingredient/dpdk/build DESTDIR=/home/ubuntu/scylla/build/release/seastar/_cooking/stow/dpdk T=arm64-armv8a-linuxapp-gcc install && /usr/bin/cmake -E touch /h
nabil@jetson-nano:~/go/src/gorgonia.org/gorgonia/examples/convnet_cuda$ time go run -tags='CUDA' main.go -epochs 1 2> /dev/null
Epoch 0 599 / 600 [=============================================] 99.83% 00m01s
real 26m36,597s
user 56m27,960s
sys 2m30,760s
nabil@jetson-nano:~/go/src/gorgonia.org/gorgonia/examples/convnet_cuda$ time go run main.go -epochs 1 2>
/dev/null
Epoch 0 599 / 600 [=============================================] 99.83% 00m01s
real 26m2,410s
user 55m14,600s
venv) (base) [Z22NSERV@FRL-PF1359QP analyze]$ OPENSSL_ia32cap="~0x200000200000000" openssl speed -elapsed -evp aes-128-cbc
You have chosen to measure elapsed time instead of user CPU time.
Doing aes-128-cbc for 3s on 16 size blocks: 47770686 aes-128-cbc's in 3.00s
Doing aes-128-cbc for 3s on 64 size blocks: 14469494 aes-128-cbc's in 3.00s
Doing aes-128-cbc for 3s on 256 size blocks: 3555371 aes-128-cbc's in 3.00s
Doing aes-128-cbc for 3s on 1024 size blocks: 941955 aes-128-cbc's in 3.00s
Doing aes-128-cbc for 3s on 8192 size blocks: 116602 aes-128-cbc's in 3.00s
Doing aes-128-cbc for 3s on 16384 size blocks: 59135 aes-128-cbc's in 3.00s
OpenSSL 1.1.1d FIPS 10 Sep 2019
built on: Thu Oct 3 00:00:00 2019 UTC
@blackrez
blackrez / chart.py
Last active February 18, 2020 08:08
Creation of a basic chart.
from googleapiclient.discovery import build
from google.oauth2 import service_account
SCOPES = ["https://www.googleapis.com/auth/drive", "https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/spreadsheets"]
SERVICE_ACCOUNT_FILE = 'credentials.json'
credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
service = build('sheets', 'v4', credentials=credentials)
nabilservais@penguin:~/skynet$ tinygo build -o wasm.wasm -target=wasm ./main.go
# encoding/gob
../../../usr/local/go/src/encoding/gob/type.go:39:24: Map not declared by package sync
../../../usr/local/go/src/encoding/gob/type.go:801:26: Map not declared by package sync
../../../usr/local/go/src/encoding/gob/type.go:802:26: Map not declared by package sync
../../../usr/local/go/src/encoding/gob/type.go:844:8: rt.Name undefined (type reflect.Type has no field or method Name)
../../../usr/local/go/src/encoding/gob/type.go:866:8: rt.Name undefined (type reflect.Type has no field or method Name)
../../../usr/local/go/src/encoding/gob/type.go:867:9: rt.PkgPath undefined (type reflect.Type has no field or method PkgPath)
../../../usr/local/go/src/encoding/gob/type.go:868:21: rt.Name undefined (type reflect.Type has no field or method Name)
../../../usr/local/go/src/encoding/gob/type.go:870:21: rt.PkgPath undefined (type reflect.Type has no field or method PkgPath)