Skip to content

Instantly share code, notes, and snippets.

Avatar

Adrian Macal amacal

  • FlixBus
  • Munich, Germany
View GitHub Profile
View intercepting-ecs-logs.py
from time import sleep
from boto3 import client
from botocore.config import Config
config = Config(
retries = {
'max_attempts': 10,
'mode': 'adaptive'
}
)
View cprofiler-ecs-after.txt
296089294 function calls (290809304 primitive calls) in 832.918 seconds
Ordered by: internal time
ncalls tottime percall cumtime percall filename:lineno(function)
19706 74.185 0.004 289.294 0.015 json.py:45(process)
68618779/68617586 60.549 0.000 60.586 0.000 {built-in method builtins.len}
16765202 56.732 0.000 101.660 0.000 pipes.py:52(read)
554796 50.969 0.000 50.969 0.000 {method 'read' of '_ssl._SSLSocket' objects}
560857 49.093 0.000 49.093 0.000 {method 'write' of '_ssl._SSLSocket' objects}
View cprofiler-local-after.txt
294808988 function calls (289527877 primitive calls) in 1308.849 seconds
Ordered by: internal time
ncalls tottime percall cumtime percall filename:lineno(function)
560862 780.566 0.001 780.566 0.001 {method 'write' of '_ssl._SSLSocket' objects}
423504 260.731 0.001 260.731 0.001 {method 'read' of '_ssl._SSLSocket' objects}
11175660 39.737 0.000 39.737 0.000 {orjson.loads}
19706 27.423 0.001 113.160 0.006 json.py:45(process)
93 18.475 0.199 18.475 0.199 {method 'connect' of '_socket.socket' objects}
View cprofiler-ecs-before.txt
463723114 function calls (458443034 primitive calls) in 1209.540 seconds
Ordered by: internal time
ncalls tottime percall cumtime percall filename:lineno(function)
90969949/90968756 82.382 0.000 82.420 0.000 {built-in method builtins.len}
11175664 81.461 0.000 201.197 0.000 decoder.py:332(decode)
19706 77.013 0.004 670.440 0.034 json.py:46(process)
11175664 71.578 0.000 412.260 0.000 __init__.py:299(loads)
11175664 60.933 0.000 60.933 0.000 decoder.py:343(raw_decode)
View cprofiler-local-before.txt
462476698 function calls (457195595 primitive calls) in 1256.186 seconds
Ordered by: internal time
ncalls tottime percall cumtime percall filename:lineno(function)
560863 762.970 0.001 762.970 0.001 {method 'write' of '_ssl._SSLSocket' objects}
426597 205.799 0.000 205.799 0.000 {method 'read' of '_ssl._SSLSocket' objects}
11175669 36.491 0.000 36.491 0.000 decoder.py:343(raw_decode)
19706 22.167 0.001 161.075 0.008 json.py:46(process)
11175669 14.730 0.000 61.480 0.000 decoder.py:332(decode)
View sorting-10g-pipeline.py
def worker_sort(name, tag, bucket, input, output):
pipeline = Pipeline(name=name, steps=[
S3Download(),
NDJsonChunk(chunksize=1024*1024),
ForEachChunk(chunksize=512*1024*1024, steps=lambda index: [
NDJsonIndex(extract=lambda row: row[tag]),
QuickSort(key=lambda row: row.key),
NDJsonFlush(),
S3Upload(bucket=bucket, key=f'{output}.tmp/{index}', chunksize=128*1024*1024)
]),
View sql-date-ranges-accumulative.sql
with source as
(
select * from ecds_db.playground.time11M
),
timeline as
(
select id, start_at as happened_at, 1 as balance
from source
union all
select id, end_at as happened_at, -1 as balance
View sql-date-ranges-naive.sql
with source as
(
select * from ecds_db.playground.time11M
),
timeline as
(
select start_at as moment
from source
union
select end_at as moment
View python-asyncio-old.py
from multiprocessing import Pool, Queue, Manager
manager = Manager()
ftpQueue = manager.Queue()
jsonQueue = manager.Queue()
with Pool(20) as pool:
pool.starmap(master, [(name, ..., ftpQueue, jsonQueue) for name in fetch_names()])
View python-asyncio-all.py
from boto3 import client
from botocore.exceptions import ClientError
from ftplib import FTP
from gzip import GzipFile
from os.path import splitext
from os import getenv
from time import time as now
from hashlib import md5, sha1
from queue import Queue
from lxml.etree import iterparse