Skip to content

Instantly share code, notes, and snippets.

View anteverse's full-sized avatar
🐍

Aurelien Didier anteverse

🐍
View GitHub Profile
@anteverse
anteverse / consumer_with_commit.py
Created April 3, 2018 08:44
A Kafka consumer that commits the consumed messages
import json
from kafka import KafkaConsumer
from kafka.structs import OffsetAndMetadata, TopicPartition
consumer = KafkaConsumer(bootstrap_servers=['0.0.0.0:9092'],
key_deserializer=lambda m: m.decode('utf8'),
value_deserializer=lambda m: json.loads(m.decode('utf8')),
auto_offset_reset="earliest",
@anteverse
anteverse / producer_with_partition.py
Created April 3, 2018 09:05
Kafka producer with Round robin partitioning
import json
from kafka import KafkaProducer, TopicPartition
from kafka.partitioner import RoundRobinPartitioner
# given that topic `topic` has at least 2 partitions
partitioner = RoundRobinPartitioner(partitions=[
TopicPartition(topic='topic', partition=0),
TopicPartition(topic='topic', partition=1)
@anteverse
anteverse / consumer_with_assignor.py
Created April 3, 2018 09:08
Kafka consumer with round robin partition assignor
import json
from kafka import KafkaConsumer
from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
from kafka.structs import OffsetAndMetadata, TopicPartition
consumer = KafkaConsumer(bootstrap_servers=['0.0.0.0:9092'],
key_deserializer=lambda m: m.decode('utf8'),
value_deserializer=lambda m: json.loads(m.decode('utf8')),
@anteverse
anteverse / producer_send_kill.py
Created April 3, 2018 09:13
Kafka producer that can turn consumers off
import json
from kafka import KafkaProducer
producer = KafkaProducer(bootstrap_servers=['0.0.0.0:9092'],
key_serializer=lambda m: m.encode('utf8'),
value_serializer=lambda m: json.dumps(m).encode('utf8'))
@anteverse
anteverse / consumed_with_kill_switch.py
Created April 3, 2018 09:16
Kafka consumer that can be killed
import json
from kafka import KafkaConsumer
from kafka.structs import OffsetAndMetadata, TopicPartition
consumer = KafkaConsumer(bootstrap_servers=['0.0.0.0:9092'],
key_deserializer=lambda m: m.decode('utf8'),
value_deserializer=lambda m: json.loads(m.decode('utf8')),
auto_offset_reset="earliest",
@anteverse
anteverse / kafka.sh
Created April 12, 2018 22:40
Run a simple Kafka server
docker run -p 2181:2181 -p 9092:9092 \
--env ADVERTISED_HOST=0.0.0.0 \
--env ADVERTISED_PORT=9092 \
--env CONSUMER_THREADS=2 \
--env NUM_PARTITIONS=2 \
--env AUTO_CREATE_TOPICS \
--env TOPICS=topic \
--env ZK_CONNECT=kafka7zookeeper:2181/root/path \
--env GROUP_ID=1 \
-d --name kafka spotify/kafkaproxy
@anteverse
anteverse / fibonacci0.py
Created July 21, 2018 19:34
First non-performant implementation
def fibonacci(n: int) -> int:
if n <= 1:
return n
return fibonacci(n-1) + fibonacci(n-2)
@anteverse
anteverse / fibonacci1.py
Last active July 21, 2018 20:01
first indexed implementation
import sys
import timeit
index = {0: 0, 1: 1}
def fibonacci(n: int) -> int:
if n in index:
return index[n]
@anteverse
anteverse / fibonacci2.py
Created July 21, 2018 20:27
almost linear version
import sys
import timeit
index = {0: 0, 1: 1}
def fibonacci(n: int, m: int = 0, l: int = 1, k: int = 2) -> int:
try:
return index[n]
@anteverse
anteverse / fibonacci3.py
Created July 21, 2018 21:53
Dividing ranks per 2
import sys
import timeit
index = {0: 0, 1: 1, 2: 1}
def fibonacci1(n: int) -> int:
try: