Skip to content

Instantly share code, notes, and snippets.

Avatar
:octocat:
Githubbing

Alexey Grigorev alexeygrigorev

:octocat:
Githubbing
View GitHub Profile
View batch.py
def make_batches(seq, n):
result = []
for i in range(0, len(seq), n):
batch = seq[i:i+n]
result.append(batch)
return result
@alexeygrigorev
alexeygrigorev / progress.py
Last active May 5, 2020
A function for parallel map with tqdm
View progress.py
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
from tqdm import tqdm
num_cores = multiprocessing.cpu_count()
pool = ProcessPoolExecutor(max_workers=num_cores)
@alexeygrigorev
alexeygrigorev / get-winners.py
Last active Apr 29, 2020
Running giveaway campaigns on twitter
View get-winners.py
from glob import glob
from random import shuffle
import requests
coupon_codes = [
'mlbookcamp-1',
'mlbookcamp-2',
'mlbookcamp-3',
'mlbookcamp-4',
@alexeygrigorev
alexeygrigorev / Dockerfile
Created Feb 15, 2019
Build libwebp for aws lambda
View Dockerfile
FROM amazonlinux:2017.03
RUN yum -y install git \
python36 \
python36-pip \
python36-devel \
zip \
gcc \
gcc-c++ \
cmake \
@alexeygrigorev
alexeygrigorev / tqdm_pool.py
Created Dec 6, 2018
Track progress of ProcessPoolExecutor with tqdm
View tqdm_pool.py
from glob import glob
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
import cv2
from PIL import Image
import imagehash
from tqdm import tqdm
View create_python_files.sh
pip install grpcio-tools
wget https://github.com/tensorflow/tensorflow/archive/v1.9.0.zip -O tf-190.zip
unzip tf-190.zip && rm tf-190.zip
wget https://github.com/tensorflow/serving/archive/1.9.0.zip -O tf-serving-190.zip
unzip tf-serving-190.zip && rm tf-serving-190.zip
mv serving-1.9.0/tensorflow_serving tensorflow-1.9.0
@alexeygrigorev
alexeygrigorev / cloudwatch_logs_capturer.py
Created Jul 18, 2018
Capturing stdout and sending it to cloudwatch logs
View cloudwatch_logs_capturer.py
import sys
import time
import threading
from io import StringIO
from multiprocessing import Process
from queue import Queue
import boto3
from botocore.exceptions import ClientError
@alexeygrigorev
alexeygrigorev / mp_capture.py
Created Jul 16, 2018
Python stdout sharing between chind & parent processes
View mp_capture.py
import sys
import time
from io import StringIO
import subprocess
from multiprocessing import Process, Pipe
from threading import Thread
@alexeygrigorev
alexeygrigorev / BeanToRecordConverter.java
Created Jan 17, 2018
Use reflection to write arbitrary java beans to parquet with Avro
View BeanToRecordConverter.java
package avro;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.reflect.ReflectData;
import java.util.ArrayList;
import java.util.List;
@alexeygrigorev
alexeygrigorev / bi-kmeans.py
Last active Nov 16, 2017
Bisecting K-Means
View bi-kmeans.py
import heapq
import numpy as np
from sklearn.cluster import KMeans, MiniBatchKMeans
def sklearn_bisecting_kmeans_lineage(X, k, verbose=0):
N, _ = X.shape
labels = np.zeros(N, dtype=np.int)
lineage = np.zeros((k, N), dtype=np.int)
You can’t perform that action at this time.