Skip to content

Instantly share code, notes, and snippets.

@ripiuk
ripiuk / s3_async_dowload_folder.py
Last active August 7, 2023 17:23
A tool for downloading a list of files from s3 using asyncio
import os
import time
import asyncio
import aiobotocore
import aiofiles
AWS_ACCESS_KEY_ID = '' # aws access key
AWS_SECRET_ACCESS_KEY = '' # aws secret key
READ_TIMEOUT = 500
@ripiuk
ripiuk / sql_generate_and_write_random_data.py
Last active December 20, 2017 14:53
Generate data and write it into sql table
import pymysql
import random
SCHEMA = 'schema_name'
TABLE_NAME = 'table_name'
TABLE_FIELDS = [
'currency',
'language',
]
USER = 'root'
@ripiuk
ripiuk / async_upload.py
Created December 20, 2017 15:08
Upload data to s3 in tornado asynchronously
import boto3
from tornado.gen import coroutine, Task
from tornado.httpclient import HTTPError, AsyncHTTPClient, HTTPRequest
_http_client = AsyncHTTPClient()
@coroutine
def upload_to_s3(bucket: str, region: str, data, key: str,
aws_access_key_id: str, aws_secret_access_key: str) -> str:
@ripiuk
ripiuk / extract_tars.py
Created December 20, 2017 16:02
Extract all tars in the directory
import os
import tarfile
PATH_TO_DIR_WITH_TARS = '/home/username/sub_folder/'
files = [file for file in os.listdir(PATH_TO_DIR_WITH_TARS) if
os.path.isfile(os.path.join(PATH_TO_DIR_WITH_TARS, file))]
for file in files:
file_name, file_extension = os.path.splitext(file)
@ripiuk
ripiuk / simple_nsq_publisher.py
Last active March 29, 2018 11:25
A simple nsq publisher using requests
import requests
import json
url = 'http://localhost:4151'
topic = 'is-hotel-active'
url = '{url}/pub?topic={topic}'.format(url=url, topic=topic)
rows = [{'hotel_id': '81', 'is_hotel_active': '0'},
{'hotel_id': '23', 'is_hotel_active': '1'},
{'hotel_id': '82', 'is_hotel_active': '1'}]
@ripiuk
ripiuk / simple_nsq_reader.py
Created March 29, 2018 11:27
A simple nsq reader
import nsq
lookup_http_addresses = ['http://127.0.0.1:4161']
topic = 'hotel-active'
chanel = 'hotel'
def handler(message):
print(message.body)
return True
@ripiuk
ripiuk / simple_dictionary_validation.py
Last active March 29, 2018 11:45
A simple dictionary structure validation
import trafaret as t
from trafaret import Dict
some_template = {
'api': {
'status': 'ok',
'locale': 'Some str',
'error': 'None',
'data': {
'result': {
@ripiuk
ripiuk / find_diff.py
Created April 1, 2018 09:31
To find the difference between two files
FIRST_FILE_PATH = 'simple_1.csv'
SECOND_FILE_PATH = 'simple_2.csv'
OUTPUT_FILE_PATH = 'file_diff.csv'
with open(FIRST_FILE_PATH, 'r') as first_file, open(SECOND_FILE_PATH, 'r') as second_file:
first_reader = first_file.readlines()
second_reader = second_file.readlines()
with open(OUTPUT_FILE_PATH, 'w+') as output:
for line in first_reader:
@ripiuk
ripiuk / redis_cache.py
Created April 1, 2018 09:46
An example of an interface for using redis as a cache
import redis
import logging
def create_logger(logger_name):
logger = logging.getLogger(logger_name)
return logger
class BatchDateCache:
@ripiuk
ripiuk / sort_by_vowels.py
Created April 1, 2018 13:13
Sort words by the number of vowels
data = ['some str', 'longer str', 'the biggest string', 'small one']
vowels = 'eyuioa'
data.sort(key=lambda x: sum(ch in vowels for ch in x))