This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module Main where | |
import Control.Applicative (liftA2) | |
import Data.Char (toLower, toUpper) | |
import Data.Function (on) | |
import qualified Data.Map.Strict as Map | |
import Text.Parsec | |
import Text.Parsec.String (Parser) | |
data Color |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from typing import Any, TypeVar, Callable, Generator | |
from contextlib import contextmanager | |
T = TypeVar("T") | |
class BufferedOp: | |
"""Class for adding elements to a _buffer and flushing when full. | |
Note that last batch may is not usually flushed automatically as usually |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Credits to answer in https://stackoverflow.com/questions/53173927/pandas-extensive-describe-include-count-the-null-values | |
def stats(df: pd.DataFrame) -> pd.DataFrame: | |
"""Like describe() but return dataframe, with datatypes and ratio of NaN values""" | |
st = df.describe() | |
total_n = len(df) | |
st.loc["dtype"] = df.dtypes | |
st.loc["n"] = total_n | |
st.loc["nan_n"] = total_n - df.count() # number of NaNs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Using sample data from https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads | |
# See https://wiki.postgresql.org/wiki/Sample_Databases for table schema & instructions how to populate | |
# Simple script to test the idea of keeping Queue full of database rows ready for consumption, using appropriate-sized | |
# chunks of rows | |
import multiprocessing as mp | |
import queue | |
import psycopg2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import time | |
import sys | |
def exp_backoff_retry( | |
fun=None, | |
*, | |
retry_exceptions=(BaseException,), | |
max_retries=3, | |
exp_base=2.0, | |
max_seconds=300, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from contracts import contract, new_contract | |
import pytest | |
from contracts.interface import ContractNotRespected | |
import json | |
# There are probably bugs, but this should check that given | |
# argument is a valid object which can be encoded as json. | |
# | |
# It assumes the following Python structure, if type hints | |
# could be recursive: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
SELECT h.schema_name, | |
h.table_name, | |
h.id AS table_id, | |
h.associated_table_prefix, | |
((round(row_estimate.row_estimate::numeric / 1000000, 3))::text || 'M') as estimated_rows | |
FROM _timescaledb_catalog.hypertable h | |
CROSS JOIN LATERAL ( SELECT sum(cl.reltuples) AS row_estimate | |
FROM _timescaledb_catalog.chunk c | |
JOIN pg_class cl ON cl.relname = c.table_name | |
WHERE c.hypertable_id = h.id |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -eu | |
PYTHON_VERSION=3.7.5 | |
POETRY_URL=https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | |
has() { | |
type $1 > /dev/null 2>&1 | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
set -eu | |
PYTHON_VERSION=3.7.5 | |
POETRY_URL=https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | |
has() { | |
type $1 > /dev/null 2>&1 | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
mktask () { | |
name=$1 | |
cmd="${@:2}" | |
preamble="import os\nimport sys\nfrom invoke import task" | |
if [ -z $cmd ] | |
then | |
echo "mktask <name> <cmd with any args>" | |
return 0 | |
fi |
NewerOlder