Skip to content

Instantly share code, notes, and snippets.

@c-w
c-w / analyze_csv.py
Last active February 5, 2022 19:06
Analyze item counts in CSV columns
from collections import Counter
from argparse import ArgumentParser, FileType
from csv import DictReader, DictWriter
from sys import stdout
parser = ArgumentParser()
parser.add_argument("infile", type=FileType("r", encoding="utf-8"))
parser.add_argument("column_name")
parser.add_argument("-o", "--outfile", type=FileType("w", encoding="utf-8"), default=stdout)
args = parser.parse_args()
@c-w
c-w / json2types.py
Last active November 8, 2021 04:22
Generating Python type annotations from JSON
import typing
def convert(it, context: str):
if isinstance(it, dict):
return typing.TypedDict(
context,
{
k: convert(v, f"{context.capitalize()}{k.capitalize()}")
for (k, v) in it.items()
@c-w
c-w / databricksParallelNotebookDriver.scala
Created January 4, 2019 20:53
Running notebooks in parallel on Azure Databricks
// define the name of the Azure Databricks notebook to run
val notebookToRun = ???
// define some way to generate a sequence of workloads to run
val jobArguments = ???
// define the number of workers per job
val workersPerJob = ???
import java.util.concurrent.Executors
@c-w
c-w / docker-entrypoint.sh
Last active April 12, 2024 04:22
Docker entrypoint that sources dotenv file secrets
#!/usr/bin/env sh
# This docker-entrypoint populates environment variables from docker secrets.
# The docker secrets are assumed to be files in dotenv syntax. The requested
# secrets should be declared in the environment variable DOTENV_SECRETS, with
# multiple secret names separated by a semi-colon.
if [ ! -d /run/secrets ]; then
exec "$@"
fi
#!/usr/bin/env python
import re
ARQ_COL = '|'
ARQ_HLINE = '-'
ARQ_HHLINE = '='