Skip to content

Instantly share code, notes, and snippets.

@michalc
michalc / decrypt-ses-emails-in-s3.py
Last active Feb 15, 2022
Decrypt KMS-encrypted SES emails in an S3 bucket
View decrypt-ses-emails-in-s3.py
import base64
import json
import boto3
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
s3 = boto3.resource('s3')
bucket = s3.Bucket('my-bucket')
kms_client = boto3.client('kms')
@michalc
michalc / make_8gb_legacy_zip.py
Created Jan 3, 2022
Generating a Zip 2.0 file that's (just under) 8GiB
View make_8gb_legacy_zip.py
# Often it's claimed that a Zip 2.0 file cannot be bigger than 4GiB
# Here's how to make one that's just under 8GiB
from datetime import datetime
from stream_zip import stream_zip, ZIP_32
now = datetime.now()
perms = 0o600
def files():
for i in range(0, 0xffff):
View pipelines-2.0-pipeline.py
class MyPipeline(_PipelineV2):
# Everything is a method so nothing happens on import time for flexibility (although possibly
# does a bit of discovery magic... need to think about that...)
# Everything is a _static_ method: nothing on self since things are run on different bits of hardware,
# and gets any run-time dependencies injected in
#
# _PipelineV2 won't actually have any code: other parts of the system will interrogate its
# subclasses as needed. For example
# - Code in Data Flow would construct a DAG
# - The test harness would the run this and upstream pipelines synchronously
@michalc
michalc / libcrypto-decrypt-aes-ctr-little-endian.py
Last active Sep 13, 2021
Use libcrypto (OpenSSL) directly from Python with ctypes without compiling anything: AES decrypt with a little endian CTR counter
View libcrypto-decrypt-aes-ctr-little-endian.py
from contextlib import contextmanager
from ctypes import POINTER, cdll, c_char_p, c_void_p, c_int, create_string_buffer, byref
from sys import platform
# Uses a _little_ endian CTR counter, which OpenSSL doesn't directly support.
# Could be used to decrypt AES-encrypted ZIP files
def decrypt_aes_256_ctr_little_endian(
key, ciphertext_chunks,
get_libcrypto=lambda: cdll.LoadLibrary({'linux': 'libcrypto.so', 'darwin': 'libcrypto.dylib'}[platform])
):
@michalc
michalc / postman-hawk.js
Last active Sep 21, 2021
Postman pre-request script for Hawk authentication in custom header
View postman-hawk.js
/*****************************************************************************/
const hawkId = pm.variables.get('hawk_id');
const hawkKey = pm.variables.get('hawk_key');
const hawkHeader = pm.variables.get('hawk_header') || 'authorization';
/*****************************************************************************/
const timestamp = parseInt(new Date().getTime() / 1000);
const nonce = CryptoJS.enc.Base64.stringify(CryptoJS.lib.WordArray.random(6));
@michalc
michalc / sqlite.py
Last active May 14, 2022
Use libsqlite3 directly from Python with ctypes: without using the built-in sqlite3 Python package, and without compiling anything
View sqlite.py
# From https://stackoverflow.com/a/68876046/1319998, which is itself inspired by https://stackoverflow.com/a/68814418/1319998
from contextlib import contextmanager
from collections import namedtuple
from ctypes import cdll, byref, string_at, c_char_p, c_int, c_double, c_int64, c_void_p
from ctypes.util import find_library
from sys import platform
def query(db_file, sql, params=()):
@michalc
michalc / spec.json
Last active Jul 4, 2021
World Bank + Vega-lite
View spec.json
{
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
"width": 630,
"height": 630,
"data": {
"values": [
{
"type": "Feature",
"id": 0,
"geometry": {
@michalc
michalc / table_csv_view.py
Created Mar 7, 2019
Django + gevent + psycopg2 download whole PostgreSQL table as CSV
View table_csv_view.py
import csv
import logging
import gevent
from psycopg2 import (
connect,
sql,
)
from django.conf import (
@michalc
michalc / getaddrinfo_via_libc.py
Last active Dec 2, 2018
Own implementation of getaddrinfo by calling libc directly from Python + ctypes, supporting IPv4 and IPv6 (experimental)
View getaddrinfo_via_libc.py
from ctypes import (
byref,
c_char_p, c_uint32, c_uint16, c_ubyte,
CDLL,
POINTER,
pointer,
Structure, Union,
sizeof,
)
import platform
@michalc
michalc / path_lock_local_read_write_ancestor.py
Last active Nov 25, 2018
Path lock using a read/write/ancestor lock on each ancestor path
View path_lock_local_read_write_ancestor.py
import asyncio
import contextlib
import weakref
from fifolock import FifoLock
class ReadAncestor(asyncio.Future):
@staticmethod
def is_compatible(holds):
return not holds[Write]