Skip to content

Instantly share code, notes, and snippets.

@mattfysh
mattfysh / delta_log.py
Created April 4, 2024 00:58
read delta table as a stream
import os
import json
from typing import Any
SKIP_OPERATIONS = ["OPTIMIZE", "VACUUM START", "VACUUM END"]
def is_subset(a: dict[str, Any], b: dict[str, Any]):
sub = a.items()
full = b.items()
@mattfysh
mattfysh / frankenstein.js
Created March 18, 2024 12:09
declaring drizzle types for a table defined elsewhere
import {
SQLiteTableWithColumns,
SQLiteTextBuilderInitial,
SQLiteTextJsonBuilderInitial,
SQLiteIntegerBuilderInitial,
SQLiteColumnBuilderBase,
} from 'drizzle-orm/sqlite-core'
import type { BuildColumns, HasDefault, NotNull } from 'drizzle-orm'
@mattfysh
mattfysh / frankenstein.js
Created March 14, 2024 04:06
using push-generated statements in `generate:sqlite`
import { get, patch, wrapImports } from './rewire'
const push = get('logSuggestionsAndReturn2')
const move = patch('_moveDataStatements', function (tableName, json) {
// set dataLoss=false to generate copy statements
return move.call(this, tableName, json, false)
})
wrapImports('init_migrate', {
@mattfysh
mattfysh / kafka.py
Created March 13, 2024 01:04
bytewax KafkaBuilder
from typing import Dict, List
from bytewax import operators as op
from bytewax.dataflow import Dataflow, Stream
from bytewax.connectors.kafka import operators as kop
from bytewax.connectors.kafka.serde import SchemaDeserializer
class KeyDeserializer(SchemaDeserializer[bytes, str]):
def de(self, data):
return data.decode()
const mf = new Miniflare({
script: `
export default {
async fetch(request, env, ctx) {
let id = env.DOO.idFromName('foo')
return await env.DOO.get(id).fetch(request)
}
}
const SECONDS = 1000
@mattfysh
mattfysh / enrich_join.py
Last active February 23, 2024 04:48
Custom bytewax operators
from copy import deepcopy
from typing import Optional, Any, TypeVar, Tuple
from bytewax.dataflow import operator
from bytewax import operators as op
from bytewax.operators import KeyedStream
RETAIN = False
DISCARD = True
E = TypeVar("E")
@mattfysh
mattfysh / policy.json
Created December 12, 2023 22:44
Danger, Will Robinson!
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"iam:*",
"eks:*",
"ec2:*",
"autoscaling:*",
@mattfysh
mattfysh / pulumi.ts
Last active February 8, 2024 05:41
Pulumi Kubernetes - deployment preview error
import * as k8s from '@pulumi/kubernetes'
const provider = new k8s.Provider('k8s-provider', {
context: 'docker-desktop',
namespace: 'testns',
})
const ns = new k8s.core.v1.Namespace(
'ns',
{ metadata: { name: 'testns' } },
@mattfysh
mattfysh / index.ts
Last active December 17, 2023 03:13
Pulumi deploy k8s redpanda + console
import * as k8s from '@pulumi/kubernetes'
import dedent from 'dedent'
new k8s.apps.v1.Deployment('redpanda', {
metadata: {
labels: {
appClass: 'redpanda',
},
},
spec: {
@mattfysh
mattfysh / genzod.ts
Last active December 8, 2023 09:32
import fs from 'node:fs/promises'
import path from 'node:path'
import rimraf from 'rimraf'
import { glob } from 'glob'
import { generate } from 'ts-to-zod'
import ts from 'typescript'
import { v4 as uuidv4 } from 'uuid'
const OUTDIR = 'packages/types/gen'
const { factory } = ts