Skip to content

Instantly share code, notes, and snippets.

View ashb's full-sized avatar

Ash Berlin-Taylor ashb

View GitHub Profile
@ashb
ashb / test_ti_creation.py
Created January 5, 2022 13:09 — forked from pingzh/test_ti_creation.py
for airflow perf test for ti creation inside the dag_run verify_integrity. The test is against a database without other traffic
import time
import logging
from airflow.utils.db import create_session
from airflow.utils import timezone
from airflow.models import TaskInstance, DagRun
from airflow.models.serialized_dag import SerializedDagModel
from airflow.utils.types import DagRunType
logger = logging.getLogger(__name__)
from sqlalchemy import create_engine, Column, String
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.orm import sessionmaker, scoped_session
engine = create_engine(
'mysql://root@mysql/airflow',
echo=True
)
db_session = scoped_session(
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime(2018, 1, 1),
'email_on_failure': False,
'email_on_retry': False,
@ashb
ashb / plugins__staticfiles.py
Created September 2, 2019 13:30
Serving custom static files form Airflow Plugin
from flask import Blueprint
import os
"""
Create this file as ./plugins/staticfiles.py in your ARIFLOW_HOME directory
It will serve all files under ./static/ available under /a/static/ (The "a" comes from the ``url_prefix`` argument.)
"""
@ashb
ashb / proxychains4.rb
Last active February 4, 2019 12:31 — forked from allenhuang/proxychains4_formula.rb
Unofficial brew formula for proxychains 4
# Unofficial brew formula for proxychains 4
# Instruction:
# $ git clone https://gist.github.com/628c1d98fab731d809d883c24eedd9d4.git gist-3792521
# $ brew install --HEAD gist-3792521/proxychains4.rb
#
# The default config file will be located in /usr/local/etc/proxychains.conf
#
require 'formula'
class Proxychains4 < Formula
@ashb
ashb / git-crypt-delete
Last active January 14, 2019 17:29
git-crypt-delete
#!/bin/bash -e
set -o pipefail
case "$1" in
--list|-l)
list_only=1
;;
--dry-run|-n)
@ashb
ashb / tf-plan-diff.sh
Created April 9, 2018 16:38
Show a diff of single line JSON objects (IAM policy docs) in terraform
#!/bin/bash
#
# shellcheck disable=SC2001
set -e -o pipefail
input="$1"
[[ -n "$input" ]] || {
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowListingOfOwnState",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:ListBucket",
"Resource": "arn:aws:s3:::my-terraform-state",
"Condition": {
@ashb
ashb / hacky.sh
Created May 10, 2017 08:50
Update ACS k8s API server to run with OIDC based-auth
ensure_kube_apiserver_config() {
local dry_run="$2"
# If we want to make changes to the apiserver manifest we do it via an
# Azure CustomLinuxCommand extension
# https://github.com/Azure/custom-script-extension-linux which will run the
# command for us without having to log in. The down side to this is that it
# takes a while to "deploy" the extension, which is espeically a pain if
# the config is already correct.
#
# To speed it up and only deploy the extension if something is changed we
@ashb
ashb / lock.py
Last active May 8, 2017 10:26
Lock using Amazon SimpleDB - not needed since terraform 0.9
#!/usr/bin/env python
from __future__ import print_function
import argparse
import boto
import boto.provider
import boto.sdb
import getpass
import os
import signal
import subprocess