Skip to content

Instantly share code, notes, and snippets.

View ashb's full-sized avatar

Ash Berlin-Taylor ashb

View GitHub Profile
@ashb
ashb / test_ti_creation.py
Created January 5, 2022 13:09 — forked from pingzh/test_ti_creation.py
for airflow perf test for ti creation inside the dag_run verify_integrity. The test is against a database without other traffic
import time
import logging
from airflow.utils.db import create_session
from airflow.utils import timezone
from airflow.models import TaskInstance, DagRun
from airflow.models.serialized_dag import SerializedDagModel
from airflow.utils.types import DagRunType
logger = logging.getLogger(__name__)
from sqlalchemy import create_engine, Column, String
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.orm import sessionmaker, scoped_session
engine = create_engine(
'mysql://root@mysql/airflow',
echo=True
)
db_session = scoped_session(
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime(2018, 1, 1),
'email_on_failure': False,
'email_on_retry': False,
@ashb
ashb / plugins__staticfiles.py
Created September 2, 2019 13:30
Serving custom static files form Airflow Plugin
from flask import Blueprint
import os
"""
Create this file as ./plugins/staticfiles.py in your ARIFLOW_HOME directory
It will serve all files under ./static/ available under /a/static/ (The "a" comes from the ``url_prefix`` argument.)
"""
@ashb
ashb / proxychains4.rb
Last active February 4, 2019 12:31 — forked from allenhuang/proxychains4_formula.rb
Unofficial brew formula for proxychains 4
# Unofficial brew formula for proxychains 4
# Instruction:
# $ git clone https://gist.github.com/628c1d98fab731d809d883c24eedd9d4.git gist-3792521
# $ brew install --HEAD gist-3792521/proxychains4.rb
#
# The default config file will be located in /usr/local/etc/proxychains.conf
#
require 'formula'
class Proxychains4 < Formula
@ashb
ashb / tf-plan-diff.sh
Created April 9, 2018 16:38
Show a diff of single line JSON objects (IAM policy docs) in terraform
#!/bin/bash
#
# shellcheck disable=SC2001
set -e -o pipefail
input="$1"
[[ -n "$input" ]] || {
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowListingOfOwnState",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:ListBucket",
"Resource": "arn:aws:s3:::my-terraform-state",
"Condition": {
@ashb
ashb / hacky.sh
Created May 10, 2017 08:50
Update ACS k8s API server to run with OIDC based-auth
ensure_kube_apiserver_config() {
local dry_run="$2"
# If we want to make changes to the apiserver manifest we do it via an
# Azure CustomLinuxCommand extension
# https://github.com/Azure/custom-script-extension-linux which will run the
# command for us without having to log in. The down side to this is that it
# takes a while to "deploy" the extension, which is espeically a pain if
# the config is already correct.
#
# To speed it up and only deploy the extension if something is changed we
@ashb
ashb / Dockerfile
Last active November 2, 2016 16:39
FROM alpine
RUN ls /etc/profile.d /usr/local/bin /usr
COPY resources/etc/ /etc
COPY resources/usr/ /usr
RUN ls /etc/profile.d/ /usr/local/bin /usr
@ashb
ashb / gist:cce8edd491de4de9b36017e50cd0c542
Last active April 13, 2016 11:19 — forked from gtmtech/gist:9af0453db143c8a18e4fbd4460e00a91
Realtime dumping of HTTP traffic headers
#!/usr/bin/perl
my $port = shift or "80";
open (FH, "tshark -d tcp.port==$port,http -V -Y 'http.request || http.response' port $port |") or die "Could not run tshark";
%requests=();
while (<FH>) {
if (/^[^\s]/) {
# Headers