Skip to content

Instantly share code, notes, and snippets.

Nate Kupp natekupp

Block or report user

Report or block natekupp

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
from dagster import lambda_solid, pipeline
def hello_world():
return "hello, world"
def basic_pipeline():
import datetime
import os
from airflow.contrib.operators import dataproc_operator
from airflow.contrib.operators.bigquery_operator import BigQueryOperator
from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator
from airflow.models import DAG
from airflow.utils import trigger_rule
natekupp /
Last active May 16, 2019
Minimal repro example of Dask exception
import dask
import dask.distributed
from tornado import gen
PUBSUB_NAME = 'test'
class Task:
def __init__(self, key, num, upstream_tasks):
class DataProcSparkOperator(BaseOperator):
Start a Cloud DataProc cluster, run a Spark job, then shut down the Spark cluster.
template_fields = ['arguments']
ui_color = '#0273d4'
def __init__(
natekupp /
Created Sep 26, 2015
impyla+multiprocessing crash example
import multiprocessing
import impala.dbapi
class Impala(obect):
def __init__(self, host, database, port, username, password, lock=None):
self.conn = impala.dbapi.connect(host=host,
View gist:2954743
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Terminal visualization of 2D numpy arrays
# Copyright (c) 2009 Nicolas P. Rougier
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
natekupp / gist:2628399
Created May 7, 2012
running node.js and simple HTTP server
View gist:2628399
# start http server
self.p_http = subprocess.Popen(
["python", "-m", "SimpleHTTPServer"],
preexec_fn = sigint_replace,
cwd = self.resource_dir
# start node.js server
self.p_node = subprocess.Popen(
natekupp / gist:2090168
Created Mar 19, 2012
Installation for py2cairo on OSX Lion 10.7
View gist:2090168
python waf clean
export PYTHONPATH=/Library/Frameworks/Python.framework/Versions/2.7/
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7/:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/Library/Frameworks/Python.framework/Versions/2.7/lib:$LD_LIBRARY_PATH
export LINKFLAGS='-search_dylibs_first -L /Library/Frameworks/Python.framework/Versions/2.7/lib/'
export ARCHFLAGS='-arch x86_64'
export CC=/usr/bin/gcc-4.2
export PKG_CONFIG_PATH=/usr/local/Cellar/cairo/1.10.2/lib/pkgconfig/
python waf configure --prefix=$PYTHONPATH
natekupp / gist:2047390
Created Mar 15, 2012
FFX Performance Test
View gist:2047390
def buildmodel():
EPS = 0.001
data = pandas.read_csv('iris.csv')
xtrain = data.ix[:50,0:2]
xtest = data.ix[51:100,0:2]
ytrain = data.ix[:50,2]
ytest = data.ix[51:100,2]
# Use pandas.DataFrame
models =, ytrain, xtest, ytest)
View gist:1763661
class BTreeNode(object):
"""A B-Tree Node.
leaf : boolean, determines whether this node is a leaf.
keys : list, a list of keys internal to this node
c : list, a list of children of this node
def __init__(self, leaf=False):
You can’t perform that action at this time.