Skip to content

Instantly share code, notes, and snippets.

Avatar

Timothy Renner timothyrenner

View GitHub Profile
@timothyrenner
timothyrenner / bigfoot_streamlit_app.py
Last active Jun 23, 2020
Bigfoot Sightings Streamlit App
View bigfoot_streamlit_app.py
import streamlit as st
import pandas as pd
import altair as alt
import pydeck as pdk
import os
from dateutil.parser import parse
try:
from dotenv import load_dotenv, find_dotenv
@timothyrenner
timothyrenner / gfl_with_h3.py
Created Dec 11, 2019
Graph Fused Lasso + H3
View gfl_with_h3.py
from h3 import h3
from pygfl.easy import solve_gfl
def build_neighbor_edges(hexids):
# Hash the hexid to the position so we can easily look
# up where the original hexid position is in the array.
hexid_to_position = {h:ii for ii,h in enumerate(hexids)}
edges = []
for h in hexids:
for n in h3.k_ring(h,1):
View pyspark_pandas_udf_call.py
data_frame.withColumn(
"prediction",
predict_pandas_udf(col("feature1"), col("feature2"), ...)
)
View pyspark_pandas_udf_creation.py
import pandas as pd
from pyspark.sql.functions import pandas_udf
from pyspark.sql.types import DoubleType
@pandas_udf(returnType=DoubleType())
def predict_pandas_udf(*features):
""" Executes the prediction using numpy arrays.
View pyspark_partition_call.py
my_data.rdd.mapPartitions(predict_partition).toDF()
View pyspark_partition_definition.py
import pandas as pd
# We'll need this handy list more than once. It enforces the
# column order required by the model.
FEATURES = ["feature1", "feature2", "feature3", ...]
def predict_partition(rows):
""" Calls a vectorized prediction by loading the partition into memory.
View pyspark_udf_call.py
my_df.withColumn(
"predicted_score",
predict_udf(col("feature1"), col("feature2"), ...)
)
View pyspark_udf_creation.py
from pyspark.sql.functions import udf
from pyspark.sql.types import DoubleType
predict_udf = udf(predict, DoubleType())
View pyspark_udf_definition.py
import numpy as np
def predict(*features):
""" Performs a prediction on the features.
Parameters
----------
features : List[float]
The feature values the model needs to make a prediction.
@timothyrenner
timothyrenner / average_agreement.py
Created Sep 13, 2018
Average Agreement, Attempt 3
View average_agreement.py
def average_agreement(list1, list2, max_depth):
# Empty lists evaluate to false.
if (not list1) or (not list2):
return 0.0
### NEW CODE ###
# Truncate the depth
max_list_len = max(len(list1), len(list2))
max_depth = min(max_depth, max_list_len)