Skip to content

Instantly share code, notes, and snippets.

View smoothml's full-sized avatar

Paul Harrison smoothml

View GitHub Profile
[alias]
st = status
br = branch
co = checkout
cm = commit -m
# Pull in remote changes for the current repository and all its submodules
pr = !"git pull; git submodule foreach git pull origin master"
# Switch to a branch, creating it if necessary
@smoothml
smoothml / .tmux.conf
Last active May 1, 2019 08:01
Tmux configuration
# Prefix key
unbind C-b
set -g prefix C-a
# Pnae switching
bind -n M-Left select-pane -L
bind -n M-Right select-pane -R
bind -n M-Up select-pane -U
bind -n M-Down select-pane -D
@smoothml
smoothml / .vimrc
Last active June 19, 2019 16:30
Vim config
set nocompatible " be iMproved, required
filetype off " required
" set the runtime path to include Vundle and initialize
set rtp+=~/.vim/bundle/Vundle.vim
call vundle#begin()
" let Vundle manage Vundle, required
Plugin 'VundleVim/Vundle.vim'
@smoothml
smoothml / scikit-learn-predictions-on-spark.py
Last active October 18, 2023 19:07
How to apply a Scikit Learn machine learning model at scale using Apache Spark.
from pyspark.sql import functions as F
from pyspark.sql.types import DoubleType
import pandas as pd
from sklearn.externals import joblib
def make_predictions(sc, df, feature_cols, model_path):
"""
Make predictions.
@smoothml
smoothml / tests.py
Created February 23, 2018 09:36
2018-02-23 Serverless computing tests.py
class TestClass(TestCase):
def setUp(self):
self.test_response = TEST_RESPONSE
self.test_response_reformatted = TEST_RESPONSE_REFORMATTED
def test_reformat_json(self):
print(self.test_response)
test_response_rates = self.test_response["rates"]
test_response_updated = reformat_json(self.test_response)
@smoothml
smoothml / mocked_requests_get.py
Created February 23, 2018 09:33
2018-02-23 Serverless computing mocked_requests_get.py
def mocked_requests_get(*args, **kwargs):
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
def json(self):
return self.json_data
if args[0] == "http://api.fixer.io/latest?base=GBP":
@smoothml
smoothml / reformat_json.py
Created February 23, 2018 09:30
2018-02-23 Serverless computing reformat_json.py
def reformat_json(response):
# rearrange the response to not be nested
# i.e. {"rates": {"EUR": 1.2}} -> {"EUR": 1.2}
rates = response.pop("rates")
response.update(rates)
# update response to DynamoDB's required format
response_updated = {}
for key, val in response.items():
# date and base fields are strings (DynamoDB has no date type) whereas everything else is a
@smoothml
smoothml / handle.py
Last active February 23, 2018 09:29
2018-02-23 Serverless computing handle.py
def handle(event, context):
# initialise DynamoDB client
dynamodb = boto3.client('dynamodb', 'eu-west-2')
# perform API call
response = requests.get("http://api.fixer.io/latest?base=GBP")
# handle bad responses
if response.status_code == 200:
logger.info("Request to Fixer successful")