seanickle

import sys
import base64
import datetime
import hashlib
import hmac
import requests
import json
import urlparse

# This file is derived from the example here

seanickle

import traceback
from multiprocessing import Process, Pipe

def parallel_async_invoke(payloads, work_func):
    # => https://aws.amazon.com/blogs/compute/parallel-processing-in-python-with-aws-lambda/
    # create a list to keep all processes
    processes = []

seanickle

# Nice technique from https://srome.github.io/Covariate-Shift,-i.e.-Why-Prediction-Quality-Can-Degrade-In-Production-and-How-To-Fix-It/ 
# ... put two histograms on same plot ...

def produce_overlayed_hists_for_col_dfs(col, dfs):
    fig = plt.figure(figsize=(12,12))

 ax = fig.add_subplot(121)

seanickle

Source: Copied the below from stackoverflow

With SNI

If the remote server is using SNI (that is, sharing multiple SSL hosts on a single IP address) you will need to send the correct hostname in order to get the right certificate.

openssl s_client -showcerts -servername www.example.com -connect www.example.com:443 

seanickle

Quick back of the manilla envelope for comparing first level of two dictionaries

• Below, diff has just the left, right keys for which d1 and d2 do not agree
• Inserting NOKEY when left or right do not have that value.

import hashlib
import json
hsh = lambda x: hashlib.sha512(json.dumps(x)).hexdigest()[:5]
make_sorted_ordered = lambda d: OrderedDict(sorted(d.items(), key=lambda x:x[0]))

def delta(d1, d2):

seanickle

import requests
import time
import json
from functools import partial
from requests.exceptions import ConnectionError, Timeout

seanickle

 <img src="https://blahblahblah/blah.png" alt="alt text" width="654" height="208">
 

seanickle

.

import math
def get_partitions(vec, slice_size):
    assert slice_size > 0
    assert isinstance(vec, list)
    num_slices = int(math.ceil(len(vec)/slice_size))
    size_remainder = len(vec) - num_slices*slice_size
    slices = [vec[k*slice_size:k*slice_size+slice_size] for k in range(num_slices)]

seanickle

Copy from s3 to local

aws --profile blahmyprofile s3 cp  s3://mybucket/test2.txt .

seanickle

Normalize unicode with handy vanilla python

• note this only works for unicode and crashes if s is not unicode.

In [19]: s = u'Cañon City, cool'

In [20]: s.encode('ascii', 'replace')
Out[20]: 'Ca?on City, cool'

In [21]: s.encode('ascii', 'ignore')
Out[21]: 'Caon City, cool'