austinbrian

import subprocess

# this should work in any directory, whether a git repo or not
git_dict = dict(
  x.split('=') 
  for x in 
  subprocess.check_output(
    ['git','config','--list']
  ).decode('ascii').strip().split('\n'))

austinbrian

import contextlib
import os
import tempfile


half_lambda_memory = 10**6 * (
    int(os.getenv('AWS_LAMBDA_FUNCITON_MEMORY_SIZE', '0')) / 2)


@contextlib.contextmanager

austinbrian

#Check the status:
sudo service docker status
# If there isn’t running:
sudo service docker start
# And then to auto-start after reboot:
sudo systemctl enable docker
#Don’t forget to add your ec2-user to the docker group:
sudo usermod -aG docker ec2-user
# And then reboot or just logoff/login again
$ sudo reboot

austinbrian

austinbrian

.PHONY: clean data lint requirements sync_data_to_s3 sync_data_from_s3

#################################################################################
# GLOBALS                                                                       #
#################################################################################

PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
BUCKET = <s3 bucket name>
PROFILE = <aws profile name>
PROJECT_NAME = <project name>

austinbrian

import pandas as pd
import boto3

s3 = boto3.resource('s3')

data_file = s3.Object(bucket_name='challenge-1-data-09122019', key='data/wiki_movie_plots_deduped.csv')
resp = data_file.get()
data = pd.read_csv(resp['Body'])

austinbrian

austinbrian

"""
Data access utilities
"""
from collections.abc import Mapping
import os
import boto3
import botocore.client


class Bucket(Mapping):

austinbrian

austinbrian

match (n)
unwind labels(n) as s
return distinct s, count(s)