thekensta

# Extracting date components from a Date column in Pandas using IPython
# Converting to DatetimeIndex is 100x faster than using DataFrame.apply() 

import pandas as pd
dates = pd.DataFrame({"Date": pd.date_range(start="1970-01-01", end="2037-12-31")})

print(dates.head())
#         Date
# 0 1970-01-01
# 1 1970-01-02

thekensta

##
## Wrap forecast auto.arima(..) and forecast(..) into a data.frame 
## Embeds the forecast into the data.frame
##
## Allow passing an EndDate so that the forecast can start mid-actuals 
## (helps with visualization and exaplantion) 
##
## Usage:
## Forecast.df <- AutoArimaForecast(Monthly.df, # DataFrame with 
##                                  H = 6,      # Predict 6 months forward   

thekensta

import pandas as pd
import numpy as np

#  
td = pd.DataFrame({'Date': pd.date_range('2014-01-01', '2015-12-31')})
td['Timedelta'] = td['Date'].max() - td['Date']

td.dtypes
# Date          datetime64[ns]
# Timedelta    timedelta64[ns]

thekensta

# Quick summary to access stuff in R from ipython
# Useful link but summary somehwat buried
# http://rpy.sourceforge.net/rpy2/doc-2.4/html/interactive.html

import numpy as np

%load_ext rpy2.ipython

# %R [-i INPUT] [-o OUTPUT] [-n] [-w WIDTH] [-h HEIGHT] [-p POINTSIZE]
# [-b BG] [–noisolation] [-u {px,in,cm,mm}] [-r RES] [code [code ...]]

thekensta

-- Reference
-- https://technet.microsoft.com/en-us/library/bb522495(v=sql.105).aspx

-- TODO: add more detail, this is syntax reference for me 

Select fname, food, sum(total)
From lateral(
    values
    ('Bob', 'Pies', 3), 
    ('Charlie', 'Pies', 1), 

thekensta

# Quick reminder of least squares calculations in python

import numpy as np

def least_sq_numpy(x, y):
  """Calculate y = mx + c from x, y returning m, c using numpy."""
  A = np.vstack([x, np.ones(x.size)]).T
  fit = np.linalg.lstsq(A, y)
  return fit[0]

thekensta

# Ipython code using SVD to extract components of an image

%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib.cm as cmap
import numpy as np
from scipy import ndimage

# Any image file here, this is colourso convert to greyscale 
DOG_IMAGE_FILE = "dog2.jpg"

thekensta

# Storing basic operations here, as I tend to forget them!
# Fill as required (or fill as forgotten?? :-)

# Repeat and Tile

# Repeat copies by element and flattens
# Tile copies sequences and preserves shape
a = np.array([1, 2, 3])

print(np.tile(a, 2))

thekensta

# For a local environment
# Install hadoop and apache-spark via homebrew

# Apache Spark conf file 
# libexec/conf/spark-defaults.conf
# Make the AWS jars available to Spark
spark.executor.extraClassPath      /usr/local/Cellar/hadoop/2.7.1/libexec/share/hadoop/tools/lib/aws-java-sdk-1.7.4.jar:/usr/local/Cellar/hadoop/2.7.1/libexec/share/hadoop/tools/lib/hadoop-aws-2.7.1.jar
spark.driver.extraClassPath      /usr/local/Cellar/hadoop/2.7.1/libexec/share/hadoop/tools/lib/aws-java-sdk-1.7.4.jar:/usr/local/Cellar/hadoop/2.7.1/libexec/share/hadoop/tools/lib/hadoop-aws-2.7.1.jar

# Add file 

thekensta

# submit with spark-submit hello_pyspark.py
# Spark 1.5.1 

from pyspark import SparkContext, SparkConf
from pyspark.sql import SQLContext

conf = SparkConf().setAppName("showMeTheSchema").setMaster("local")
sc = SparkContext(conf=conf)

sqlContext = SQLContext(sc)