mattf

from pyspark.sql.types import *
from pyspark.sql.functions import mean

with open("data.csv", "w") as fp:
    fp.write("""
a,
b,1
c, 
d,1
""")

mattf

mattf

def numDecodings(s: str) -> int:
    if(not s):return 1
    elif(len(s)==1):
        if(s=="0"):return 0
        else:return 1
            
    if(s[0]=="0"):return 0
    prev=1
    cur=0 if s[1]=="0" else 1
    if(int(s[:2])<=26): cur+=1

mattf

import random
def inside(p):
  x, y = random.random(), random.random()
  return x*x + y*y < 1

count = spark.sparkContext.parallelize(xrange(0, 100000000)).filter(inside).count()
print "Pi is roughly %f" % (4.0 * count / 100000000)

mattf

: Start up an OpenShift Origin cluster locally
oc cluster up

: Install Oshinko components
oc create -f http://radanalytics.io/resources.yaml

: Install Apache Kafka for OpenShift components
oc create -f https://raw.githubusercontent.com/mattf/openshift-kafka/master/resources.yaml

: Launch an Apache Kafka instance, it is 1-pod, not replicated, not persistent, just-enough kafka

mattf

kind: Template
apiVersion: v1
template: oshinko
metadata:
  name: oshinko
objects:

- kind: ServiceAccount
  apiVersion: v1
  metadata:

mattf

: Start up an OpenShift Origin cluster locally
oc cluster up

: Launch Oshinko in the current project
oc new-app -f http://goo.gl/ZU02P4
: Authorize Oshinko to write to the OpenShift API
oc policy add-role-to-user edit -z oshinko

: Anayltical application demo - counting words by Joyce
: joyce writes text to a socket

mattf

kind: Template
apiVersion: v1
template: oshinko
metadata:
  name: oshinko
objects:

- kind: Service
  apiVersion: v1
  metadata:

mattf

#!/bin/sh

# mv /bin/hostname /bin/hostname.real
# install $0 as /bin/hostname (a+rx)

TARGET="kubelet"

IFACE=eth0

PARENT=$(ps -oucmd= $PPID)

mattf

create node group
 - master
 - namenode, oozie, resourcemanager, historyserver

create node group
 - worker
 - datanode, nodemanager

create cluster
 - master: 1