bradkarels

# Spark local environment variables
export SPARK_HOME=/home/bkarels/spark/current
export SPARK_MASTER_IP=127.0.0.1
export SPARK_MASTER_PORT=7077
export SPARK_MASTER_WEBUI_PORT=8080
#SPARK_MASTER-OPTS=
export SPARK_LOCAL_DIRS=$SPARK_HOME/work
export SPARK_WORKER_CORES=2
export SPARK_WORKER_MEMORY=4G
#export SPARK_WORKER_WEBUI_PORT=8081

bradkarels

@GrabConfig(systemClassLoader=true)
@Grab(group='com.h2database', module='h2', version='1.3.176')

import java.sql.*
import groovy.sql.Sql
import org.h2.jdbcx.JdbcConnectionPool

println("More groovy...")

def sql = Sql.newInstance("jdbc:h2:things", "sa", "sa", "org.h2.Driver")  // DB files for 'things' in current directory (./hello.h2.db)

bradkarels

890  cd ~/Downloads/
891  wget http://mirror.cc.columbia.edu/pub/software/apache/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz
915  cd ~
917  tar xzf Downloads/hadoop-2.6.0.tar.gz 
920  mv hadoop-2.6.0/ hadoop/
923  cd hadoop/
924  vim ~/.bashrc
Set HADOOP_HOME and add HADOOP_HOME/bin to PATH
925  . ~/.bashrc
926  ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa

bradkarels

yarn.scheduler.capacity.maximum-am-resource-percent=0.2
yarn.scheduler.capacity.maximum-applications=10000
yarn.scheduler.capacity.node-locality-delay=40
yarn.scheduler.capacity.root.acl_administer_queue=*
yarn.scheduler.capacity.root.capacity=100
yarn.scheduler.capacity.root.ds.acl_administer_jobs=*
yarn.scheduler.capacity.root.ds.acl_submit_applications=*
yarn.scheduler.capacity.root.ds.capacity=40
yarn.scheduler.capacity.root.ds.maximum-capacity=50
yarn.scheduler.capacity.root.eng.acl_administer_jobs=*

bradkarels

yarn.scheduler.capacity.maximum-am-resource-percent=0.2
yarn.scheduler.capacity.maximum-applications=10000
yarn.scheduler.capacity.node-locality-delay=40
yarn.scheduler.capacity.root.acl_administer_queue=bkarels hdpAdmins
yarn.scheduler.capacity.root.capacity=100
yarn.scheduler.capacity.root.ds.acl_administer_jobs=dsAdmin,bkarels,nadelman dsAdmins
yarn.scheduler.capacity.root.ds.acl_submit_applications=dsAdmin,dsUser0,dsUser1 dsAdmins,mlGroup,analyticsGroup
yarn.scheduler.capacity.root.ds.capacity=40
yarn.scheduler.capacity.root.ds.maximum-capacity=50
yarn.scheduler.capacity.root.eng.acl_administer_jobs=bkarels hdpAdmins

bradkarels

import java.io.ByteArrayOutputStream
import java.io.ObjectOutputStream
import java.io.Serializable
import com.twitter.chill.{Input, Output, ScalaKryoInstantiator}

class Person extends Serializable {
  var name: String = ""

  def this(name:String) {
    this()

bradkarels

// scala> :cp lib/chill_2.10-0.5.2.jar

//bkarels@ahimsa:~/spark/current$ ./bin/spark-shell --master local[*] --jars lib/mongo-java-driver-3.0.0.jar,lib/mongo-hadoop-core-1.3.2.jar,lib/chill_2.10-0.5.2.jar

import com.esotericsoftware.kryo.io.{Input, Output}
import com.twitter.chill.ScalaKryoInstantiator
import java.io.ByteArrayOutputStream
class Person(val name:String) extends Serializable
val p0:Person = new Person("p0")
val p1:Person = new Person("p1")

bradkarels

sudo tcpdump -nn -i eth0 port 6667

bradkarels

import java.nio.file.Files
import java.nio.charset.Charset
import java.nio.charset.StandardCharsets
import java.nio.file.Paths
import java.nio.file.StandardOpenOption
import collection.JavaConverters._

val utf8:Charset = StandardCharsets.UTF_8

Files.write(Paths.get("foo.txt"), "foo".getBytes(utf8))

bradkarels

val key0 = ("0","0")
val key1 = ("1","0")
val key2 = ("2","0")
val key3 = ("3","0")
 
val core:Map[(String,String),Option[String]] = Map(key0 -> Some("a"), key1 -> Some("b"), key2 -> Some("c"))
val overlay:Map[(String,String),Option[String]] = Map(key2 -> Some("y"), key3 -> Some("z"))
 
//val expected = Map(key0 -> Some("a"), key1 -> Some("b"), key2 -> Some("y"), key3 -> Some("z"))