shivaram

package org.apache.spark.scheduler

import scala.util.Random
import org.apache.spark.storage.BlockManagerId
import org.apache.spark.util.collection.{Utils => CollectionUtils}

object TopkBench extends testing.Benchmark {

  val toTake = sys.props("toTake").toInt
  val numHosts = 1000

shivaram

#!/bin/bash
# Shows Spark PR diff using cdiff https://github.com/ymattw/cdiff

if [[ $# -ne 1 ]];
then
  echo "Usage spark-diff <pr_num>"
  exit 1
fi

command -v cdiff >/dev/null 2>&1 || { echo >&2 "Install cdiff using pip."; exit 1; }

shivaram

#!/bin/bash

FWDIR="$(cd "`dirname "$0"`"; pwd)"
pushd $FWDIR >/dev/null

echo -n "JIRA Password:";
read -s password

echo ""

shivaram

#!/bin/bash

for i in `cat /root/mesos-ec2/slaves`; 
do 
  ssh $i 'echo -n `hostname`" "; nslookup `hostname` | grep Address | grep 10'; 
done | awk '{print $1" "$3}' > /tmp/internal

for i in `cat /root/mesos-ec2/slaves`; 
do 
  ssh $i "echo -n $i' '; nslookup $i | grep Address | grep 10"; 

shivaram

# If you are using Spark 1.4, then launch SparkR with the command
#
# ./bin/sparkR --packages com.databricks:spark-csv_2.10:1.0.3
# as the `sparkPackages=` flag was only added in Spark 1.4.1.

# # This will work in Spark 1.4.1. 
sc <- sparkR.init(spark_link, sparkPackages = "com.databricks:spark-csv_2.10:1.0.3") 
sqlContext <- sparkRSQL.init(sc) 

flights <- read.df(sqlContext, "s3n://sparkr-data/nycflights13.csv","com.databricks.spark.csv", header="true")

shivaram

# Fair Scheduler Pools
pools:
  sample_pool:
    minMaps: 5
    maxMaps: 25
    maxReduces: 25
    minSharePremptionTimeout: 300

# User limits
users:

shivaram

{
  "pools": {
    "sample_pool": {
      "minMaps": 5, 
      "maxMaps": 25, 
      "maxReduces": 25, 
      "minSharePremptionTimeout": 300
    }
  }, 
  "fairSharePreemptionTimeout": 600, 

shivaram

[info] ReplSuite:
[info] - simple foreach with accumulator
[info] - external vars
[warn] /home/shivaram/projects/spark/core/src/test/scala/spark/FileServerSuite.scala:49: method toURL in class File is deprecated: see corresponding Javadoc for more information.
[warn]     val partitionSumsWithSplit = nums.mapPartitionsWithSplit {
[warn]                                       ^
[info] - external classes
[info] - external functions
[warn] Note: /home/shivaram/projects/spark/streaming/src/test/java/spark/streaming/JavaAPISuite.java uses unchecked or unsafe operations.
[warn] Note: Recompile with -Xlint:unchecked for details.

shivaram

Keybase proof

I hereby claim:

• I am shivaram on github.
• I am shivaram (https://keybase.io/shivaram) on keybase.
• I have a public key ASAcRXjNGcE3Wivd9PLqf-4EpoDcjMUuhxSEANR88silxQo

To claim this, I am signing this object:

shivaram

Sys.setenv(SPARK_HOME="/Users/shivaram/spark-1.4.1")
.libPaths(c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib"), .libPaths()))
library(SparkR)
sc <- sparkR.init(master="local")
sqlContext <- sparkRSQL.init(sc)

df <- createDataFrame(sqlContext, faithful)
# Select one column
head(select(df, df$eruptions))