squito

import scala.collection.mutable.Map

import org.apache.spark.{Accumulator, AccumulatorParam, SparkContext}
import org.apache.spark.scheduler.{SparkListenerStageCompleted, SparkListener}
import org.apache.spark.SparkContext._


/**
 * just print out the values for all accumulators from the stage.
 * you will only get updates from *named* accumulators, though

mneedham

name := "playground"

version := "1.0"

scalaVersion := "2.10.4"

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.1.0"

libraryDependencies += "net.sf.opencsv" % "opencsv" % "2.3"

ahoy-jon

package org.apache.spark.sql.utils

import org.apache.spark.Partitioner
import org.apache.spark.rdd.{CoGroupedRDD, RDD}
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, ScalaReflection}
import org.apache.spark.sql.execution.LogicalRDD
import org.apache.spark.sql.types.{ArrayType, StructField, StructType}
import org.apache.spark.sql.{SQLContext, DataFrame, Row}
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag

bernhardschaefer

# Spark Streaming Logging Configuration
# See also: http://spark.apache.org/docs/2.0.2/running-on-yarn.html#debugging-your-application

log4j.rootLogger=INFO, stderr

# application namespace configuration
log4j.logger.de.inovex.mysparkapp=stderr, stdout

# Write all logs to standard Spark stderr file
log4j.appender.stderr=org.apache.log4j.RollingFileAppender

tomron

from __future__ import print_function

import sys
from math import sqrt
import argparse
from collections import defaultdict
from random import randint

from pyspark import SparkContext

lesstif

#!/bin/bash
#
# tomcat
#
# chkconfig: 345 96 30
# description:  Start up the Tomcat servlet engine.
#
# processname: java
# pidfile: /var/run/tomcat.pid
#

frgomes

// this flavour is pure magic...
def toDouble: (Any) => Double = { case i: Int => i case f: Float => f case d: Double => d }
  
// whilst this flavour is longer but you are in full control...
object any2Double extends Function[Any,Double] {
  def apply(any: Any): Double = 
    any match { case i: Int => i case f: Float => f case d: Double => d }
}

// like when you can invoke any2Double from another similar conversion...  

bernhardschaefer

#!/bin/bash

# Minimum TODOs on a per job basis:
# 1. define name, application jar path, main class, queue and log4j-yarn.properties path
# 2. remove properties not applicable to your Spark version (Spark 1.x vs. Spark 2.x)
# 3. tweak num_executors, executor_memory (+ overhead), and backpressure settings

# the two most important settings:
num_executors=6
executor_memory=3g

iamaziz

#!/usr/bin/env python

__author__ = 'Aziz'

"""
Convert all ipython notebook(s) in a given directory into the selected format and place output in a separate folder.
usages: python cipynb.py `directory` [-to FORMAT]

Using: ipython nbconvert and find command (Unix-like OS).

gbishop