This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
val ts = sc.parallelize(0 to 100, 10) | |
val window = 3 | |
class StraightPartitioner(p: Int) extends Partitioner { | |
def numPartitions = p | |
def getPartition(key: Int) = key * p/0.5 | |
} | |
val partitioned = ts.mapPartitionsWithIndex((i, p) => { | |
val overlap = p.take(window - 1).toArray |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.neolitec.examples; | |
import org.apache.commons.codec.binary.Base64; | |
import org.apache.commons.lang.StringUtils; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import javax.servlet.*; | |
import javax.servlet.http.HttpServletRequest; | |
import javax.servlet.http.HttpServletResponse; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<!-- place this in an %angular paragraph --> | |
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.5/leaflet.css" /> | |
<div id="map" style="height: 800px; width: 100%"></div> | |
<script type="text/javascript"> | |
function initMap() { | |
var map = L.map('map').setView([30.00, -30.00], 3); | |
L.tileLayer('http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.apache.spark.sql.utils | |
import org.apache.spark.Partitioner | |
import org.apache.spark.rdd.{CoGroupedRDD, RDD} | |
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, ScalaReflection} | |
import org.apache.spark.sql.execution.LogicalRDD | |
import org.apache.spark.sql.types.{ArrayType, StructField, StructType} | |
import org.apache.spark.sql.{SQLContext, DataFrame, Row} | |
import scala.reflect.ClassTag | |
import scala.reflect.runtime.universe.TypeTag |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io._ | |
@SerialVersionUID(15L) | |
class Animal(name: String, age: Int) extends Serializable { | |
override def toString = s"Animal($name, $age)" | |
} | |
case class Person(name: String) | |
// or fork := true in sbt |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
val n = 9 | |
val s = Math.sqrt(n).toInt | |
type Board = IndexedSeq[IndexedSeq[Int]] | |
def solve(board: Board, cell: Int = 0): Option[Board] = (cell%n, cell/n) match { | |
case (r, `n`) => Some(board) | |
case (r, c) if board(r)(c) > 0 => solve(board, cell + 1) | |
case (r, c) => | |
def cells(i: Int) = Seq(board(r)(i), board(i)(c), board(s*(r/s) + i/s)(s*(c/s) + i%s)) | |
def guess(x: Int) = solve(board.updated(r, board(r).updated(c, x)), cell + 1) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.apache.spark | |
import org.apache.spark.sql.catalyst.util._ | |
import org.apache.spark.sql.types._ | |
@SQLUserDefinedType(udt = classOf[PointType]) | |
case class Point(mac: String, start: Long, end: Long) { | |
override def hashCode(): Int = { | |
31 * (31 * mac.hashCode) + start.hashCode | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
//================================================================== | |
// SPARK INSTRUMENTATION | |
//================================================================== | |
import com.codahale.metrics.{MetricRegistry, Meter, Gauge} | |
import org.apache.spark.{SparkEnv, Accumulator} | |
import org.apache.spark.metrics.source.Source | |
import org.joda.time.DateTime | |
import scala.collection.mutable |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
http://localhost:4040/metrics/json/ | |
spark.conf.set("spark.sql.streaming.metricsEnabled", "true") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.lang.reflect.Method | |
import java.net.URI | |
import org.apache.spark.sql.SparkSession | |
import org.apache.spark.sql.catalyst.TableIdentifier | |
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} | |
import org.apache.spark.sql.execution.command.ShowCreateTableCommand | |
import org.apache.spark.sql.types.StructType | |
def showCreateTableCommand( |
OlderNewer