juanrh

package streaming.experiments

object OpenSparkStreamingContexts extends App {
  import org.apache.spark._
  import org.apache.spark.streaming._
  import org.apache.spark.streaming.StreamingContext._
  import org.apache.spark.streaming.dstream.ConstantInputDStream
  
  val conf = new SparkConf().setMaster("local[5]").setAppName("OpenSparkStreamingContexts")
  val sc = new SparkContext(conf)

juanrh

import org.apache.spark._
import org.apache.spark.streaming.{StreamingContext, Duration}
import scala.reflect._
import org.apache.spark.streaming.receiver.ActorHelper
import akka.actor.{Actor, Props}

class ReceiverActorFoo[A:ClassTag]
      extends Actor with ActorHelper {
  override def receive = {
    case msg : A => store[A](msg)

juanrh

import org.apache.spark._
import org.apache.spark.streaming.{StreamingContext, Duration}
import org.apache.spark.streaming.dstream.DStream
import scala.reflect._
import org.apache.spark.streaming.receiver.ActorHelper
import akka.actor.{Actor, Props, ActorSelection}

object ProxyReceiverActor {
  def createActorDStream[A](ssc : StreamingContext, receiverActorName : String)
                           (implicit aCt : ClassTag[A]) : DStream[A] =

juanrh

// 
// TODO: stats not being send, check Direct kafka dstream and other tutorial
/** 
 * Much less efficient than DynSeqQueueInputDStream, which is based on List
 * instead of maps
 * Following https://github.com/apache/spark/blob/master/streaming/src/main/scala/org/apache/spark/streaming/dstream/QueueInputDStream.scala
 * */
class TestCaseDictInputDStream[A: ClassTag]
  (@transient _ssc : StreamingContext)
  // extends InputDStream[A](_ssc) FIXME remove when consolidated

juanrh

package es.ucm.fdi.sscheck.testing

import org.junit.runner.RunWith
import org.specs2.runner.JUnitRunner
import org.specs2.execute.{AsResult,Result}
import org.specs2.matcher.Matcher
import org.specs2.matcher.MatchersImplicits._

import org.apache.spark._
import org.apache.spark.rdd.RDD

juanrh

package es.ucm.fdi.sscheck.gen

import scala.language.implicitConversions

import org.scalacheck.Gen

import scala.collection.JavaConverters._
import java.util.concurrent.{ConcurrentLinkedQueue => JConcurrentLinkedQueue}
import java.util.{Queue => JQueue}

juanrh

def map_f(_title, text):
  for word in text.split(' '):
    yield((word, 1))

juanrh

package es.ucm.fdi.sscheck.gen

import org.specs2.Specification
import org.specs2.ScalaCheck

import org.scalacheck.Gen.Parameters
import org.scalacheck.Test
import org.scalacheck.Prop
import org.scalacheck.Gen

juanrh

package com.github.juanrh.di

/* The consumer class declares its dependencies as
 * implicit parameters in the constructor  
 * */
class Consumer(implicit private[this] val dep: Dependency) {
  def hello(): Unit = println(dep.sayHello("world"))
}

/* This trait declares which dependencies (analogous to beans)

juanrh

import org.apache.spark._
import org.apache.spark.streaming.{StreamingContext, Duration}
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import scala.reflect._

import org.apache.spark.streaming.receiver.ActorHelper
import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverStarted, StreamingListenerBatchCompleted}
import akka.actor.{Actor, Props, ActorSelection}

import scala.concurrent._