hadoopsters hadoopsters

## CassandraLoader.scala
import com.datastax.spark.connector.cql.CassandraConnectorConf
import org.apache.spark.sql.{SaveMode, SparkSession}
import org.apache.spark.sql.cassandra._

object CassandraLoader extends Serializable {

  /** Representative of the some_keyspace.some_table table. */
  case class MyCassandraTable(user_id: String, `type`: Int, key: String, value: String)

  def main(args: Array[String]) { // scalastyle:off method.length

## ExampleStreamingListener.scala
package hadoopsters.spark.scala.monitoring.listeners

import org.apache.spark.streaming.kafka010.OffsetRange
import org.apache.spark.streaming.scheduler._
import org.joda.time.DateTime

/**
  * :: ExampleStreamingListener ::
  * A simple StreamingListener that accesses summary statistics across Spark Streaming batches; inherits from DeveloperAPI.
  *

## InfluxDBWriter.scala
package tv.spotx.scala.dbutils

import org.apache.logging.log4j.scala.Logging
import scalaj.http.{Http, HttpOptions, HttpResponse}

case class InfluxConfig(hostname: String = "console",
                        port: Int = 8086,  // scalastyle:off magic.number
                        database: String = "devtest",
                        ssl: Boolean = false,
                        username: Option[String] = None,

## MySQLConnection.scala
package tv.spotx.scala.dbutils

import java.sql.{Connection, DriverManager}
import java.util.Properties

import org.apache.commons.pool2.impl.{DefaultPooledObject, GenericObjectPool}
import org.apache.commons.pool2.{BasePooledObjectFactory, PooledObject}
import org.apache.logging.log4j.scala.Logging


## SpotXSparkStreamingListener.scala
package tv.spotx.scala.monitoring.listeners

import org.apache.spark.streaming.kafka010.OffsetRange
import org.apache.spark.streaming.scheduler._
import org.joda.time.DateTime
import tv.spotx.scala.dbutils.{ConnectionPool, InfluxDBWriter, MySQLConnection}

/**
  * :: SpotXSparkStreamingListener ::
  * A simple StreamingListener that logs summary statistics across Spark Streaming batches; inherits from DeveloperAPI.

## random_proportion_control.hql
select * from my_table
where rand() <= 0.0001
distribute by rand()
sort by rand()
limit 10000;

## random_distribution.hql
select * from my_table
distribute by rand()
sort by rand()
limit 10000;

## sort_by_rand.hql
select * from my_table
sort by rand()
limit 10000;

## order_by_rand.hql
select * from my_table
order by rand()
limit 10000;

## limiting.hql
select * from my_table
limit 10000;