HeartSaVioR

package net.heartsavior;

import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;

public class ClassInvestigator {

  public static void main(String[] args) {
    if (args.length < 1) {

HeartSaVioR

/////////////////////////////////////////////////////////////////////////////////////////////
// 1. select with swapping columns, and apply where
/////////////////////////////////////////////////////////////////////////////////////////////
import spark.implicits._

import org.apache.spark.sql.{DataFrame, Dataset}

case class Hello(id: Int, name1: String, name2: String)

val ds = List(Hello(1, "Alice", "Bob"), Hello(2, "Bob", "Alice")).toDS

HeartSaVioR

import java.util.Properties

import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.functions.{from_json, struct, to_json}
import org.apache.spark.sql.streaming.{OutputMode, StreamingQueryListener, Trigger}
import org.apache.spark.sql.types._
import org.apache.commons.logging.LogFactory

object StreamingAnalyticsTruckingRefApp {

HeartSaVioR

// assuming we paste the code to `spark-shell`
// spark-shell --packages org.apache.spark:spark-sql-kafka-0-10_2.11:2.3.1

import org.apache.spark.sql.functions.{from_json, struct, to_json}
import org.apache.spark.sql.types._
import spark.implicits._

val df = spark
    .read
    .format("kafka")

HeartSaVioR

package net.heartsavior.spark.trial

import java.nio.charset.StandardCharsets

import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeProjection, UnsafeRow}
import org.apache.spark.sql.types.StructType
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.SizeEstimator

object SparkTrialCalculateUnsafeRowSize {

HeartSaVioR

// This is to answer the question regarding structured streaming on the Apache Spark mailing list
// http://apache-spark-user-list.1001560.n3.nabble.com/can-we-use-mapGroupsWithState-in-raw-sql-tp31885p31893.html

/* 
Input:

id | amount     | my_timestamp
-------------------------------------------
1  |      5     |  2018-04-01T01:00:00.000Z
1  |     10     |  2018-04-01T01:10:00.000Z

HeartSaVioR

package net.heartsavior.spark.trial

import org.apache.spark.sql.catalyst.expressions.StructsToJson
import org.apache.spark.sql.functions._
import org.apache.spark.sql.streaming.{OutputMode, Trigger}
import org.apache.spark.sql.types._

object SparkTrial2 {

  def main(args: Array[String]): Unit = {

HeartSaVioR

package net.heartsavior.spark.trial

import org.apache.spark.sql.functions._
import org.apache.spark.sql.streaming.{OutputMode, Trigger}
import org.apache.spark.sql.types._

object SparkTrial {
  def main(args: Array[String]): Unit = {
    import org.apache.spark.sql.SparkSession

HeartSaVioR

#!/bin/bash

current_test=1
test_count=3

storm_directory="`pwd`"

part1=`dirname "${storm_directory}"`
part2=`basename "${storm_directory}"`

HeartSaVioR

#!/bin/bash

current_test=1
test_count=3

storm_directory="`pwd`"

part1=`dirname "${storm_directory}"`
part2=`basename "${storm_directory}"`