venkatkalluru

import scala.concurrent.duration._
import scala.concurrent.ExecutionContext
import scala.concurrent.Future
import akka.pattern.after
import akka.actor.Scheduler

/**
 * Given an operation that produces a T, returns a Future containing the result of T, unless an exception is thrown,
 * in which case the operation will be retried after _delay_ time, if there are more possible retries, which is configured through
 * the _retries_ parameter. If the operation does not succeed and there is no retries left, the resulting Future will contain the last failure.

venkatkalluru

import akka.actor.ActorSystem
import akka.stream.ActorFlowMaterializer
import akka.stream.scaladsl.{Flow, Sink, Source}

import scala.collection.immutable
import scala.util.Random

object InputCustomer {
  def random():InputCustomer = {
    InputCustomer(s"FirstName${Random.nextInt(1000)} LastName${Random.nextInt(1000)}")

venkatkalluru

//HA Proxy
global
	daemon
	maxconn 256

defaults
	mode http
	timeout connect 5000ms
	timeout client 50000ms
	timeout server 50000ms

venkatkalluru

Path to change memory options ~/Library/Preferences/IntelliJIdea2018.1/idea.vmoptions (change version accordingly)

venkatkalluru

/home/user/code/github/spark

machine:spark $ cat start-spark.sh 
#!/bin/bash
export SPARK_LOCAL_IP=127.0.0.1
sbin/start-master.sh --ip 127.0.0.1
SPARK_WORKER_INSTANCES=3 sbin/start-slave.sh spark://${SPARK_LOCAL_IP}:7077
 

machine:spark $ cat stop-spark.sh 

venkatkalluru

 <plugin>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-assembly-plugin</artifactId>
    <version>2.2-beta-5</version>
    <executions>
       <execution>
          <id>jar-with-dependencies</id>
          <phase>package</phase>
          <goals>
             <goal>single</goal>

venkatkalluru

./bin/spark-submit --class org.apache.spark.examples.streaming.DirectKafkaWordCount --master spark://ip-10-204-99-241.dqa.capitalone.com:6066 --deploy-mode cluster --supervise /home/ucf347/spark/examples/target/spark-examples_2.11-2.3.0-SNAPSHOT-sources.jar 10.204.99.230:9092 msg-topic-4 schema-topic-4

venkatkalluru

Venkat_CDC Table Schema

Column Name	        #	DataType     TypeMod	Length	Precision	 Scale	NotNull	DefaultComment
-----------------------------------------------------------------------------------------
INTCOL                  1	NUMBER	      [NULL]	22	     [NULL]	   [NULL]	true	    [NULL]	
TEXTCOL	                2	VARCHAR2      [NULL]	100	     [NULL]	   [NULL]	false	    [NULL]	
FLOATCOL	        3	FLOAT	      [NULL]	22	     126	     [NULL]	false	    [NULL]	
REAL_COL	        4	FLOAT	      [NULL]	22	     63	          [NULL]	false	    [NULL]	
BINARY_DOUBLE_COL	5	BINARY_DOUBLE [NULL]	8	     [NULL]	   [NULL]	false	    [NULL]	
NUM_3_COL	        6	NUMBER	      [NULL]	22	     3	       [NULL]	false	    [NULL]	

venkatkalluru

while(<>) {
    if(m/FROM\s+(\S+)/) {
        print $1, "\n";
    }   
}

venkatkalluru

kafka handler
  http://docs.oracle.com/goldengate/bd1221/gg-bd/GADBD/GUID-2561CA12-9BAC-454B-A2E3-2D36C5C60EE5.htm#GADBD449
example
  https://www.confluent.io/blog/streaming-data-oracle-using-oracle-goldengate-kafka-connect/