Suresh Chaganti SureshChaganti

## spark-submit-streaming-yarn.sh
#!/bin/bash

# Minimum TODOs on a per job basis:
# 1. define name, application jar path, main class, queue and log4j-yarn.properties path
# 2. remove properties not applicable to your Spark version (Spark 1.x vs. Spark 2.x)
# 3. tweak num_executors, executor_memory (+ overhead), and backpressure settings

# the two most important settings:
num_executors=6
executor_memory=3g

## Building the Jar with Maven
There are 3 plugins for generating to create a JAR-File in maven:
- maven-jar-plugin
- maven-assembly-plugin
- maven-shade-plugin

maven-jar-plugin:This plugin provides the capability to build and sign jars.But it just compiles the java files under **src/main/java** and **/src/main/resources/**.It doesn't include the dependencies JAR files.
```
<!--exclude all xml files from the jar-->
<plugin>
    <groupId>org.apache.maven.plugins</groupId>

## reading-property-from-file.sh
#!/bin/sh

PROPERTY_FILE=apps.properties

function getProperty {
   PROP_KEY=$1
   PROP_VALUE=`cat $PROPERTY_FILE | grep "$PROP_KEY" | cut -d'=' -f2`
   echo $PROP_VALUE
}

## Step 1
$ brew update
$ brew install hadoop
$ brew install hive
$ brew install mysql

## gist:a68360edcf9f3527d1c2d1c88747a00e
def uuid = java.util.UUID.randomUUID.toString

## PomDependenciesToSbt
#!/usr/bin/env amm

// This script converts Maven dependencies from a pom.xml to sbt dependencies.
// It is based on the answers of George Pligor and Mike Slinn on http://stackoverflow.com/questions/15430346/
// - install https://github.com/lihaoyi/Ammonite
// - make this script executable: chmod +x PomDependenciesToSbt
// - run it with from your shell (e.g bash):
// $ ./PomDependenciesToSbt /path/to/pom.xml

import scala.xml._
	#!/bin/bash

	# Minimum TODOs on a per job basis:
	# 1. define name, application jar path, main class, queue and log4j-yarn.properties path
	# 2. remove properties not applicable to your Spark version (Spark 1.x vs. Spark 2.x)
	# 3. tweak num_executors, executor_memory (+ overhead), and backpressure settings

	# the two most important settings:
	num_executors=6
	executor_memory=3g
	There are 3 plugins for generating to create a JAR-File in maven:
	- maven-jar-plugin
	- maven-assembly-plugin
	- maven-shade-plugin

	maven-jar-plugin:This plugin provides the capability to build and sign jars.But it just compiles the java files under src/main/java and /src/main/resources/.It doesn't include the dependencies JAR files.
	```
	<!--exclude all xml files from the jar-->
	<plugin>
	<groupId>org.apache.maven.plugins</groupId>
	#!/bin/sh

	PROPERTY_FILE=apps.properties

	function getProperty {
	PROP_KEY=$1
	PROP_VALUE=`cat $PROPERTY_FILE \| grep "$PROP_KEY" \| cut -d'=' -f2`
	echo $PROP_VALUE
	}
	$ brew update
	$ brew install hadoop
	$ brew install hive
	$ brew install mysql
	#!/usr/bin/env amm

	// This script converts Maven dependencies from a pom.xml to sbt dependencies.
	// It is based on the answers of George Pligor and Mike Slinn on http://stackoverflow.com/questions/15430346/
	// - install https://github.com/lihaoyi/Ammonite
	// - make this script executable: chmod +x PomDependenciesToSbt
	// - run it with from your shell (e.g bash):
	// $ ./PomDependenciesToSbt /path/to/pom.xml

	import scala.xml._