shredder47

mvn archetype:generate -DgroupId=com.shredder -DartifactId=livy-experiment -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false

shredder47

## Java
sudo apt-get update
sudo apt-get install default-jdk

## Scala
sudo apt-get remove scala-library scala
sudo wget https://downloads.lightbend.com/scala/2.12.12/scala-2.12.12.deb
sudo dpkg -i scala-2.12.12.deb
sudo apt-get update

shredder47

java -version
sudo yum remove java
sudo yum install java-1.8.0-openjdk
source ~/.bash_profile

Download Spark 2.4.7 With Hadoop 2.6 (Tar)
Extract contents.
Move the contents of the folder to :

/usr/local/spark

shredder47

  val spark = SparkSession
    .builder()
    .appName("Spark")
    .master("local[*]")
    .enableHiveSupport()
    .getOrCreate()

  spark.sparkContext.setLogLevel("ERROR")

shredder47

   git init
   git add .
   git commit -m "question1 added"
   git branch -M main
   git remote add origin remote_url
   git push -u origin main

//For removing exising origin and adding new

 git remote show origin

shredder47

---------------
SUBSTRING_INDEX
---------------
SUBSTRING_INDEX(column_name,' ' , 1) // Not 0 Indexed!

--------
to_date
--------
to_date(col_with_date_as_string,'MM/yyyy')  //parse date string to date type using the specified format - Returns it as yyyy-MM-dd

shredder47

<build>
        <finalName>Project Name</finalName>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>11</source>
                    <target>11</target>

shredder47

mvn install:install-file -Dfile=shredder-engine-sdk.jar -DgroupId=com.shredder -DartifactId=shredder-sdk -Dversion=1.0-SNAPSHOT -Dpackaging=jar

// -Dfile = full path to the JAR file. Else put the JAR in project's root Dir and then run mvn command from there if you want to specify only the Jar file name.