syedatifakhtar

package com.syedatifakhtar.pipelines

import scala.concurrent.duration.Duration

object Pipelines {

  import scala.concurrent.{Await, ExecutionContext, Future}
  import scala.util.{Failure, Success, Try}

syedatifakhtar

# WSL2 network port forwarding script v1
#   for enable script, 'Set-ExecutionPolicy -ExecutionPolicy Bypass -Scope CurrentUser' in Powershell,
#   for delete exist rules and ports use 'delete' as parameter, for show ports use 'list' as parameter.
#   written by Daehyuk Ahn, Aug-1-2020

# Display all portproxy information
If ($Args[0] -eq "list") {
    netsh interface portproxy show v4tov4;
    exit;
} 

syedatifakhtar

import java.io.{BufferedWriter, DataOutputStream, File, FileWriter}

import scala.io.Source.fromFile
import scala.util.Random
import scala.util.parsing.json.JSON

object DBGenSansSpark {
  val master = "local[2]"
  val appName = "testing"

syedatifakhtar

import org.apache.spark.SparkConf
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SaveMode, SparkSession}

import scala.io.Source.fromFile
import scala.util.Random
import scala.util.parsing.json.JSON

object DBGenMain2 {

syedatifakhtar

import scala.io.Source

object Anagrams {


  type Word = String

  type Sentence = List[Word]
  type Occurrences = List[(Char, Int)]

syedatifakhtar

import scala.util.Try

println("Hello World")


//Functions as first class citizen

def someFunction = {a: Int=> a + 10}
val zz = someFunction

syedatifakhtar

%spark.dep
z.reset() // clean up previously added artifact and repository

// add maven repository
z.addRepo("Millhouse Bintray").url("http://dl.bintray.com/themillhousegroup/maven")

z.load("com.themillhousegroup:scoup_2.10:0.2.295")

import com.themillhousegroup.scoup.Scoup
import com.themillhousegroup.scoup.ScoupImplicits._

syedatifakhtar

SESSION 1
What is Big Data?Why Big Data?
Some Examples of where Big Data is necessary
Thinking in terms of MapReduce
HDFS vs S3 vs Local File System
Resources/Containers/Nodes
WordCount - Exercise

Extras-:
JOINS in MapReduce

syedatifakhtar

val someMap = Map(
  typeTag[scala.math.BigDecimal].tpe -> 0
)

def someFun[A: TypeTag](value: Option[A]): Unit = {
  val someType = typeTag[scala.math.BigDecimal]
  val myType = someType.tpe
  val targ: _root_.scala.reflect.runtime.universe.Type = typeOf[A]
  println(s"TARG: ${targ.normalize} and mytpe = ${myType}")
  val typeTagValue: _root_.scala.reflect.runtime.universe.TypeTag[A] = typeTag[A]

syedatifakhtar

/*************************************Spark Basics************************************************
**************************************************************************************************/


val someSentences = List("The quick brown fox jumped over the wall","Another sentence","Some more")

someSentences.filter(sentence=> !sentence.toLowerCase.startsWith("some"))

someSentences.map(x=>x.split(" "))
someSentences.flatMap(x=>x.split(" "))