spark-notes.txt - for spark tutorial

spark-notes.txt

wget redrockdigimark.com/apachemirror/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz
tar -xzvf spark-2.3.1-bin-hadoop2.7.tgz
sudo mv spark-2.3.1-bin-hadoop2.7 /usr/local/bigdata
sudo mv spark-2.3.1-bin-hadoop2.7 spark

export SPARK_HOME=/usr/local/bigdata/spark
export PATH=$PATH:/usr/local/bigdata/spark/bin

rdd1 = sc.textFile ("file:///home/shrini/women.txt")
rdd1.collect()
rdd1.count()
rdd1.distinct().count()
rdd1.countByValue()
rdd1.first()
rdd1.take(3)
rdd1.takeSample(False,4,1)
rdd1.sample(True,0.5,3).collect()
rdd1.map(lambda i : i.split(" ")).take(3)
rdd1.flatMap(lambda i : i.split(" ")).take(3)
rdd1.filter(lambda i : ("Chennai" in i)).count()
rdd1.getNumPartitions()

sudo mv log4j.properties.template log4j.properties

/usr/local/bigdata/spark/bin/spark-submit ~/counting.py
/usr/local/bigdata/spark/bin/spark-submit ~/populations.py