counting.py - for spark tutorial

counting.py

from pyspark import SparkConf, SparkContext

conf = SparkConf().setMaster("local").setAppName("WordCount")
sc = SparkContext(conf = conf)

rdd1 = sc.textFile("file:///home/shrini/smp.csv")

def cols(data):
        sno,fname,lname,age,desig,mob,location = data.split(",")
        return sno,fname,lname,age,desig,mob,location	
        
dict1 = rdd1.countByValue()	         
dict2 = rdd1.map(cols).filter(lambda line: int(line[3])>=30).countByValue()

managers=0
for i,j in dict1.items():	
	if "Manager" in i:
		managers = managers+j
		
seniors=0
for j in dict2.values():	
	seniors = seniors+j	


print("Total No. of records:",str(rdd1.count()))
print("Distinct records:",str(rdd1.distinct().count()))
print("Toal No.of Managers:",str(managers))
print("No. of Seniors (age>30):",str(seniors))