The Big Data Bravura: Introducing Apache Spark

1.py

from pyspark import SparkConf, SparkContext

conf = SparkConf().setMaster("local").setAppName("SubjectsByClass")
sc = SparkContext(conf=conf)


def get_class_and_subject(entry):
    fields = entry.split(',')
    class_id = fields[1]
    number_of_subjects = int(fields[2])
    return (class_id, number_of_subjects)


student_subject_rdd = sc.textFile("./student_subject.csv")

class_subject_rdd = student_subject_rdd.map(get_class_and_subject)

modified_class_subject_rdd = class_subject_rdd.mapValues(
    lambda value: (value, 1))

total_by_class_rdd = modified_class_subject_rdd.reduceByKey(
    lambda currentValue, nextvalue: (
        currentValue[0] + nextvalue[0], currentValue[1] + nextvalue[1]))

average_by_class_rdd = total_by_class_rdd.mapValues(
    lambda value: value[0] / value[1])

average_by_class_array = average_by_class_rdd.collect()

for result in average_by_class_array:
    print('The average number of subjects for class ',
          result[0], ' is ', f'{result[1]:.2f}')

student_subject.csv