Sample of word count using spark from a local file. Order the result in a descending order.

sparkWordCount.py

import re
from pyspark import SparkContext
print "-----------------===========================-----------------"
print "-----------------==========Staring==========-----------------"
print "-----------------===========================-----------------"
sc = SparkContext(appName = "simple app")

print "-----------------===========================-----------------"
print "-----------------==========Loaded file======-----------------"
print "-----------------===========================-----------------"
text_file = sc.textFile("lorem.txt")

print "-----------------===========================-----------------"
print "-----------------==========  Process  ======-----------------"
print "-----------------===========================-----------------"
counts = text_file.flatMap(lambda line: re.split(r'\W*', line.rstrip())) \
         .filter(lambda word: word != '') \
         .map(lambda word: (word, 1)) \
         .reduceByKey(lambda a, b: a + b) \
         .map(lambda (a,b): (b, a)) \
         .sortByKey(False)

output = counts.collect()
counts.saveAsTextFile("test.txt")
sc.stop()
for x in output:
    print (x[0], x[1])