Pyspark streaming from and to file

pyspark_streaming_from_and_to_file.py

from pyspark.sql.functions import explode
from pyspark.sql.functions import split
from pyspark.streaming import StreamingContext
from pyspark.sql import SparkSession
import time

spark = SparkSession.builder \
        .appName('Streaming') \
        .getOrCreate()

sc = spark._sc

#lines =spark.readStream \
#    .format('socket') \
#    .option('host', 'localhost') \
#    .option('port', 9999) \
#    .load()

lines =spark.readStream \
    .format('text') \
    .option('path', '/home/sallso/tmp/') \
    .load()

words = lines.select(
        explode(
            split(lines.value, ' ')
        ).alias('word')
)
wordCounts = words.groupBy('word').count()

query = wordCounts \
    .writeStream \
    .outputMode("complete") \
    .format("console") \
    .start()

query.awaitTermination()
sc.stop()

~                                                                                                                                                                                                           
~                                                                                                                                                                                                           
~                                                                                                                                                                                                           
~                                                                                                                                                                                                           
~