path2rdd_l.py

path = "./*html"

def rdd_l(path):
    """path to rdd builder
    
    takes path as argument and returns rdd 
    """

    return sc.sparkContext.wholeTextFiles("./*.html")
    
 def df(rdd_l):
    """ rdd to df builder
    
    takes list of rdd (rdd_l) as argument and returns rdd dataframe (df).
    
    rdd stores values as a tuple of filnename and the actual value (HTML doc) in this case
    """
    return rdd_l.toDF(schema=["filename", "text"]).select("text")