Sample code for computing difference

difference.py

def calc_diff():
    os.environ['SPARK_HOME'] = "/usr/hdp/current/spark-client/bin"
    conf = SparkConf().setMaster("local[*]").setAppName("TestApp")
    sc = SparkContext(conf=conf)
    sqlcontext = HiveContext(sc)

    df = sqlcontext.createDataFrame(
        [(1, "a", 23.0), (2, "B", 23.0), (3, "C", 33.0)], ("x1", "x2", "x3"))

    r = df.select("x3")
    rdd = r.rdd
    rdd = rdd.map(lambda p: float(p.x3)).zipWithIndex().map(lambda (x, y): (y, x))

    indexes = rdd.map(lambda x: x[0]).map(lambda x: (1, x)).keys().reduce(lambda x, y: x + y)
    print "No of elements: ", indexes

    df.show()

    temp_list = sc.parallelize([])
    for i in range(indexes - 1):
        temp_list = temp_list.union(sc.parallelize([(rdd.lookup(i) + rdd.lookup(i + 1))]))
        print temp_list.collect()

    print temp_list.map(lambda x: x[0] + x[1]).collect()
    result_rdd = temp_list.map(lambda x: x[0] - x[1])
    print
    parts = result_rdd.map(lambda p: Row(difference=p))
    result_df = sqlcontext.createDataFrame(parts)
    result_df.show()
    print

    sc.stop()

if __name__ == '__main__':
    calc_diff()