starhashmi
starhashmi
/ spark-create-rdd.py
Created
May 24, 2018 12:16
— forked from dgadiraju/spark-create-rdd.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| orderItems = sc.textFile("/public/retail_db/order_items") | |
| type(orderItems) | |
| help(orderItems) | |
| orderItems.first() | |
| for i in orderItems.take(10): print(i) |
starhashmi
/ spark-create-rdd-parallelize.py
Created
May 24, 2018 12:18
— forked from dgadiraju/spark-create-rdd-parallelize.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| l = range(1, 10000) | |
| lRDD = sc.parallelize(l) | |
| productsRaw = open("/data/retail_db/products/part-00000").read().splitlines() | |
| type(productsRaw) | |
| productsRDD = sc.parallelize(productsRaw) | |
| type(productsRDD) | |
| productsRDD.first() | |
| for i in productsRDD.take(10): print(i) | |
| productsRDD.count() |
starhashmi
/ string-manipulation.py
Created
May 25, 2018 05:27
— forked from dgadiraju/string-manipulation.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #String Manipulation | |
| orders = sc.textFile("/public/retail_db/orders") | |
| s = orders.first() | |
| #first character from a string | |
| s[0] | |
| #first 10 characters from a string | |
| s[:10] |
starhashmi
/ spark-create-dataframe.py
Created
May 25, 2018 05:27
— forked from dgadiraju/spark-create-dataframe.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| sqlContext.load("/public/retail_db_json/order_items", "json").show() | |
| sqlContext.read.json("/public/retail_db_json/order_items").show() |
starhashmi
/ spark-flatmap-wordcount.py
Created
May 26, 2018 10:31
— forked from dgadiraju/spark-flatmap-wordcount.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #flatMap | |
| linesList = ["How are you", "let us perform", "word count using flatMap", "to understand flatMap in detail"] | |
| lines = sc.parallelize(linesList) | |
| words = lines.flatMap(lambda l: l.split(" ")) | |
| tuples = words.map(lambda word: (word, 1)) | |
| for i in tuples.countByKey(): print(i) |
starhashmi
/ spark-join.py
Created
May 27, 2018 12:18
— forked from dgadiraju/spark-join.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #joins | |
| orders = sc.textFile("/public/retail_db/orders") | |
| orderItems = sc.textFile("/public/retail_db/order_items") | |
| ordersMap = orders. \ | |
| map(lambda o:(int(o.split(",")[0]), o.split(",")[1])) | |
| orderItemsMap = orderItems. \ | |
| map(lambda oi:(int(oi.split(",")[1]), float(oi.split(",")[4]))) |
starhashmi
/ spark-outer-join.py
Created
May 27, 2018 12:26
— forked from dgadiraju/spark-outer-join.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #outer join | |
| orders = sc.textFile("/public/retail_db/orders") | |
| orderItems = sc.textFile("/public/retail_db/order_items") | |
| ordersMap = orders. \ | |
| map(lambda o:(int(o.split(",")[0]), o.split(",")[3])) | |
| orderItemsMap = orderItems. \ | |
| map(lambda oi:(int(oi.split(",")[1]), float(oi.split(",")[4]))) |
starhashmi
/ spark-compute-revenue-totals.py
Created
May 27, 2018 12:35
— forked from dgadiraju/spark-compute-revenue-totals.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #Aggregations - total | |
| orderItems = sc.textFile("/public/retail_db/order_items") | |
| orderItems.count() | |
| #Aggregations - total - Get revenue for given order_id | |
| orderItems = sc.textFile("/public/retail_db/order_items") | |
| orderItemsFiltered = orderItems. \ | |
| filter(lambda oi: int(oi.split(",")[1]) == 2) | |
| orderItemsSubtotals = orderItemsFiltered. \ | |
| map(lambda oi: float(oi.split(",")[4])) |
starhashmi
/ spark-min-revenue-totals.py
Created
May 27, 2018 12:48
— forked from dgadiraju/spark-min-revenue-totals.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Get order item details which have minimum order_item_subtotal for given order_id | |
| orderItems = sc.textFile("/public/retail_db/order_items") | |
| orderItemsFiltered = orderItems. \ | |
| filter(lambda oi: int(oi.split(",")[1]) == 2) | |
| orderItemsFiltered. \ | |
| reduce(lambda x, y: | |
| x if(float(x.split(",")[4]) < float(y.split(",")[4])) else y | |
| ) |
starhashmi
/ spark-get-count-by-status.py
Created
May 27, 2018 12:55
— forked from dgadiraju/spark-get-count-by-status.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #Get count by status - countByKey | |
| orders = sc.textFile("/public/retail_db/orders") | |
| ordersStatus = orders. \ | |
| map(lambda o: (o.split(",")[3], 1)) | |
| countByStatus = ordersStatus.countByKey() | |
| for i in countByStatus: print(i) |
OlderNewer