Skip to content

Instantly share code, notes, and snippets.

View dgadiraju's full-sized avatar

Durga Gadiraju dgadiraju

2.3k followers · 2 following
View GitHub Profile
@dgadiraju
dgadiraju / spark-core-subtract-get-customer-details.scala
Created September 10, 2019 13:18
val orderCustomers1 = sc.textFile("/public/retail_db/orders").
filter(o => o.split(",")(1).contains("2013-08")).
map(o => o.split(",")(2).toInt)
val orderCustomers2 = sc.textFile("/public/retail_db/orders").
filter(o => o.split(",")(1).contains("2013-09")).
map(o => o.split(",")(2).toInt)
val customers = sc.textFile("/public/retail_db/customers").
map(c => (c.split(",")(0).toInt, c))
@dgadiraju
dgadiraju / spark-core-intersection-01-customer-ids.scala
Last active September 10, 2019 13:17
val orderCustomers1 = sc.textFile("/public/retail_db/orders").
filter(o => o.split(",")(1).contains("2013-08")).
map(o => o.split(",")(2).toInt)
val orderCustomers2 = sc.textFile("/public/retail_db/orders").
filter(o => o.split(",")(1).contains("2013-09")).
map(o => o.split(",")(2).toInt)
orderCustomers1.count
orderCustomers2.count
@dgadiraju
dgadiraju / spark-core-union-01-all.scala
Last active September 10, 2019 12:26
val orderCustomers1 = sc.textFile("/public/retail_db/orders").
filter(o => o.split(",")(1).contains("2013-08")).
map(o => o.split(",")(2))
val orderCustomers2 = sc.textFile("/public/retail_db/orders").
filter(o => o.split(",")(1).contains("2013-09")).
map(o => o.split(",")(2))
orderCustomers1.count
orderCustomers2.count
@dgadiraju
dgadiraju / spark-core-outer-join-orders-count-by-status.scala
Created September 9, 2019 13:30
val orders = sc.textFile("/public/retail_db/orders")
val ordersMap = orders.
map(o => (o.split(",")(0).toInt, o.split(",")(3)))
val orderItems = sc.textFile("/public/retail_db/order_items")
val orderItemsMap = orderItems.
map(oi => (oi.split(",")(1).toInt, oi))
val ordersLeftOuterJoin = ordersMap.
leftOuterJoin(orderItemsMap)
@dgadiraju
dgadiraju / spark-core-outer-join-orders-and-order-items-filter.scala
Created September 9, 2019 13:25
val orders = sc.textFile("/public/retail_db/orders")
val ordersMap = orders.
map(o => (o.split(",")(0).toInt, o))
val orderItems = sc.textFile("/public/retail_db/order_items")
val orderItemsMap = orderItems.
map(oi => (oi.split(",")(1).toInt, oi))
val ordersLeftOuterJoin = ordersMap.
leftOuterJoin(orderItemsMap)
@dgadiraju
dgadiraju / spark-core-outer-join-orders-and-order-items.scala
Created September 9, 2019 13:23
val orders = sc.textFile("/public/retail_db/orders")
val ordersMap = orders.
map(o => (o.split(",")(0).toInt, o))
val orderItems = sc.textFile("/public/retail_db/order_items")
val orderItemsMap = orderItems.
map(oi => (oi.split(",")(1).toInt, oi))
val ordersJoin = ordersMap.
join(orderItemsMap)
@dgadiraju
dgadiraju / spark-core-01-read-orders.scala
Created September 9, 2019 11:00
val orders = sc.textFile("/public/retail_db/orders")
val ordersMap = orders.
map(o => (o.split(",")(0).toInt, o))
@dgadiraju
dgadiraju / spark-core-groupByKey-ranking.scala
Created September 5, 2019 14:48
val products = sc.textFile("/public/retail_db/products")
val productsMap = products.
filter(p => p.split(",")(4) != "").
map(p => (p.split(",")(1).toInt, p))
productsMap.
groupByKey.
flatMap(p => {
p._2.toList.sortBy(k => k.split(",")(4).toFloat)(Ordering.Float.reverse).take(3)
@dgadiraju
dgadiraju / spark-core-groupByKey-sorting.scala
Created September 5, 2019 14:46
val products = sc.textFile("/public/retail_db/products")
val productsMap = products.
filter(p => p.split(",")(4) != "").
map(p => (p.split(",")(1).toInt, p))
productsMap.
groupByKey.
flatMap(p => {
p._2.toList.sortBy(k => k.split(",")(4).toFloat)(Ordering.Float.reverse)
@dgadiraju
dgadiraju / spark-core-aggregateByKey-get-revenue-and-count-per-order-id.scala
Created September 5, 2019 14:25
val orderItems = sc.textFile("/public/retail_db/order_items")
val orderItemsMap = orderItems.
map(oi => (oi.split(",")(1).toInt, oi.split(",")(4).toFloat))
val revenueAndCountPerOrder = orderItemsMap.
aggregateByKey((0.0, 0))(
(agg, ele) => (agg._1 + ele, agg._2 + 1),
(total, inter) => (total._1 + inter._1, total._2 + inter._2)
)