Skip to content

Instantly share code, notes, and snippets.

Durga Gadiraju dgadiraju

Block or report user

Report or block dgadiraju

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View pandas-tasks-01-read-order_items.py
import pandas as pd
# Reading order_items
order_items_path = "/Users/itversity/Research/data/retail_db/order_items/part-00000"
order_items_schema = [
"order_item_id",
"order_item_order_id",
"order_item_product_id",
"order_item_quantity",
View create-dataframe-for-orders.py
orders_path = "/Users/itversity/Research/data/retail_db/orders/part-00000"
orders_schema = [
"order_id",
"order_date",
"order_customer_id",
"order_status"
]
orders = pd.read_csv(orders_path,
header=None,
View convert-field-of-dataframe-to-series.py
orders_path = "/Users/itversity/Research/data/retail_db/orders/part-00000"
orders_schema = [
"order_id",
"order_date",
"order_customer_id",
"order_status"
]
orders = pd.read_csv(orders_path,
header=None,
names=orders_schema
View convert-dict-to-series.py
d = {"JAN": 10, "FEB": 15, "MAR": 12, "APR": 16}
import pandas as pd
s = pd.Series(d)
s.count()
s.sum()
s.min()
s.max()
View spark-df-orders-csv.scala
val ordersDF = spark.
read.
schema("order_id INT, order_date STRING, order_customer_id INT, order_status STRING").
csv("/Users/itversity/Research/data/retail_db/orders")
val ordersDF = spark.
read.
schema("order_id INT, order_date STRING, order_customer_id INT, order_status STRING").
format("csv").
load("/Users/itversity/Research/data/retail_db/orders")
View spark-udf-toDateUDF.scala
def toDate(d: String) = {
val a = d.split("/")
(a(2) + "%02d".format(a(0).toInt) + "%02d".format(a(1).toInt)).toInt
}
toDate("7/3/2019")
toDate("10/3/2019")
val toDateUDF = org.apache.spark.sql.functions.udf(toDate _)
View spark-core-spark-submit-local.sh
spark-submit \
--master local \
--class retail.DailyProductRevenue \
--packages com.typesafe:config:1.3.2 \
target/scala-2.11/retail_2.11-0.1.jar dev
View spark-core-daily-product-revenue.scala
package retail
import com.typesafe.config.ConfigFactory
import org.apache.spark.{SparkConf, SparkContext}
/**
* Created by itversity on 05/06/17.
*/
object DailyProductRevenue {
def main(args: Array[String]): Unit = {
View DailyProductRevenue01.scala
package retail
import com.typesafe.config.ConfigFactory
import org.apache.spark.{SparkConf, SparkContext}
/**
* Created by itversity on 05/06/17.
*/
object DailyProductRevenue {
def main(args: Array[String]): Unit = {
View spark-core-create-spark-context.scala
import com.typesafe.config.ConfigFactory
import org.apache.spark.{SparkConf, SparkContext}
val props = ConfigFactory.load()
val envProps = props.getConfig("dev")
val conf = new SparkConf().
setAppName("sbt console").
setMaster(envProps.getString("execution.mode"))
You can’t perform that action at this time.