MrPowers

tx_cities.coalesce(1).write
.format("com.databricks.spark.csv")
.option("header", "true")
.save(System.getProperty("user.home") + "/Desktop/texas_cities")

MrPowers

val df = sqlContext.read
.format("com.databricks.spark.csv")
.option("header", "true")
.option("inferSchema", "true")
.load(System.getProperty("user.home") + "/Desktop/cities.csv")

MrPowers

val df = sqlContext.read
.format("com.databricks.spark.csv")
.option("header", "true")
.option("inferSchema", "true")
.load(System.getProperty("user.home") + "/Desktop/people/*.csv")

MrPowers

  val df = sqlContext.read
  .format("com.databricks.spark.csv")
  .option("header", "true")
  .option("inferSchema", "true")
  .load(System.getProperty("user.home") + "/Desktop/people/*.gz")

MrPowers

val df = sqlContext.read
.format("com.databricks.spark.csv")
.option("header", "true")
.option("inferSchema", "true")
.load("s3n://some_bucket/data/states/*.csv")

MrPowers

val accessKeyId = System.getenv("AWS_ACCESS_KEY_ID")
val secretAccessKey = System.getenv("AWS_SECRET_ACCESS_KEY")
sc.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", accessKeyId)
sc.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", secretAccessKey)

MrPowers

df.coalesce(1).write
.format("com.databricks.spark.csv")
.option("header", "true")
.save("s3n://some_bucket/data/states/all_states/")

MrPowers

MrPowers

MrPowers

import pathlib

import shutil
import deltalake as dl
import pandas as pd
import pyarrow.dataset as ds
from pyspark.sql import SparkSession
from delta import *
import chispa