PySpark Localstack S3 read CSV example

bootstrap-s3.sh

#!/usr/bin/env bash
set -x
awslocal s3 mb s3://my-bucket

cat > requests.csv <<EOF
"email1"
"email2"
"email3"
EOF

awslocal s3 cp requests.csv s3://my-bucket/

docker-compose.yml

version: '3.7'

services:
  localstack:
    image: localstack/localstack
    environment:
      - SERVICES=s3
      - DEFAULT_REGION=eu-west-1
      - AWS_ACCESS_KEY_ID=foo
      - AWS_SECRET_ACCESS_KEY=foo
    ports:
      - 4566:4566
    volumes:
      - ./bootstrap-s3.sh:/docker-entrypoint-initaws.d/bootstrap-s3.sh

run.sh

#!/usr/bin/env bash

# run Localstack
docker-compose up -d

# run spark in local mode
export SPARK_LOCAL_IP=127.0.0.1
spark-submit \
  --packages software.amazon.awssdk:s3:2.17.52,org.apache.hadoop:hadoop-aws:3.1.2 \
  --conf spark.hadoop.fs.s3a.endpoint=http://localhost:4566 \
  --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
  --conf spark.hadoop.fs.s3a.access.key=foo \
  --conf spark.hadoop.fs.s3a.secret.key=foo \
  --conf spark.hadoop.fs.s3a.path.style.access=true \
  spark-s3-test.py

#don't forget to run docker-compose down

spark-s3-test.py

# run this in pyspark shell
from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("spark_localstack_demo") \
    .getOrCreate()

# On EMR you can use `s3` instead of `s3a`
spark.read.csv('s3a://my-bucket/requests.csv').show()

# expected output
# +------+
# |   _c0|
# +------+
# |email1|
# |email2|
# |email3|
# +------+

ref: https://www.kdnuggets.com/2021/08/development-testing-etl-pipelines-aws-locally.html