[spark/Scala - Access S3 data] Access S3 data from spark #spark #scala #s3

s3_access.sc

// Set up AWS credentials
sc.hadoopConfiguration.set("fs.s3a.access.key", "AKI*****************")
sc.hadoopConfiguration.set("fs.s3a.secret.key", "kd8***********************************")

def wordcount() = {
    // Read & process S3 file - word count
    val abc_file = sc.textFile("s3a://prms-s3/data/abc.txt")
    val counts = abc_file.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
    //counts.saveAsTextFile("s3a://s3-to-ec2/output")
    counts.toDF().show()
}

def s3_csv() = {
    val s1_csv_rdd = sc.textFile("s3a://prms-s3/data/s1.csv")
                            .map( line => line.split(","))
    s1_csv_rdd.toDF().show()
}

// Call Word count
wordcount()

//call s3 csv load
s3_csv()