sumitsu

hadoopConf.set("fs.s3a.s3.client.factory.impl", classOf[NonChunkedDefaultS3ClientFactory].getName)

sumitsu

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.S3ClientOptions;
import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;

import java.io.IOException;
import java.net.URI;

public class NonChunkedDefaultS3ClientFactory extends DefaultS3ClientFactory {
  @Override

sumitsu

import org.apache.hadoop.fs.s3a.S3AFileSystem

private val hadoopConf = spark.sparkContext.hadoopConfiguration

hadoopConf.set("fs.s3.impl", classOf[S3AFileSystem].getName)
hadoopConf.set("fs.s3a.endpoint", "http://localhost:9999")
hadoopConf.set("fs.s3a.access.key", "abc")
hadoopConf.set("fs.s3a.secret.key", "xyz")
hadoopConf.set("fs.s3a.attempts.maximum", "3")
hadoopConf.set("fs.s3a.path.style.access", "true")

sumitsu

  val spark: SparkSession = {
    val sparkConf = new SparkConf()
      .set("spark.driver.host", "127.0.0.1")
      .setMaster("local[2]")
      .setAppName("TestSparkApp")
    val sparkSession = SparkSession.builder.config(sparkConf).getOrCreate
    sparkSession.sql("set spark.sql.caseSensitive=true")
    sparkSession
  }

sumitsu

val MockS3ServerPortEnvVar: String = "MOCK_SERVER_PORT"
val MockS3ServerPortDefault: Int = 9999
val AwsEndpointUriStr: String =
  s"http://localhost:${Option(System.getenv(MockS3ServerPortEnvVar)).getOrElse(MockS3ServerPortDefault)}/"
val TestBucketName: String =
  s"s3-mocktest-demo-${UUID.randomUUID.toString}"
val MockAWSAccessKey: String = "abc"
val MockAWSSecretKey: String = "zyx"

def buildLocalMockTestS3Client(): AmazonS3 = {