medium_ref_sstable_exporter.scala

val keyspace = "keyspace"
val tableName = "tableName"
val inputPathStr = "inputPathStr"
val cassHosts = "host1:9092,host2:9092"
val sizeInMB = 61440
val primaryKeys = Array("key1", "key2", "...")
val writePartitions = Math.max(sizeInMB / 256, 1)

val df = session.read.parquet(inputPathStr)
val createTableStatement = getCreateTableStatement(keyspace, tableName, df, primaryKeys)
val insertStatement = getInsertStatement(keyspace, tableName, df)

df.repartition(writePartitions).sortWithinPartitions(df.schema.fields(0).name).rdd.mapPartitions(
  it => SSTableWriter(it, cassHosts, createTableStatement, insertStatement)
)