PeterCorless

val writer = df.write.cassandraFormat(table = "test", keyspace = "test")
// writer: org.apache.spark.sql.DataFrameWriter[org.apache.spark.sql.Row] = org.apache.spark.sql.DataFrameWriter@6cf47d05

PeterCorless

{
    "AAPL": {
        "quote": {
            "latestPrice": 221.43,
            "latestSource": "IEX real time price",
            "latestUpdate": 1537455071032,
            "latestVolume": 18919004,
            "previousClose": 218.37,
            "symbol": "AAPL"
        }

PeterCorless

tar -xzf confluent-5.0.0-2.11.tar.gz
cd confluent-5.0.0/

PeterCorless

for i=0 to 3:
  cassandra-stress write no-warmup n=175000000 cl=ALL
    -rate threads=500
    -schema 'replication(factor=2) keyspace=keyspace'
    -col 'size=FIXED(4000) n=FIXED(1)'
    -mode cql3 native connectionsPerHost=66
    -pop 'seq=((i*n)+1)..((i+1)*n)' # where n=175000000
    -node node2
    -errors ignore

PeterCorless

# On one node, start the master:
spark-2.4.0-bin-hadoop2.7 $ ./sbin/start-master.sh
# On the same node, and on the other nodes, start the Spark workers:
spark-2.4.0-bin-hadoop2.7 $ SPARK_WORKER_INSTANCES=8 SPARK_WORKER_CORES=2 ./sbin/start-slave.sh spark://<spark-master-ip>:7077

PeterCorless

#!/usr/bin/python3
import yaml
a = yaml.load("string: hello world!")
print(a['string'])

PeterCorless

cqlsh> desc SCHEMA
CREATE KEYSPACE catalog WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} AND durable_writes = true;

CREATE TABLE catalog.apparel (
    sku text,
    color text,
    size text,
    brand text,
    gender text,
    group text,

PeterCorless

val connector = 
  new CassandraConnector(CassandraConnectorConf(sparkContext.getConf))

PeterCorless

$ tar -xzf confluent-5.2.0-2.12.tar.gz
$ cd confluent-5.2.0/

PeterCorless

CREATE TABLE ks.tbl (
    uuid int,
    time timestamp,
    property text,
    PRIMARY KEY (uuid, time))