brianmhess

A. DSEFS
--------
0.1. Enable
vi /etc/default/dse
  Edit to enable Spark
  
vi /etc/dse/dse.yaml
  Edit to enable dsefs
  dsefs_option:
    enabled: true

brianmhess

package com.datastax.loader;

import java.util.Map;

import com.datastax.driver.core.Session;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.CloseFuture;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Statement;

brianmhess

## If you did not install in the default directory, replace '/opt/datastax/cassandraodbc'
## with the correct location.

[ODBC]
Trace=no

[ODBC Data Sources]
cassandra64=DataStax Cassandra ODBC Driver 64-bit
spark64=Simba Spark ODBC Driver 64-bit
spark64old=Simba Spark ODBC Driver 64-bit

brianmhess

mkdir -p /tmp/opt/connector
cd /tmp/opt/connector

rm *.jar

curl -o ivy-2.3.0.jar \
  'http://search.maven.org/remotecontent?filepath=org/apache/ivy/ivy/2.3.0/ivy-2.3.0.jar'
curl -o spark-cassandra-connector_2.10-1.0.0-beta1.jar \
  'http://search.maven.org/remotecontent?filepath=com/datastax/spark/spark-cassandra-connector_2.10/1.0.0-beta1/spark-cassandra-connector_2.10-1.0.0-beta1.jar'

brianmhess

Setup

First we need to create the Cassandra table and populate with simple data.

In CQLSH:

CREATE KEYSPACE sparktest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};
CREATE TABLE sparktest.test(pkey INT, ccol INT, x INT, y INT, PRIMARY KEY ((pkey), ccol));
INSERT INTO sparktest.test(pkey,ccol,x,y) VALUES (1,1,1,1);
INSERT INTO sparktest.test(pkey,ccol,x,y) VALUES (2,2,2,2);
INSERT INTO sparktest.test(pkey,ccol,x,y) VALUES (3,3,3,3);

brianmhess

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#

brianmhess

## If you did not install in the default directory, replace '/opt/datastax/cassandraodbc' 
## with the correct location.

[ODBC]
Trace=no

[ODBC Data Sources]
spark64=Simba Spark ODBC Driver 64-bit

[spark64]

brianmhess

/*
java -cp ".:/home/ubuntu/UnityJDBC/mongodb_unityjdbc_full.jar" -Djavax.net.ssl.trustStore=/home/ubuntu/downloads/certs/rds-truststore.jks -Djavax.net.ssl.trustStorePassword=<PASSWORD> JdbcSql mongodb.jdbc.MongoDriver "jdbc:mongodb://<USERNAME>:<PASSWORD>@<HOSTNAME>:27017/ab3?ssl=true" "SELECT * FROM ab3.venue"

java -cp ".:/home/ubuntu/CData/CData_MongoDB/lib/cdata.jdbc.mongodb.jar" JdbcSql cdata.jdbc.mongodb.MongoDBDriver "jdbc:mongodb:Server="<HOSTNAME>";User="<USERNAME>";Password="<PASSWORD>";SSLClientCert="";SSLClientCertType="PEMKEY_FILE";AuthMechanism="SCRAM-SHA-1";SSLServerCert="/home/ubuntu/rds-combined-ca-bundle.pem";UseSSL="True";" "SELECT * FROM ab3.venue;"

java -cp ".:../../Progress/DataDirect/JDBC_60/lib/mongodb.jar" -Djavax.net.ssl.trustStore=/home/ubuntu/downloads/certs/rds-truststore.jks -Djavax.net.ssl.trustStorePassword=<STOREPWD> JdbcSql com.ddtek.jdbc.mongodb.MongoDBDriver "jdbc:datadirect:mongodb://<HOSTNAME>:27017;EncryptionMethod=SSL;TrustStore=/home/ubuntu/downloads/certs/rds-trust

brianmhess

sc.textFile("/mnt/disk2/data/data_99-rand.csv").map(x => x.split(",")).map(x => (x(0).toLong, x(1).toLong, x(2).toLong, x(3).toLong, x(4).toLong, x(5).toLong, x(6).toLong, x(7).toLong, x(8).toLong, x(9).toLong)).saveToCassandra("test", "test10", SomeColumns("pkey", "ccol", "col1", "col2", "col3", "col4", "col5", "col6", "col7", "col8"))

brianmhess

package org.apache.cassandra.db.compaction;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;