Skip to content

Instantly share code, notes, and snippets.

View RussellSpitzer's full-sized avatar
🤷‍♀️
...

Russell Spitzer RussellSpitzer

🤷‍♀️
...
View GitHub Profile
import com.datastax.bdp.spark.DseSparkConfHelper;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.Session;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.api.java.JavaSchemaRDD;
import org.apache.spark.sql.api.java.Row;
@RussellSpitzer
RussellSpitzer / AbstractTypeTest.scala
Created June 25, 2014 05:18
Seq[AnyRef] to Java Driver Set
if (typeSet != immutable.Nil)
{
typeColTables.foreach { table =>
val statement = session.prepare(s"INSERT INTO $table (pkey, data1) VALUES (?,?)")
val javaSet = seqAsJavaList(typeSet)
typeData.foreach { value =>
val normstatement = new SimpleStatement(s"INSERT INTO $table (pkey, data1) VALUES (?,?)", value, javaSet)
//val bs = statement.bind(value, javaSet)
//println(seqAsJavaList(typeSet).getClass.toString)
//println(statement)
@RussellSpitzer
RussellSpitzer / PythonPuzzlers.py
Created July 18, 2014 20:03
Fun with Int Cache
min,max = 0,0
while min is min * 1 or max is max * 1:
if min is min * 1:
min = min - 1
if max is max * 1:
max = max + 1
print min+1,max-1
@RussellSpitzer
RussellSpitzer / sparksql.java
Last active August 29, 2015 14:13
Loading a table with java cassandra context and registering it in the hive context
package test;
/**
* Created by russellspitzer on 12/4/14.
*/
import java.io.Serializable;
import java.util.List;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
@RussellSpitzer
RussellSpitzer / RDDFunctions.scala
Created January 28, 2015 03:14
partitionByRepliac
package com.datastax.spark.connector
import java.net.InetAddress
import com.datastax.spark.connector.cql.{CassandraConnector, Schema}
import com.datastax.spark.connector.rdd.{ReadConf, CassandraRDD, ValidRDDType, SpannedRDD}
import com.datastax.spark.connector.writer._
import com.datastax.spark.connector.writer.ReplicaMapper
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
package com.datastax.spark.connector.rdd
import java.net.InetAddress
import com.datastax.driver.core.BatchStatement.Type
import com.datastax.driver.core.ConsistencyLevel
import com.datastax.spark.connector.{SomeColumns, NamedColumnRef, AllColumns, ColumnSelector}
import com.datastax.spark.connector.cql.CassandraConnector
import com.datastax.spark.connector.rdd.partitioner.CassandraPartition
import com.datastax.spark.connector.rdd.reader.RowReaderFactory
package com.datastax.spark.connector
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import com.datastax.spark.connector.cql._
import com.datastax.spark.connector.rdd.{CassandraPartitionKeyRDD, CassandraRDD, ValidRDDType, SpannedRDD}
import com.datastax.spark.connector.writer._
import com.datastax.spark.connector.rdd.reader._
package com.datastax.spark.connector
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import com.datastax.spark.connector.cql._
import com.datastax.spark.connector.rdd.{CassandraPartitionKeyRDD, CassandraRDD, ValidRDDType, SpannedRDD}
import com.datastax.spark.connector.writer._
import com.datastax.spark.connector.rdd.reader._
package com.datastax.spark.connector.rdd
import java.net.InetAddress
import org.apache.spark.{Partitioner, TaskContext, Partition}
import org.apache.spark.rdd.RDD
import org.apache.spark.SparkContext._
import scala.reflect.ClassTag
protected lazy val rowTransformer = {
logDebug("Row Transformer time")
logDebug(s"Table def null? ${tableDef == null}")
logDebug(s"Implictly[RowReaderFactory[R] null? ${implicitly[RowReaderFactory[R]] == null}")
logDebug(s"What is R? ${ct.toString()}")
implicitly[RowReaderFactory[R]].rowReader(tableDef)
}