Created
February 19, 2014 10:15
-
-
Save rklaehn/9089304 to your computer and use it in GitHub Desktop.
Benchmark for inserting wide rows into a Cassandra DB using both CQL and Thrift APIs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package cassandra; | |
import java.util.Iterator; | |
import com.netflix.astyanax.ColumnListMutation; | |
import com.netflix.astyanax.serializers.AsciiSerializer; | |
import com.netflix.astyanax.serializers.LongSerializer; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import com.netflix.astyanax.AstyanaxContext; | |
import com.netflix.astyanax.Keyspace; | |
import com.netflix.astyanax.MutationBatch; | |
import com.netflix.astyanax.connectionpool.NodeDiscoveryType; | |
import com.netflix.astyanax.connectionpool.OperationResult; | |
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; | |
import com.netflix.astyanax.connectionpool.impl.ConnectionPoolConfigurationImpl; | |
import com.netflix.astyanax.connectionpool.impl.CountingConnectionPoolMonitor; | |
import com.netflix.astyanax.impl.AstyanaxConfigurationImpl; | |
import com.netflix.astyanax.model.Column; | |
import com.netflix.astyanax.model.ColumnFamily; | |
import com.netflix.astyanax.model.ColumnList; | |
import com.netflix.astyanax.thrift.ThriftFamilyFactory; | |
public class AstClient { | |
private static final Logger logger = LoggerFactory.getLogger(AstClient.class); | |
private AstyanaxContext<Keyspace> context; | |
private Keyspace keyspace; | |
private ColumnFamily<Long, String> EMP_CF; | |
private static final String EMP_CF_NAME = "employees2"; | |
public void init() { | |
logger.debug("init()"); | |
context = new AstyanaxContext.Builder() | |
.forCluster("Test Cluster") | |
.forKeyspace("test1") | |
.withAstyanaxConfiguration(new AstyanaxConfigurationImpl() | |
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE) | |
) | |
.withConnectionPoolConfiguration(new ConnectionPoolConfigurationImpl("MyConnectionPool") | |
.setPort(9160) | |
.setMaxConnsPerHost(1) | |
.setSeeds("127.0.0.1:9160") | |
) | |
.withAstyanaxConfiguration(new AstyanaxConfigurationImpl() | |
.setCqlVersion("3.0.0") | |
.setTargetCassandraVersion("2.0.5")) | |
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor()) | |
.buildKeyspace(ThriftFamilyFactory.getInstance()); | |
context.start(); | |
keyspace = context.getClient(); | |
EMP_CF = ColumnFamily.newColumnFamily( | |
EMP_CF_NAME, | |
LongSerializer.get(), | |
AsciiSerializer.get()); | |
} | |
public void insert(long time) { | |
MutationBatch m = keyspace.prepareMutationBatch(); | |
ColumnListMutation<String> x = | |
m.withRow(EMP_CF, time); | |
for(int i=0;i<10000;i++) | |
x.putColumn(Integer.toString(i), Integer.toString(i)); | |
try { | |
@SuppressWarnings("unused") | |
Object result = m.execute(); | |
} catch (ConnectionException e) { | |
logger.error("failed to write data to C*", e); | |
throw new RuntimeException("failed to write data to C*", e); | |
} | |
logger.debug("insert ok"); | |
} | |
public void createCF() { | |
} | |
public void read(long time) { | |
OperationResult<ColumnList<String>> result; | |
try { | |
result = keyspace.prepareQuery(EMP_CF) | |
.getKey(time) | |
.execute(); | |
ColumnList<String> cols = result.getResult(); | |
// process data | |
// a) iterate over columsn | |
for (Iterator<Column<String>> i = cols.iterator(); i.hasNext(); ) { | |
Column<String> c = i.next(); | |
String v = c.getStringValue(); | |
System.out.println(c.getName() + " " + v); | |
} | |
} catch (ConnectionException e) { | |
logger.error("failed to read from C*", e); | |
throw new RuntimeException("failed to read from C*", e); | |
} | |
} | |
public static void main(String[] args) { | |
AstClient c = new AstClient(); | |
c.init(); | |
long t00 = System.nanoTime(); | |
for(int i=0;i<1000;i++) { | |
long t0 = System.nanoTime(); | |
c.insert(i); | |
long dt = System.nanoTime() - t0; | |
System.out.println((1.0e9/dt) + " " + i); | |
} | |
long dtt = System.nanoTime() - t00; | |
c.read(0); | |
System.out.println(dtt / 1e9); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package cassandra | |
import com.datastax.driver.core._ | |
object CassandraTestMinimized extends App { | |
val keyspace = "test" | |
val table = "wide" | |
val tableName = s"$keyspace.$table" | |
def createKeyspace = s""" | |
CREATE KEYSPACE IF NOT EXISTS ${keyspace} | |
WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } | |
""" | |
def createWideTable = s""" | |
CREATE TABLE IF NOT EXISTS ${tableName} ( | |
time varchar, | |
name varchar, | |
value varchar, | |
PRIMARY KEY (time,name)) | |
WITH COMPACT STORAGE | |
""" | |
def writeTimeNameValue(time: String) = s""" | |
INSERT INTO ${tableName} (time, name, value) | |
VALUES ('$time', ?, ?) | |
""" | |
val cluster = Cluster.builder.addContactPoints("127.0.0.1").build | |
val session = cluster.connect() | |
session.execute(createKeyspace) | |
session.execute(createWideTable) | |
for(i<-0 until 1000) { | |
val entries = | |
for { | |
i <- 0 until 10000 | |
name = i.toString | |
value = name | |
} yield name -> value | |
val batchPreparedStatement = writeMap(i, entries) | |
val t0 = System.nanoTime() | |
session.execute(batchPreparedStatement) | |
val dt = System.nanoTime() - t0 | |
println(i + " " + (dt/1.0e9)) | |
} | |
def writeMap(time: Long, update: Seq[(String, String)]) : BatchStatement = { | |
val template = session | |
.prepare(writeTimeNameValue(time.toString)) | |
.setConsistencyLevel(ConsistencyLevel.ONE) | |
val batch = new BatchStatement(BatchStatement.Type.UNLOGGED) | |
for ((k, v) <- update) | |
batch.add(template.bind(k, v)) | |
batch | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment