Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Performance experiment for apache/accumulo#275 and ACCUMULO-4667
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.LinkedHashSet;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.rfile.RFile;
import org.apache.accumulo.core.client.rfile.RFileWriter;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.io.Text;
public class LocGroupPerfTest {
public static void main(String[] args) throws Exception {
String filename = createFile(100, 10, 10000, 20);
scan(filename, 5, 340);
}
private static String toRow(int r) {
return String.format("r%08d", r);
}
private static String toFamily(int f) {
return String.format("f%04d", f);
}
private static String toQual(int q) {
return String.format("q%04d", q);
}
private static void scan(String filename, int ... fams) throws Exception {
Scanner scanner = RFile.newScanner().from(filename).build();
for (int f : fams) {
scanner.fetchColumnFamily(new Text(toFamily(f)));
}
for(int i = 0; i < 100; i++) {
int count = 0;
long t1 = System.currentTimeMillis();
for (Entry<Key,Value> entry : scanner) {
count++;
}
long t2 = System.currentTimeMillis();
System.out.println(count+" "+(t2 -t1));
}
}
private static String createFile(int rows, int groups, int families, int quals) throws Exception {
String filename = String.format("test-r%d-g%d-f%d-q%d.rf", rows, groups, families, quals);
if(new File(filename).exists()) {
return filename;
}
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(filename));
RFileWriter writer = RFile.newWriter().to(out).build();
int famsPerGroup = families / groups;
for(int g = 0; g < groups; g++) {
Set<String> familySet = new LinkedHashSet<>();
if(g == groups - 1) {
for(int f = g * famsPerGroup; f < families; f++) {
familySet.add(toFamily(f));
}
writer.startDefaultLocalityGroup();
} else {
for(int f = g * famsPerGroup; f < (g+1)*famsPerGroup; f++) {
familySet.add(toFamily(f));
}
writer.startNewLocalityGroup("lg-"+g, familySet);
}
//System.out.println("group : " + g);
int count = 0;
for (int r = 0; r < rows; r++) {
for (String family : familySet) {
for (int q = 0; q < quals; q++) {
Key k = new Key(toRow(r), family, toQual(q), "", 42);
writer.append(k, k.hashCode() + "");
count++;
// System.out.printf(" %s %s %s\n", toRow(r), family, toQual(q));
}
}
}
System.out.println("added "+count+" keys");
}
writer.close();
return filename;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.