Skip to content

Instantly share code, notes, and snippets.

@thomasjungblut
Created October 19, 2012 09:39
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save thomasjungblut/3917183 to your computer and use it in GitHub Desktop.
Save thomasjungblut/3917183 to your computer and use it in GitHub Desktop.
Profiling helper for disk serialization
import java.text.NumberFormat;
import org.apache.hadoop.io.IntWritable;
import de.jungblut.datastructure.DiskList;
import de.jungblut.datastructure.PrefetchCache;
public class DiskSerializationProfile {
static final NumberFormat NUMBER_FORMAT = NumberFormat.getNumberInstance();
public static void main(String[] args) throws Exception {
NUMBER_FORMAT.setMaximumFractionDigits(2);
int megaBytesToTarget = 512;
long size = (megaBytesToTarget * 1024 * 1024) / 4;
int its = 50000;
for (int i = 0; i < its; i++) {
IntWritable instance = new IntWritable();
long start = System.currentTimeMillis();
try (DiskList<IntWritable> diskList = new DiskList<>("C:/tmp/list" + i + ".bin")) {
for (int x = 0; x < size; x++) {
instance.set(x);
diskList.add(instance);
}
diskList.openRead();
long finish = System.currentTimeMillis();
time("Written", megaBytesToTarget, start, finish);
start = System.currentTimeMillis();
try (PrefetchCache<IntWritable> cache = new PrefetchCache<>(diskList,
IntWritable.class, 10000)) {
int inc = 0;
IntWritable poll = null;
while ((poll = cache.poll()) != null) {
if (inc++ != poll.get()) {
throw new RuntimeException("FAIL!");
}
}
}
finish = System.currentTimeMillis();
time("Read", megaBytesToTarget, start, finish);
}
System.out.println(i + "/" + its);
}
}
public static void time(String msg, int megaBytesToTarget, long start, long finish) {
System.out.println(msg + " " + megaBytesToTarget + "mb in " + (finish - start)
+ "ms! That is "
+ NUMBER_FORMAT.format((megaBytesToTarget / ((finish - start) / 1000d))) + "mb/s!");
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment