Created
October 19, 2012 09:39
-
-
Save thomasjungblut/3917183 to your computer and use it in GitHub Desktop.
Profiling helper for disk serialization
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.text.NumberFormat; | |
import org.apache.hadoop.io.IntWritable; | |
import de.jungblut.datastructure.DiskList; | |
import de.jungblut.datastructure.PrefetchCache; | |
public class DiskSerializationProfile { | |
static final NumberFormat NUMBER_FORMAT = NumberFormat.getNumberInstance(); | |
public static void main(String[] args) throws Exception { | |
NUMBER_FORMAT.setMaximumFractionDigits(2); | |
int megaBytesToTarget = 512; | |
long size = (megaBytesToTarget * 1024 * 1024) / 4; | |
int its = 50000; | |
for (int i = 0; i < its; i++) { | |
IntWritable instance = new IntWritable(); | |
long start = System.currentTimeMillis(); | |
try (DiskList<IntWritable> diskList = new DiskList<>("C:/tmp/list" + i + ".bin")) { | |
for (int x = 0; x < size; x++) { | |
instance.set(x); | |
diskList.add(instance); | |
} | |
diskList.openRead(); | |
long finish = System.currentTimeMillis(); | |
time("Written", megaBytesToTarget, start, finish); | |
start = System.currentTimeMillis(); | |
try (PrefetchCache<IntWritable> cache = new PrefetchCache<>(diskList, | |
IntWritable.class, 10000)) { | |
int inc = 0; | |
IntWritable poll = null; | |
while ((poll = cache.poll()) != null) { | |
if (inc++ != poll.get()) { | |
throw new RuntimeException("FAIL!"); | |
} | |
} | |
} | |
finish = System.currentTimeMillis(); | |
time("Read", megaBytesToTarget, start, finish); | |
} | |
System.out.println(i + "/" + its); | |
} | |
} | |
public static void time(String msg, int megaBytesToTarget, long start, long finish) { | |
System.out.println(msg + " " + megaBytesToTarget + "mb in " + (finish - start) | |
+ "ms! That is " | |
+ NUMBER_FORMAT.format((megaBytesToTarget / ((finish - start) / 1000d))) + "mb/s!"); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment