krisskross

Word frequency summation in a nested directory tree

Objective

The primary goal of this task is to assess proficiency in the following areas in Java.

• working with the JDK libraries
• multi-threading and concurrency
• memory safety

krisskross

Keybase proof

I hereby claim:

• I am krisskross on github.
• I am deephacks (https://keybase.io/deephacks) on keybase.
• I have a public key ASCp_y2k54TlnmBhUZn-rpO-vW6XWp7KvcPOGHAeDSxHpAo

To claim this, I am signing this object:

krisskross

    List<ConsumerRecord<byte[], byte[]>> lastRecords = new ArrayList<>();
    try (KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps)) {
      for (PartitionInfo info : consumer.partitionsFor(topic)) {
        List<TopicPartition> list = new ArrayList<>();
        list.add(new TopicPartition(topic, info.partition()));
        consumer.assign(list);
        consumer.seekToEnd(list);
        ConsumerRecord<byte[], byte[]> last = consumer.poll(10000).iterator().next();
        lastRecords.add(last);

krisskross

ByteBuffer seekKey = ByteBuffer.allocateDirect(4).putInt(4);

try (Cursor cursor = db.openCursor(tx)) {
  if (cursor.get(seekKey, MDB_SET_KEY)) {
    // regular users
    ByteBuffer key = cursor.key();
    ByteBuffer val = cursor.val();
    do {
      // advanced users
      long keyAddress = cursor.keyAddress();

krisskross

// Generated by JavaCPP version 1.2: DO NOT EDIT THIS FILE


#ifdef _WIN32
    #define _JAVASOFT_JNI_MD_H_

    #define JNIEXPORT __declspec(dllexport)
    #define JNIIMPORT __declspec(dllimport)
    #define JNICALL __stdcall

krisskross

public class PeriodEntries<T extends Timestamped> implements Iterable<List<T>> {
  private final Iterator<List<T extends Timestamped>> entriesIt;
  private final long interval;
  private PeriodEntries(Iterable<List<T>> entriesIt, long interval) {
    this.entriesIt = entriesIt.iterator();
    this.interval = interval;
  }

  public static <T extends Timestamped> PeriodEntries<T> create(Iterable<List<T>> entriesIt, long interval) {
   return new PeriodEntries<T>(entriesIt, interval);

krisskross

TextRowDecoder decoder = new TextRowDecoder(4, comma);
FileReader<byte[][]> reader = FileReader.create(decoder, file.listFiles());
for (List<byte[][]> chunk : reader) {
  // do something with each chunk
}

krisskross

TrueFxDecoder decoder = new TrueFxDecoder();
FileReader<TrueFxData> reader = FileReader.create(decoder, file.listFiles());
long periodLength = TimeUnit.DAYS.toMillis(1);
PeriodEntries<TrueFxData> periods = PeriodEntries.create(reader, periodLength);
 
for (List<TrueFxData> entries : periods) {
   // data for each day
   for (TrueFxData entry : entries) {
     // process each entry
   }

krisskross

public class FileReader implements Iterable<List<T>> {
  private static final long CHUNK_SIZE = 4096;
  private final Decoder<T> decoder;
  private Iterator<File> files;
 
  private FileReader(Decoder<T> decoder, File... files) {
    this(decoder, Arrays.asList(files));
  }
  private FileReader(Decoder<T> decoder, List<File> files) {
    this.files = files.iterator();

krisskross

public class TextRowDecoder implements Decoder<byte[][]> {
  private static final byte LF = 10;
  private final int numFields;
  private final byte delimiter;
  public TextRowDecoder(int numFields, byte delimiter) {
   this.numFields = numFields;
   this.delimiter = delimiter;
  }
  @Override
  public byte[][] decode(ByteBuffer buffer) {