app.d

import leveldb;
import std.stdio;
import leveldb.writebatch;
import std.bitmanip;
import std.datetime;
import std.random;
import std.file;
import std.string;
import std.conv;
import core.memory;

void main()
{
    auto opt = new Options;
    opt.create_if_missing = true;
    opt.compression = 1; //almost no effect
    opt.write_buffer_size = 8  * 1024* 1024;  //has some but limited effect on write speed
    auto wo = new WriteOptions;
    wo.sync = false;
    
    
    auto db = new DB(opt, "/tmp/db");
    
    
    auto batch = new WriteBatch;
        
    //file is <integer_key_as_string><tab><interger_value as string>
    auto filename = "relationships.txt";
    int counter = 0;
    
    TickDuration last;
    StopWatch sw;
    sw.start();
    
    ubyte[16] key_buffer;
 
    //GC.disable();
    
    //empty slice for no value
    Slice empty = Slice();

    foreach(line;  filename.File.byLine() )
    {
      counter++;
        
      // auto values = w.split("\t");
      // long key1 = to!long(values[0]);
      // long key2 = to!long(values[1]);
      line.munch(" \t"); // skip ws
      immutable key1 = line.parse!long;
      line.munch(" \t"); // skip ws
      immutable key2 = line.parse!long;
        
      //by setting the value as the 2nd half of the key we basically simmulate a Set
      key_buffer[0..8] = nativeToLittleEndian(key1);
      key_buffer[8..16] = nativeToLittleEndian(key2);
      batch.put(key_buffer, empty);
         
    
      if (counter % 4_000_000 == 0) {
//          writeln(counter);
            db.write(batch, wo);
            batch.clear();
//          writeln(counter, sw.peek().msecs);
      }
      
      //debug output to see progress
      if (counter % 100_000 == 0) {	
      	writeln(counter, "\t", (sw.peek() - last).msecs);
      	last = sw.peek();
      }    

      
    }
    db.write(batch, wo);
    auto msecs = sw.peek().msecs;
    
    writeln("finished, took ", msecs, " msecs");
    //GC.enable();
        

    
    
    
}