Minimalist append-only key/value store written in Ruby.

README.md

A minimalist append-only key/value store written in Ruby.

Ruby API

c = Collection.new('example.db')

c.set('hello', 'world')
c.get('hello') # => "world"

c.delete('hello')
c.get('hello') # => nil

c.set('foo', {'baz' => 'qux'})
c.get('foo')['baz'] # => "qux"

c.set('bar', [1,2,3])
c.get('bar').first # => 1

c.keys # => ["foo", "bar"]

Every update to the collection results in a JSON object being appended to the data file (example.db in the case above). Yes, even on deletes. Indices are kept in memory for speedy reads and the file can periodically be compacted to keep it from getting too unwieldy in size.

c.compact

Socket Interface

Included is a simple socket server interface built with EventMachine. Start listening on a given port and writing to the specified data file by running:

$ ruby server.rb 3456 example.db

Example client usage:

$ telnet localhost 3456
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
get foo
{"baz":"qux"}
delete foo
OK
set hello world
OK
get hello
"world"
keys
["bar","hello"]
compact
OK
exit

server.rb

require 'json'

class Handlers
  class << self
    def handlers
      @handlers ||= {}
    end
    
    def on(command, respond = false, &blk)
      handlers[command.to_s] = Proc.new do |args|
        begin
          result = instance_exec(*args, &blk)
          reply(respond ? result.to_json : 'OK')
        rescue Exception
          reply('ERROR')
        end
      end
    end
    
    def handler(command)
      handlers[command] || Proc.new { reply('Unrecognized command') }
    end
  end
end

class CollectionHandlers < Handlers
  on(:get, true) { |key| @collection.get(key) }
  on(:set) { |key, *value| @collection.set(key, value.join(' ')) }
  on(:delete) { |key| @collection.delete(key) }
  on(:keys, true) { @collection.keys }
  on(:compact) { @collection.compact }
  on(:exit) { close_connection_after_writing }
end

module Server
  def initialize(collection)
    @collection = collection
  end
  
  def reply(data)
    send_data("#{data}\n")
  end
  
  def receive_data(data)
    command, *args = data.split(' ')
    handler = CollectionHandlers::handler(command.downcase)
    instance_exec(args, &handler)
  end
end

if __FILE__ == $0
  $: << File.dirname(__FILE__)
  
  require 'eventmachine'
  require 'store'
  
  EventMachine::run do
    port = ARGV[0]
    collection = Collection.new(ARGV[1])
    EventMachine::start_server('0.0.0.0', port, Server, collection)
    puts "Listening on port #{port}..."
  end
end

store.rb

require 'json'

class Storage
  attr_reader :path
  
  def initialize(path)
    @path = path
    File.open(path, 'a')
  end
  
  def read(position, length)
    JSON.parse(IO.read(@path, length, position))
  end
  
  def write(record)
    File.open(@path, 'a') do |f|
      [f.size, f.write("#{record.to_json}\n")]
    end
  end
  
  def each
    File.open(@path) do |f|
      position = 0
      f.each do |line|
        record = JSON.parse(line.chomp)
        length = line.length
        yield(record, position, length)
        position += length
      end
    end
  end
end

class Collection
  def initialize(path)
    @index = {}
    @storage = Storage.new(path)
    sync
  end
  
  def get(key)
    indexed = @index[key]
    if indexed
      @storage.read(indexed[:position], indexed[:length])['value']
    end
  end
  
  def set(key, value)
    position, length = @storage.write('key' => key, 'value' => value)
    index(key, position, length)
  end
  
  def delete(key)
    if @index[key]
      @storage.write('key' => key, 'deleted' => true)
      @index.delete(key)
    end
  end
  
  def keys
    @index.keys
  end
  
  def sync
    @index.clear
    @storage.each do |record, position, length|
      if record['deleted']
        @index.delete(record['key'])
      else
        index(record['key'], position, length)
      end
    end
  end
  
  def compact
    compacted = Storage.new(@storage.path + '.compact')
    indices = keys.map do |key|
      position, length = compacted.write('key' => key, 'value' => get(key))
      [key, position, length]
    end
    File.rename(compacted.path, @storage.path)
    @index.clear
    indices.each { |i| index(*i) }
  end
  
  private
  
  def index(key, position, length)
    @index[key] = { :position => position, :length => length }
  end
end