Script used to benchmark Valium

results.txt

============================================================
Benchmarking with Ruby 1.9.3 and ActiveRecord 3.2.0.beta
============================================================
              user     system      total        real
============================================================
RSS     : 31432k (31432k)
Objects : 102195 (102195)
============================================================

SINGLE-VALUE TEST
============================================================
map      23.440000   0.060000  23.500000 ( 23.567513)
============================================================
RSS     : 33392k (1960k)
Objects : 102195 (0)
============================================================
valium    2.720000   0.010000   2.730000 (  2.737492)
============================================================
RSS     : 33392k (0k)
Objects : 102195 (0)
============================================================

MULTI-VALUE TEST
============================================================
map      27.560000   0.090000  27.650000 ( 27.721982)
============================================================
RSS     : 33392k (0k)
Objects : 102195 (0)
============================================================
valium    5.340000   0.010000   5.350000 (  5.370545)
============================================================
RSS     : 33392k (0k)
Objects : 102195 (0)
============================================================

MULTI-VALUE TEST WITH SERIALIZATION
============================================================
map     125.190000   8.630000 133.820000 (134.364609)
============================================================
RSS     : 64752k (31360k)
Objects : 102195 (0)
============================================================
valium   87.010000   4.940000  91.950000 ( 92.223318)
============================================================
RSS     : 85180k (20428k)
Objects : 102195 (0)
============================================================

valium-benchmark.rb

#!/usr/bin/env ruby
#
# Benchmarking script for Valium (http://github.com/ernie/valium)

require 'benchmark'
require 'valium'

puts '=' * 60 + "\n"
puts "Benchmarking with Ruby #{RUBY_VERSION} and ActiveRecord #{ActiveRecord::VERSION::STRING}\n"
puts '=' * 60 + "\n"

class Person < ActiveRecord::Base
  serialize :extra_info
end

def display_mem_stats
  @prev_rss = @current_rss || 0
  @current_rss = `ps -o rss= -p #{$$}`.to_i
  @prev_obj_count = @current_obj_count || 0
  @current_obj_count = ObjectSpace.count_objects[:TOTAL]
  puts "=" * 60
  puts "RSS     : #{@current_rss}k (#{@current_rss - @prev_rss}k)"
  puts "Objects : #{@current_obj_count} (#{@current_obj_count - @prev_obj_count})"
  puts "=" * 60
end

def verify_equality(first, second)
  unless first == second
    raise "Comparing apples to oranges, are we?"
  end
end

def setup_database
  ActiveRecord::Base.establish_connection(
    :adapter  => 'sqlite3',
    :database => ':memory:',
  )

  ActiveRecord::Base.silence do
    ActiveRecord::Migration.verbose = false

    ActiveRecord::Schema.define do

      create_table :people, :force => true do |t|
        t.string   :first_name
        t.string   :last_name
        t.integer  :age
        t.text     :extra_info
        t.timestamps
      end

    end
  end

  1.upto(1000) do |num|
    Person.create! :first_name => "Person", :last_name => "Number#{num}", :age => num % 99,
                   :extra_info => {:a_key => "Value Number #{num}"}
  end
end

setup_database

Benchmark.bm(7) do |x|

  display_mem_stats

  if !ARGV[0] || ARGV[0] == 'single'
    puts "\nSINGLE-VALUE TEST\n#{'=' * 60}"

    verify_equality Person.select('id').map(&:id),
                    Person[:id]

    x.report("map") do
      1000.times do
        Person.select('id').map(&:id)
      end
    end

    display_mem_stats

    x.report("valium") do
      1000.times do
        Person[:id]
      end
    end

    display_mem_stats
  end

  if !ARGV[0] || ARGV[0] == 'multiple'
    puts "\nMULTI-VALUE TEST\n#{'=' * 60}"

    verify_equality Person.select('last_name, age').map {|p| [p.last_name, p.age]},
                    Person[:last_name, :age]

    x.report("map") do
      1000.times do
        Person.select('last_name, age').map {|p| [p.last_name, p.age]}
      end
    end

    display_mem_stats

    x.report("valium") do
      1000.times do
        Person[:last_name, :age]
      end
    end

    display_mem_stats
  end

  if !ARGV[0] || ARGV[0] == 'serialize'
    puts "\nMULTI-VALUE TEST WITH SERIALIZATION\n#{'=' * 60}"

    verify_equality Person.select('last_name, age, extra_info').map {|p| [p.last_name, p.age, p.extra_info]},
                    Person[:last_name, :age, :extra_info]

    x.report("map") do
      1000.times do
        Person.select('last_name, age, extra_info').map {|p| [p.last_name, p.age, p.extra_info]}
      end
    end

    display_mem_stats

    x.report("valium") do
      1000.times do
        Person[:last_name, :age, :extra_info]
      end
    end

    display_mem_stats
  end

end

Why aren't you disabling GC? The AR instances can be GC'd during the test and so the RSS numbers wouldn't actually reflect reality.

I ran it both ways, initially with GC disabled (though I had to reduce iterations to run without killing the system). It drastically increased Valium's lead, as you might expect.

In the end, though, we run apps with GC enabled. I decided that "reality" is to run the bench with GC enabled, as well.