Ruby-based Benchmark of MessagePack vs. JSON vs. Yajl vs. Protobuffers vs. MultiJson vs. Marshal vs. YAML vs. BSON

.rvmrc

rvm --create ree@benchmarks

decode.rb

# encoding: utf-8

require 'rubygems'
require 'bundler'
require 'bench_press'
require 'multi_json'
require 'json'
require 'yajl'
require 'msgpack'
require './repository.pb'
require 'yaml'
require 'bson'

extend BenchPress
reps 10_000

data = {
  "repository" => {
    "watchers" => 173,
    "has_wiki" => true,
    "url" => "https://github.com/flori/json",
    "open_issues" => 21,
    "homepage" => "http://flori.github.com/json",
    "has_issues" => true,
    "forks" => 35,
    "fork" => false,
    "language" => "Ruby",
    "integrate_branch" => "master",
    "created_at" => "2009/08/24 15:21:39 -0700",
    "master_branch" => "master",
    "size" => 572,
    "private" => false,
    "name" => "json",
    "owner" => "flori",
    "has_downloads" => true,
    "pushed_at" => "2011/07/08 07:34:34 -0700",
    "description" => "JSON implementation for Ruby"
  }
}

json = Yajl::Encoder.encode(data)
msgpack = MessagePack.pack(data)
protobuffer = Repository.new(data['repository'])
yaml = YAML.dump(data)
marshal = Marshal.dump(data)
bson = BSON::BSON_C.serialize(data)

MultiJson.engine = :json_gem
measure "MultiJson.decode (json_gem)" do
  MultiJson.decode(json)
end

MultiJson.engine = :json_pure
measure "MultiJson.decode (json_pure)" do
  MultiJson.decode(json)
end

MultiJson.engine = :ok_json
measure "MultiJson.decode (ok_json)" do
  MultiJson.decode(json)
end

MultiJson.engine = :yajl
measure "MultiJson.decode (yajl)" do
  MultiJson.decode(json)
end

measure "JSON.parse" do
  JSON.parse(json)
end

measure "Yajl::Parser.parse" do
  Yajl::Parser.parse(json)
end

measure "MessagePack.unpack" do
  MessagePack.unpack(msgpack)
end

measure "Protobuffer.parse" do
  Repository.parse(protobuffer.to_s)
end

#measure "YAML.load" do
#  YAML.load(yaml)
#end

measure "Marshal.load" do
  Marshal.load(marshal)
end

measure "BSON_C.deserialize" do
  BSON::BSON_C.deserialize(bson)
end

#measure "BSON_RUBY.deserialize" do
#  BSON::BSON_RUBY.deserialize(bson)
#end

decode_result.txt

Decode
======
Date: October 20, 2011

System Information
------------------
    Operating System:    Ubuntu 11.04
    CPU:                 AMD Phenom(tm) II X6 1055T Processor
    Processor Count:     6
    Memory:              7.81 GB


--------------------------------------------------------------------------------
 - ruby 1.9.2p290 (2011-07-09 revision 32553) [x86_64-linux]
--------------------------------------------------------------------------------

"MessagePack.unpack" is up to 98% faster over 10,000 repetitions:

    MessagePack.unpack              0.10505175590515137 secs    Fastest
    Marshal.load                    0.2254018783569336  secs    53% Slower
    Yajl::Parser.parse              0.23709726333618164 secs    55% Slower
    JSON.parse                      0.23826956748962402 secs    55% Slower
    MultiJson.decode (yajl)         0.2485332489013672  secs    57% Slower
    MultiJson.decode (json_gem)     0.24927568435668945 secs    57% Slower
    MultiJson.decode (ok_json)      0.25182557106018066 secs    58% Slower
    MultiJson.decode (json_pure)    0.25347399711608887 secs    58% Slower
    BSON_C.deserialize              0.4782674312591553  secs    78% Slower
    Protobuffer.parse               2.7903316020965576  secs    96% Slower
    BSON_RUBY.deserialize           4.0376081466674805  secs    97% Slower
    YAML.load                       5.904003620147705   secs    98% Slower


--------------------------------------------------------------------------------
 - ruby 1.8.7 (2011-06-30 patchlevel 352) [x86_64-linux]
--------------------------------------------------------------------------------

"MessagePack.unpack" is up to 98% faster over 10,000 repetitions:

    MessagePack.unpack              0.135149002075195 secs    Fastest
    Marshal.load                    0.186592102050781 secs    27% Slower
    JSON.parse                      0.244688034057617 secs    44% Slower
    Yajl::Parser.parse              0.270131826400757 secs    49% Slower
    MultiJson.decode (yajl)         0.289369106292725 secs    53% Slower
    MultiJson.decode (ok_json)      0.290427923202515 secs    53% Slower
    MultiJson.decode (json_pure)    0.291253805160522 secs    53% Slower
    MultiJson.decode (json_gem)     0.291429042816162 secs    53% Slower
    YAML.load                       0.75546407699585  secs    82% Slower
    BSON_C.deserialize              2.18496608734131  secs    93% Slower
    Protobuffer.parse               5.09628200531006  secs    97% Slower
    BSON_RUBY.deserialize           10.4131319522858  secs    98% Slower


--------------------------------------------------------------------------------
 - ruby 1.8.7 (2011-02-18 patchlevel 334) [x86_64-linux], MBARI 0x6770,
   Ruby Enterprise Edition 2011.03
--------------------------------------------------------------------------------

"MessagePack.unpack" is up to 99% faster over 10,000 repetitions:

    MessagePack.unpack              0.088737964630127 secs    Fastest
    Marshal.load                    0.162353992462158 secs    45% Slower
    JSON.parse                      0.197911977767944 secs    55% Slower
    Yajl::Parser.parse              0.219813823699951 secs    59% Slower
    MultiJson.decode (json_gem)     0.234611988067627 secs    62% Slower
    MultiJson.decode (yajl)         0.237901926040649 secs    62% Slower
    MultiJson.decode (json_pure)    0.238424062728882 secs    62% Slower
    MultiJson.decode (ok_json)      0.23918890953064  secs    62% Slower
    YAML.load                       0.611996173858643 secs    85% Slower
    BSON_C.deserialize              2.04893088340759  secs    95% Slower
    Protobuffer.parse               4.42018413543701  secs    97% Slower
    BSON_RUBY.deserialize           9.160728931427    secs    99% Slower


--------------------------------------------------------------------------------
 - rubinius 2.0.0dev (1.8.7 be002873 yyyy-mm-dd JI) [x86_64-unknown-linux-gnu]
--------------------------------------------------------------------------------

"Marshal.load" is up to 77% faster over 10,000 repetitions:

    Marshal.load                    0.7221729755401611 secs    Fastest
    MessagePack.unpack              0.7341330051422119 secs    1% Slower
    JSON.parse                      1.0028469562530518 secs    27% Slower
    BSON_C.deserialize              1.1071460247039795 secs    34% Slower
    MultiJson.decode (json_gem)     1.2033860683441162 secs    39% Slower
    MultiJson.decode (json_pure)    1.2075448036193848 secs    40% Slower
    Yajl::Parser.parse              1.2082669734954834 secs    40% Slower
    MultiJson.decode (yajl)         1.2262511253356934 secs    41% Slower
    MultiJson.decode (ok_json)      1.234100103378296  secs    41% Slower
    Protobuffer.parse               2.629127025604248  secs    72% Slower
    BSON_RUBY.deserialize           3.036700963973999  secs    76% Slower
    YAML.load                       3.1432650089263916 secs    77% Slower

encode.rb

# encoding: utf-8

require 'rubygems'
require 'bundler'
require 'bench_press'
require 'multi_json'
require 'json'
require 'yajl'
require 'msgpack'
require './repository.pb'
require 'yaml'
require 'bson'

extend BenchPress
reps 10_000

data = {
  "repository" => {
    "watchers" => 173,
    "has_wiki" => true,
    "url" => "https://github.com/flori/json",
    "open_issues" => 21,
    "homepage" => "http://flori.github.com/json",
    "has_issues" => true,
    "forks" => 35,
    "fork" => false,
    "language" => "Ruby",
    "integrate_branch" => "master",
    "created_at" => "2009/08/24 15:21:39 -0700",
    "master_branch" => "master",
    "size" => 572,
    "private" => false,
    "name" => "json",
    "owner" => "flori",
    "has_downloads" => true,
    "pushed_at" => "2011/07/08 07:34:34 -0700",
    "description" => "JSON implementation for Ruby"
  }
}

# MultiJson json_gem

MultiJson.engine = :json_gem

measure "MultiJson.encode (json_gem)" do
  MultiJson.encode(data)
end

# MultiJson json_pure

MultiJson.engine = :json_pure

measure "MultiJson.encode (json_pure)" do
  MultiJson.encode(data)
end

# MultiJson ok_json

MultiJson.engine = :ok_json

measure "MultiJson.encode (ok_json)" do
  MultiJson.encode(data)
end

# MultiJson yajl

MultiJson.engine = :yajl

measure "MultiJson.encode (yajl)" do
  MultiJson.encode(data)
end

# JSON

measure "JSON.generate" do
  JSON.generate(data)
end

# Yajl

measure "Yajl::Encoder.encode" do
  Yajl::Encoder.encode(data)
end

# MessagePack

measure "MessagePack.pack" do
  MessagePack.pack(data)
end

# ProtocolBuffers

measure "Protobuffer.new" do
  Repository.new(data['repository'])
end

# YAML

#measure "YAML.dump" do
#  YAML.dump(data)
#end

# Marshal

measure "Marshal.dump" do
  Marshal.dump(data)
end

# BSON C

measure "BSON_C.serialize" do
  BSON::BSON_C.serialize(data)
end

# BSON Ruby

#measure "BSON_RUBY.serialize" do
#  BSON::BSON_RUBY.serialize(data)
#end

encode_result.txt

Encode
======
Date: October 20, 2011

System Information
------------------
    Operating System:    Ubuntu 11.04
    CPU:                 AMD Phenom(tm) II X6 1055T Processor
    Processor Count:     6
    Memory:              7.81 GB


--------------------------------------------------------------------------------
 - ruby 1.9.2p290 (2011-07-09 revision 32553) [x86_64-linux]
--------------------------------------------------------------------------------

"MessagePack.pack" is up to 99% faster over 10,000 repetitions:

    MessagePack.pack                0.0659017562866211  secs    Fastest
    BSON_C.serialize                0.09089255332946777 secs    27% Slower
    MultiJson.encode (ok_json)      0.12016463279724121 secs    45% Slower
    MultiJson.encode (json_gem)     0.12153267860412598 secs    45% Slower
    MultiJson.encode (json_pure)    0.12209510803222656 secs    46% Slower
    MultiJson.encode (yajl)         0.12253046035766602 secs    46% Slower
    Yajl::Encoder.encode            0.14138484001159668 secs    53% Slower
    Marshal.dump                    0.24850249290466309 secs    73% Slower
    JSON.generate                   0.42482590675354004 secs    84% Slower
    Protobuffer.new                 0.939178466796875   secs    92% Slower
    BSON_RUBY.serialize             2.908997058868408   secs    97% Slower
    YAML.dump                       7.455713987350464   secs    99% Slower


--------------------------------------------------------------------------------
 - ruby 1.8.7 (2011-06-30 patchlevel 352) [x86_64-linux]
--------------------------------------------------------------------------------

"MessagePack.pack" is up to 99% faster over 10,000 repetitions:

    MessagePack.pack                0.0610361099243164 secs    Fastest
    BSON_C.serialize                0.11319899559021   secs    46% Slower
    MultiJson.encode (json_gem)     0.136553049087524  secs    55% Slower
    MultiJson.encode (json_pure)    0.13718581199646   secs    55% Slower
    MultiJson.encode (yajl)         0.137234926223755  secs    55% Slower
    MultiJson.encode (ok_json)      0.137963056564331  secs    55% Slower
    Yajl::Encoder.encode            0.163922071456909  secs    62% Slower
    Marshal.dump                    0.214154958724976  secs    71% Slower
    JSON.generate                   0.215612173080444  secs    71% Slower
    Protobuffer.new                 1.78261613845825   secs    96% Slower
    BSON_RUBY.serialize             6.16872096061707   secs    99% Slower
    YAML.dump                       10.5404698848724   secs    99% Slower


--------------------------------------------------------------------------------
 - ruby 1.8.7 (2011-02-18 patchlevel 334) [x86_64-linux], MBARI 0x6770,
   Ruby Enterprise Edition 2011.03
--------------------------------------------------------------------------------

"MessagePack.pack" is up to 99% faster over 10,000 repetitions:

    MessagePack.pack                0.0586459636688232 secs    Fastest
    BSON_C.serialize                0.107908010482788  secs    45% Slower
    MultiJson.encode (json_gem)     0.128373861312866  secs    54% Slower
    MultiJson.encode (json_pure)    0.130517959594727  secs    55% Slower
    MultiJson.encode (yajl)         0.130743026733398  secs    55% Slower
    MultiJson.encode (ok_json)      0.131932973861694  secs    55% Slower
    Yajl::Encoder.encode            0.148689985275269  secs    60% Slower
    JSON.generate                   0.199151039123535  secs    70% Slower
    Marshal.dump                    0.208939075469971  secs    71% Slower
    Protobuffer.new                 1.46574807167053   secs    95% Slower
    BSON_RUBY.serialize             5.60331797599792   secs    98% Slower
    YAML.dump                       8.42913007736206   secs    99% Slower


--------------------------------------------------------------------------------
 - rubinius 2.0.0dev (1.8.7 be002873 yyyy-mm-dd JI) [x86_64-unknown-linux-gnu]
--------------------------------------------------------------------------------

"MultiJson.encode (json_pure)" is up to 96% faster over 10,000 repetitions:

    MultiJson.encode (json_pure)    0.4232020378112793  secs    Fastest
    MultiJson.encode (yajl)         0.4267880916595459  secs    0% Slower
    MultiJson.encode (json_gem)     0.43001604080200195 secs    1% Slower
    MultiJson.encode (ok_json)      0.4306609630584717  secs    1% Slower
    Yajl::Encoder.encode            0.4938218593597412  secs    14% Slower
    MessagePack.pack                0.5131721496582031  secs    17% Slower
    JSON.generate                   0.5701169967651367  secs    25% Slower
    Protobuffer.new                 0.5858101844787598  secs    27% Slower
    BSON_C.serialize                1.144556999206543   secs    63% Slower
    Marshal.dump                    3.744101047515869   secs    88% Slower
    BSON_RUBY.serialize             4.04749608039856    secs    89% Slower
    YAML.dump                       10.713746070861816  secs    96% Slower

Gemfile

source :rubygems

gem 'bench_press'
gem 'multi_json'
gem 'json'
gem 'yajl-ruby'
gem 'msgpack'
gem 'ruby-protocol-buffers'
gem 'bson'
gem 'bson_ext'

Gemfile.lock

GEM
  remote: http://rubygems.org/
  specs:
    activesupport (3.1.1)
      multi_json (~> 1.0)
    bench_press (0.3.1)
      activesupport (>= 2.3.5)
      facter (>= 1.5.7)
      httparty (>= 0.6.1)
      jeweler (>= 1.4.0)
    bson (1.4.1)
    bson_ext (1.4.1)
    facter (1.6.2)
    git (1.2.5)
    httparty (0.8.1)
      multi_json
      multi_xml
    jeweler (1.6.4)
      bundler (~> 1.0)
      git (>= 1.2.5)
      rake
    json (1.6.1)
    msgpack (0.4.6)
    multi_json (1.0.3)
    multi_xml (0.4.1)
    rake (0.9.2)
    ruby-protocol-buffers (1.2.1)
    yajl-ruby (1.0.0)

PLATFORMS
  ruby

DEPENDENCIES
  bench_press
  bson
  bson_ext
  json
  msgpack
  multi_json
  ruby-protocol-buffers
  yajl-ruby

repository.pb.rb

#!/usr/bin/env ruby
# Generated by the protocol buffer compiler. DO NOT EDIT!

require 'protocol_buffers'

# forward declarations
class Repository < ::ProtocolBuffers::Message; end

class Repository < ::ProtocolBuffers::Message
  optional :uint32, :watchers, 1
  optional :bool, :has_wiki, 2
  optional :string, :url, 3
  optional :uint32, :open_issues, 4
  optional :string, :homepage, 5
  optional :bool, :has_issues, 6
  optional :uint32, :forks, 7
  optional :bool, :fork, 8
  optional :string, :language, 9
  optional :string, :integrate_branch, 10
  optional :string, :created_at, 11
  optional :string, :master_branch, 12
  optional :uint32, :size, 13
  optional :bool, :private, 14
  optional :string, :name, 15
  optional :string, :owner, 16
  optional :bool, :has_downloads, 17
  optional :string, :pushed_at, 18
  optional :string, :description, 19

  gen_methods! # new fields ignored after this point
end

repository.proto

message repository {
  optional uint32 watchers         = 1;
  optional bool   has_wiki         = 2;
  optional string url              = 3;
  optional uint32 open_issues      = 4;
  optional string homepage         = 5;
  optional bool   has_issues       = 6;
  optional uint32 forks            = 7;
  optional bool   fork             = 8;
  optional string language         = 9;
  optional string integrate_branch = 10;
  optional string created_at       = 11;
  optional string master_branch    = 12;
  optional uint32 size             = 13;
  optional bool   private          = 14;
  optional string name             = 15;
  optional string owner            = 16;
  optional bool   has_downloads    = 17;
  optional string pushed_at        = 18;
  optional string description      = 19;
}

sizes.rb

# encoding: utf-8

require 'rubygems'
require 'bundler'
require 'bench_press'
require 'multi_json'
require 'json'
require 'yajl'
require 'msgpack'
require './repository.pb'
require 'yaml'
require 'bson'

data = {
  "repository" => {
    "watchers" => 173,
    "has_wiki" => true,
    "url" => "https://github.com/flori/json",
    "open_issues" => 21,
    "homepage" => "http://flori.github.com/json",
    "has_issues" => true,
    "forks" => 35,
    "fork" => false,
    "language" => "Ruby",
    "integrate_branch" => "master",
    "created_at" => "2009/08/24 15:21:39 -0700",
    "master_branch" => "master",
    "size" => 572,
    "private" => false,
    "name" => "json",
    "owner" => "flori",
    "has_downloads" => true,
    "pushed_at" => "2011/07/08 07:34:34 -0700",
    "description" => "JSON implementation for Ruby"
  }
}

json = Yajl::Encoder.encode(data)
msgpack = MessagePack.pack(data)
protobuffer = Repository.new(data['repository'])
yaml = YAML.dump(data)
marshal = Marshal.dump(data)
bson = BSON::BSON_C.serialize(data)

puts 'Message Sizes:'
puts "JSON:        #{json.to_s.bytesize} bytes"
puts "msgpack:     #{msgpack.to_s.bytesize} bytes"
puts "protobuffer: #{protobuffer.to_s.bytesize} bytes"
puts "yaml:        #{yaml.to_s.bytesize} bytes"
puts "bson:        #{bson.to_s.bytesize} bytes"

sizes.txt

Message Sizes:
JSON:        461 bytes
msgpack:     373 bytes
protobuffer: 204 bytes
yaml:        462 bytes
bson:        448 bytes

sizes.rb computes the Marshal size but doesn't output it. sizes.txt omits it. I ran it locally and got:

marshal:     560 bytes