.gitignore

citylots.json

json-stream.rb

# $ wget https://github.com/zemirco/sf-city-lots-json/raw/master/citylots.json
# $ ruby json-stream.rb citylots.json
# -[2770]% ruby json-stream.rb citylots.json
# citylots.json 189778220
# RSS 7140 2017-07-09 03:57:55 +0900
# "parse completed"
# RSS 20256 2017-07-09 04:00:18 +0900
# ruby json-stream.rb citylots.json  140.90s user 0.97s system 98% cpu 2:23.35 total

require 'json/stream'

filename = ARGV.first
puts "#{filename} #{File.stat(filename).size}"

def memstat
  rss = `ps -o rss -p #{$$}`.split(/\s+/).join(' ').strip
  puts "#{rss} #{Time.now}"
end

memstat

io = open(filename)
parser = JSON::Stream::Parser.new do
  end_document do
    p 'parse completed'
  end
end
while data = io.read(4096)
  parser << data
end

memstat

yajl.rb

# $ wget https://github.com/zemirco/sf-city-lots-json/raw/master/citylots.json
# $ ruby yajl.rb citylots.json
# -[2769]% ruby yajl.rb citylots.json
# citylots.json 189778220
# RSS 7148 2017-07-09 03:55:16 +0900
# "parse completed"
# RSS 776948 2017-07-09 03:55:24 +0900
# ruby yajl.rb citylots.json  8.24s user 0.64s system 98% cpu 9.028 total

require 'yajl'

filename = ARGV.first
puts "#{filename} #{File.stat(filename).size}"

def memstat
  rss = `ps -o rss -p #{$$}`.split(/\s+/).join(' ').strip
  puts "#{rss} #{Time.now}"
end

memstat

io = open(filename)
parser = Yajl::Parser.new
parser.on_parse_complete = proc { |obj|
  p 'parse completed'
}
while data = io.read(4096)
  parser << data
end

memstat