build.rb

require "benchmark"
require "./index"

$time = Time.now
def time(from = $time)
  "%.1fs" % ((Time.now-from))
end

index = Gem::Index.new

print "Reading marshal..."

deps = Marshal.load(open("full_index.marshal"))

puts " OK (#{time})"

$time = Time.now

deps.flatten.each do |dep|
  index.add dep[:name], dep[:number], dep[:platform], dep[:dependencies]
end

puts "Index build in #{time}"

puts "Writing full_index in msgpack"

File.open("full_index.msg.gz", "w+b") do |f|
   f.write Gem.gzip(index.to_msgpack)
   f.close
end

puts "Index summary for #{index.size} versions:"
puts " Gems: #{index.gems.count}"
puts " Version dict: #{index.versions.count}"
puts " Requirements dict: #{index.requirements.count}"
puts " Platforms dict: #{index.platforms.count}"

print "Sainty check: "

names = Marshal.load(Gem.gunzip(File.read("specs.4.8.gz")).map(&:first).uniq

fetch.rb

require "open-uri"
require "index"

$time = Time.now
def time(from = $time)
  "%.1fms" % ((Time.now-from) * 1000)
end

index = Gem::Index.new
raw = []

gems = Marshal.load(Gem.gunzip(open("https://s3.amazonaws.com/production.s3.rubygems.org/specs.4.8.gz")))

gems = gems.map(&:first).uniq
gems.each_slice(600) do |batch|
  batch.each_slice(60) do |slice|
    x = Time.now
    names = slice.join(",")

    req = "https://bundler.rubygems.org/api/v1/dependencies?gems=#{names}"
    print "GET #{req}"

    deps = Marshal.load(open(req))

    print " OK (#{time x})"
    raw << deps

    deps.each do |dep|
      index.add dep[:name], dep[:number], dep[:platform], dep[:dependencies]
    end

    puts " indexied in #{time x}"
  end

  puts "Index summary for #{index.size} versions:"
  puts " Gems: #{index.gems.count}"
  puts " Version dict: #{index.versions.count}"
  puts " Requirements dict: #{index.requirements.count}"
  puts " Platforms dict: #{index.platforms.count}"
end

puts "All stuff is downloaded."


puts "Index summary for #{index.size} versions:"
puts " Gems: #{index.gems.count}"
puts " Version dict: #{index.versions.count}"
puts " Requirements dict: #{index.requirements.count}"
puts " Platforms dict: #{index.platforms.count}"

puts "Writing full_index in msgpack"

File.open("full_index.msg.gz", "w+b") do |f|
   f.write Gem.gzip(index.to_msgpack)
   f.close
end

puts "Writing full_index in marshal"

File.open("full_index.4.8.gz", "w+b") do |f|
   f.write Gem.gzip(Marshal.dump(raw.flatten))
   f.close
end

puts "Done."

# File.open("specs.json.gz", "w+b") do |f|
#   f.write Gem.gzip(gems.to_json)
#   f.close
# end
#
# File.open("specs.msg.gz", "w+b") do |f|
#    objects = Oj.load(Gem.gunzip(File.read("specs.json.gz")))
#    f.write Gem.gzip(objects.to_msgpack)
#    f.close
# end
#
# File.open("specs.msg", "w+b") do |f|
#    objects = Oj.load(Gem.gunzip(File.read("specs.json.gz")))
#    f.write objects.tьфеуo_msgpack
#    f.close
# end
#
# Benchmark.bmbm do |x|
#   x.report("Marshal.load")      { 10.times { Marshal.load(Gem.gunzip(File.read("specs.4.8.gz")))       }}
#   x.report("Oz.load (JSON)")    { 10.times { Oj.load(Gem.gunzip(File.read("specs.json.gz")))           }}
#   x.report("MsgPackage.unpack") { 10.times { MessagePack.unpack(Gem.gunzip(File.read("specs.msg.gz"))) }}
# end

full_index.4.8.gz

index.rb

play.rb