-
-
Save y8/26650196c1ceaeb52cb7 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "benchmark" | |
require "./index" | |
$time = Time.now | |
def time(from = $time) | |
"%.1fs" % ((Time.now-from)) | |
end | |
index = Gem::Index.new | |
print "Reading marshal..." | |
deps = Marshal.load(open("full_index.marshal")) | |
puts " OK (#{time})" | |
$time = Time.now | |
deps.flatten.each do |dep| | |
index.add dep[:name], dep[:number], dep[:platform], dep[:dependencies] | |
end | |
puts "Index build in #{time}" | |
puts "Writing full_index in msgpack" | |
File.open("full_index.msg.gz", "w+b") do |f| | |
f.write Gem.gzip(index.to_msgpack) | |
f.close | |
end | |
puts "Index summary for #{index.size} versions:" | |
puts " Gems: #{index.gems.count}" | |
puts " Version dict: #{index.versions.count}" | |
puts " Requirements dict: #{index.requirements.count}" | |
puts " Platforms dict: #{index.platforms.count}" | |
print "Sainty check: " | |
names = Marshal.load(Gem.gunzip(File.read("specs.4.8.gz")).map(&:first).uniq | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "open-uri" | |
require "index" | |
$time = Time.now | |
def time(from = $time) | |
"%.1fms" % ((Time.now-from) * 1000) | |
end | |
index = Gem::Index.new | |
raw = [] | |
gems = Marshal.load(Gem.gunzip(open("https://s3.amazonaws.com/production.s3.rubygems.org/specs.4.8.gz"))) | |
gems = gems.map(&:first).uniq | |
gems.each_slice(600) do |batch| | |
batch.each_slice(60) do |slice| | |
x = Time.now | |
names = slice.join(",") | |
req = "https://bundler.rubygems.org/api/v1/dependencies?gems=#{names}" | |
print "GET #{req}" | |
deps = Marshal.load(open(req)) | |
print " OK (#{time x})" | |
raw << deps | |
deps.each do |dep| | |
index.add dep[:name], dep[:number], dep[:platform], dep[:dependencies] | |
end | |
puts " indexied in #{time x}" | |
end | |
puts "Index summary for #{index.size} versions:" | |
puts " Gems: #{index.gems.count}" | |
puts " Version dict: #{index.versions.count}" | |
puts " Requirements dict: #{index.requirements.count}" | |
puts " Platforms dict: #{index.platforms.count}" | |
end | |
puts "All stuff is downloaded." | |
puts "Index summary for #{index.size} versions:" | |
puts " Gems: #{index.gems.count}" | |
puts " Version dict: #{index.versions.count}" | |
puts " Requirements dict: #{index.requirements.count}" | |
puts " Platforms dict: #{index.platforms.count}" | |
puts "Writing full_index in msgpack" | |
File.open("full_index.msg.gz", "w+b") do |f| | |
f.write Gem.gzip(index.to_msgpack) | |
f.close | |
end | |
puts "Writing full_index in marshal" | |
File.open("full_index.4.8.gz", "w+b") do |f| | |
f.write Gem.gzip(Marshal.dump(raw.flatten)) | |
f.close | |
end | |
puts "Done." | |
# File.open("specs.json.gz", "w+b") do |f| | |
# f.write Gem.gzip(gems.to_json) | |
# f.close | |
# end | |
# | |
# File.open("specs.msg.gz", "w+b") do |f| | |
# objects = Oj.load(Gem.gunzip(File.read("specs.json.gz"))) | |
# f.write Gem.gzip(objects.to_msgpack) | |
# f.close | |
# end | |
# | |
# File.open("specs.msg", "w+b") do |f| | |
# objects = Oj.load(Gem.gunzip(File.read("specs.json.gz"))) | |
# f.write objects.tьфеуo_msgpack | |
# f.close | |
# end | |
# | |
# Benchmark.bmbm do |x| | |
# x.report("Marshal.load") { 10.times { Marshal.load(Gem.gunzip(File.read("specs.4.8.gz"))) }} | |
# x.report("Oz.load (JSON)") { 10.times { Oj.load(Gem.gunzip(File.read("specs.json.gz"))) }} | |
# x.report("MsgPackage.unpack") { 10.times { MessagePack.unpack(Gem.gunzip(File.read("specs.msg.gz"))) }} | |
# end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment