compare `quick/Marshal.4.8/#{full_name}.gemspec.rz` and dependencies stored in db

dependencies-validation.rb

#!/usr/bin/env ruby

require 'rubygems'
require 'json'
require 'net/http'
require 'pry'

ENV['RAILS_ENV'] ||= 'development'
require_relative '../config/environment'

log_file = '/tmp/some.log'
File.open(log_file, 'w') { |file| file.puts('') }

rand_versions = "select * from versions where indexed = true and platform = 'ruby' and id < 100000 order by id;"
# rand_versions = "select * from versions where indexed = true and platform = 'ruby' and id > 100000 order by random() limit 1000;"


@connection = ActiveRecord::Base.connection
versions = @connection.exec_query(rand_versions)
total = versions.count
mis_match = 0
processed = 0

versions.each do |version|
  print format("\r%.2f%% (%d/%d) complete", processed.to_f / total * 100.0, processed, total)
  full_name = version['full_name']
  number = version['number']
  version_id = version['id']
  name = full_name.rpartition('-').first

  spec_uri = URI("https://rubygems.org/quick/Marshal.4.8/#{full_name}.gemspec.rz")
  http = Net::HTTP.new(spec_uri.host, spec_uri.port)
  http.use_ssl = true
  body = http.get(spec_uri.request_uri).body
  spec = Gem::Util.inflate body
  spec_obj = Marshal.load spec

  dev_deps = spec_obj.dependencies.select { |s| s.type == :development }.map(&:name).sort
  run_deps = spec_obj.dependencies.select { |s| s.type == :runtime }.map(&:name).sort

  dev_sql = "select r.name, d.id from dependencies as d join rubygems as r on d.rubygem_id = r.id where d.version_id = #{version_id} and d.scope = 'development';"
  dev_deps_tups = @connection.exec_query(dev_sql)
  db_dev_deps = dev_deps_tups.map{ |d| d['name'] }.sort

  run_sql = "select r.name, d.id from dependencies as d join rubygems as r on d.rubygem_id = r.id where d.version_id = #{version_id} and d.scope = 'runtime';"
  run_deps_tups = @connection.exec_query(run_sql)
  db_run_deps = run_deps_tups.map{ |d| d['name'] }.sort

  msg = ""
  if dev_deps == db_dev_deps
    if run_deps == db_run_deps
      #msg = "spec and db matched for: #{full_name}"
    else
      sub = run_deps - db_run_deps
      no_dep_exists = sub.map { |r| Rubygem.find_by(name: r).nil? }.any?(true)
      next if no_dep_exists

      mis_match+=1
      dep_ids = run_deps_tups.map{ |d| d['id'] }.sort
      msg =<<-MSG
spec and db run deps don't match for: #{full_name}
spec: #{run_deps}
db  : #{db_run_deps}
dep ids: #{dep_ids}
MSG
    end
  else
    sub = dev_deps - db_dev_deps
    no_dep_exists = sub.map { |r| Rubygem.find_by(name: r).nil? }.any?(true)
    next if no_dep_exists

    mis_match+=1
    dep_ids = dev_deps_tups.map{ |d| d['id'] }.sort
    msg =<<-MSG
spec and db run deps don't match for: #{full_name}
spec: #{dev_deps}
db  : #{db_dev_deps}
dep ids: #{dep_ids}
MSG
  end
  processed+=1
  if msg.present?
    puts msg
    File.open(log_file, 'a') { |file| file.write(msg) }
  end
rescue StandardError => e
  processed+=1
  puts "skipping #{full_name}", e.message
end

puts "#{mis_match} mismatched out of #{total}"

some.log

....
spec and db run deps don't match for: easel_helpers-0.3.0
spec: []
db  : ["actionview", "activesupport", "hpricot"]
dep ids: [137121, 137122, 137123]
spec and db run deps don't match for: amazon-ec2-0.5.5
spec: ["xml-simple"]
db  : ["mocha", "perftools.rb", "relevance-rcov", "test-spec", "xml-simple"]
dep ids: [137124, 137125, 137126, 137127, 137128]
spec and db run deps don't match for: glue-0.0.1
spec: ["httparty"]
db  : ["hoe", "httparty", "mocha"]
dep ids: [137137, 137138, 137139]
spec and db run deps don't match for: gruff-0.3.6
spec: []
db  : ["hoe"]
dep ids: [137140]
spec and db run deps don't match for: rack-probe-0.0.3
spec: ["rack", "ruby-dtrace"]
db  : ["rack", "rspec", "ruby-dtrace"]
dep ids: [137148, 137149, 137150]
spec and db run deps don't match for: statsample-0.3.4
spec: ["spreadsheet", "svg-graph"]
db  : ["hoe", "spreadsheet", "svg-graph"]
dep ids: [137153, 137154, 137155]
spec and db run deps don't match for: statsample-optimization-2.0.0
spec: ["statsample"]
db  : ["hoe", "statsample"]
dep ids: [137156, 137157]
spec and db run deps don't match for: virtual_box-0.0.1
spec: []
db  : ["echoe", "flexmock"]
dep ids: [137164, 137165]
spec and db run deps don't match for: win32-eventlog-0.5.2
spec: ["windows-pr"]
db  : ["ptools", "test-unit", "windows-pr"]
dep ids: [137167, 137168, 137169]
spec and db run deps don't match for: win32-file-0.6.2
spec: ["win32-api", "win32-file-stat", "windows-pr"]
db  : ["test-unit", "win32-api", "win32-file-stat", "windows-pr"]
dep ids: [137170, 137171, 137172, 137173]
spec and db run deps don't match for: amalgalite-0.11.0
spec: ["arrayfields"]
db  : ["arrayfields", "configuration", "rake", "rake-compiler", "rspec"]
dep ids: [137218, 137219, 137220, 137221, 137222]
spec and db run deps don't match for: oxmlk-0.1.0
spec: ["activesupport", "libxml-ruby"]
db  : ["activesupport", "libxml-ruby", "rspec"]
dep ids: [137243, 137244, 137245]
spec and db run deps don't match for: grate-handle-0.1.0
spec: ["jeweler", "json", "rspec"]
db  : ["jeweler", "jeweler", "json", "json", "rspec", "rspec"]
dep ids: [137258, 137259, 137260, 142980, 142981, 142982]
spec and db run deps don't match for: grate-handle-0.1.1
spec: ["jeweler", "json", "rspec"]
db  : ["jeweler", "jeweler", "json", "json", "rspec", "rspec"]
dep ids: [137261, 137262, 137263, 142977, 142978, 142979]
spec and db run deps don't match for: sets_uuid-0.1.0
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137270, 142983]
spec and db run deps don't match for: oxmlk-0.2.0
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137278, 142953]
spec and db run deps don't match for: oxmlk-0.2.1
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137285, 142952]
spec and db run deps don't match for: phrk-0.0.0
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137289, 142984]
spec and db run deps don't match for: oxmlk-0.2.2
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137292, 142951]
spec and db run deps don't match for: oxmlk-0.2.3
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137295, 142950]
spec and db run deps don't match for: thrift_amqp_transport-0.1.0
spec: ["hoe"]
db  : ["hoe", "hoe"]
dep ids: [137376, 142985]
spec and db run deps don't match for: flvorflv-0.1.0
spec: ["rr", "rspec"]
db  : ["rr", "rr", "rspec", "rspec"]
dep ids: [137299, 137300, 142986, 142987]
spec and db run deps don't match for: rack-maintenance-0.1.1
spec: ["rspec", "yard"]
db  : ["rspec", "rspec", "yard", "yard"]
dep ids: [137338, 137339, 142992, 142993]
spec and db run deps don't match for: rack-maintenance-0.2.0
spec: ["rspec", "yard"]
db  : ["rspec", "rspec", "yard", "yard"]
dep ids: [137341, 137342, 142990, 142991]
spec and db run deps don't match for: espresso-0.0.4
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137348, 142925]
spec and db run deps don't match for: espresso-0.0.5
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137352, 142924]
spec and db run deps don't match for: espresso-0.0.6
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137356, 142923]
spec and db run deps don't match for: espresso-0.0.7
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137360, 142922]
spec and db run deps don't match for: espresso-0.0.8
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137364, 142921]
spec and db run deps don't match for: espresso-0.0.9
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137368, 142920]
spec and db run deps don't match for: espresso-0.0.10
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137372, 142927]
spec and db run deps don't match for: espresso-0.0.11
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137377, 142926]
spec and db run deps don't match for: fakeweb-1.2.6
spec: ["mocha"]
db  : ["mocha", "mocha"]
dep ids: [137381, 137579]
spec and db run deps don't match for: simpleMailer-0.1.6
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137383, 142914]
spec and db run deps don't match for: rack-maintenance-0.3.0
spec: ["rspec", "yard"]
db  : ["rspec", "rspec", "yard", "yard"]
dep ids: [137403, 137404, 142988, 142989]
spec and db run deps don't match for: espresso-0.0.12
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137410, 142919]
spec and db run deps don't match for: espresso-0.0.14
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137414, 142918]
spec and db run deps don't match for: espresso-0.0.15
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137418, 142917]
spec and db run deps don't match for: espresso-0.0.16
spec: ["thoughtbot-shoulda"]
db  : ["thoughtbot-shoulda", "thoughtbot-shoulda"]
dep ids: [137422, 142916]
spec and db run deps don't match for: oxmlk-0.3.0
spec: ["rspec"]
db  : ["rspec", "rspec"]
dep ids: [137426, 142949]
spec and db run deps don't match for: appengine-tools-0.0.3
spec: ["appengine-jruby-jars", "appengine-rack", "appengine-sdk"]
db  : ["appengine-jruby-jars", "appengine-rack"]
dep ids: [143213, 143215]