Skip to content

Instantly share code, notes, and snippets.

@bradgessler
Created March 17, 2023 21:09
Show Gist options
  • Save bradgessler/02b201c1cc94c24836195db762ada02d to your computer and use it in GitHub Desktop.
Save bradgessler/02b201c1cc94c24836195db762ada02d to your computer and use it in GitHub Desktop.
Downloads a gem and dumps its documents into a Sqlite3 file
# gem-to-sqlite.rb
#
# Usage: ruby gem-to-sqlite.rb <gem_name> <gem_version>
# Example: ruby gem-to-sqlite.rb rake 13.0.6
require 'bundler/inline'
gemfile do
source 'https://rubygems.org'
gem 'yard', '~> 0.9.28'
gem 'sqlite3', '~> 1.4'
end
require 'yard'
require 'sqlite3'
require 'net/http'
require 'tmpdir'
require 'rubygems/package'
class GemToSqlite
def initialize(gem_name, gem_version)
@gem_name = gem_name
@gem_version = gem_version
end
def download_gem
gem_url = "https://rubygems.org/downloads/#{@gem_name}-#{@gem_version}.gem"
uri = URI(gem_url)
file_path = File.join(Dir.tmpdir, "#{@gem_name}-#{@gem_version}.gem")
File.write(file_path, Net::HTTP.get(uri))
file_path
end
def extract_gem(file_path)
spec = nil
gem_dir = File.join(Dir.tmpdir, "#{@gem_name}-#{@gem_version}")
# Extract the gem to a temporary directory
Gem::Package.new(file_path).extract_files(gem_dir)
spec = Gem::Package.new(file_path).spec
# Parse the gem documentation using YARD
YARD::Registry.clear
Dir.glob(File.join(gem_dir, '**', '*.rb')).each do |file|
YARD::Parser::SourceParser.parse(file)
end
spec
end
def create_database
db = SQLite3::Database.new("#{@gem_name}-#{@gem_version}.db")
db.execute <<-SQL
CREATE TABLE IF NOT EXISTS gem_spec (
id INTEGER PRIMARY KEY,
name TEXT,
version TEXT
);
SQL
db.execute <<-SQL
CREATE TABLE IF NOT EXISTS gem_docs (
id INTEGER PRIMARY KEY,
path TEXT,
type TEXT,
doc TEXT
);
SQL
db
end
def store_gem_spec(db, spec)
db.execute("INSERT INTO gem_spec (name, version) VALUES (?, ?)", [@gem_name, @gem_version])
end
def store_docs(db)
YARD::Registry.all.each do |object|
path = object.path
type = object.type.to_s
doc = object.docstring.to_raw
db.execute("INSERT INTO gem_docs (path, type, doc) VALUES (?, ?, ?)", [path, type, doc])
end
end
def run
gem_path = download_gem
spec = extract_gem(gem_path)
db = create_database
store_gem_spec(db, spec)
store_docs(db)
puts "Documentation has been stored in #{@gem_name}-#{@gem_version}.db"
end
end
if ARGV.length != 2
puts "Usage: ruby gem-to-sqlite.rb <gem_name> <gem_version>"
exit 1
end
gem_name = ARGV[0]
gem_version = ARGV[1]
gem_to_sqlite = GemToSqlite.new(gem_name, gem_version)
gem_to_sqlite.run
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment