Skip to content

Instantly share code, notes, and snippets.

@Chocksy
Created January 6, 2015 10:06
Show Gist options
  • Save Chocksy/0d6e1dbc549a5ff4453f to your computer and use it in GitHub Desktop.
Save Chocksy/0d6e1dbc549a5ff4453f to your computer and use it in GitHub Desktop.
Transfer MongoDB collection to SQL
# Taken from a rails app using mongoid and activrecord at the same time.
def transfer_mongo_to_sql(collection_name, fields=[])
ic = Mongoid.default_session.collections.select { |c| c.name==collection_name }.first
total_items = ic.find.count
per_batch = 1000
0.step(total_items, per_batch) do |offset|
to_fields = fields.map { |c| c[:to_name] }.join(', ')
insert_raw_sql = "INSERT INTO #{collection_name} (#{to_fields}) VALUES "
before = Time.now
vars = []
raw_inserts = []
ic.find.limit(per_batch).skip(offset).each do |item|
raw_inserts << "(#{['?'].cycle(fields.length).to_a.join(', ')})"
fields.each do |field|
field_value = nil
if field[:method].blank?
field_value = item[field[:name]]
else
field_value = item[field[:name]].send(field[:method])
end
# if we have a default value set and our parsed field_value is blank
if field[:default] and field_value.blank?
field_value = field[:default]
end
vars << field_value
end
end
insert_raw_sql = insert_raw_sql + raw_inserts.join(', ')
sanitized_sql = ActiveRecord::Base.send(:sanitize_sql_array, [insert_raw_sql, *vars])
puts "nr items: #{raw_inserts.length}"
ActiveRecord::Base.connection.execute(sanitized_sql)
time = Time.now - before
puts "Job finished in #{time} secs"
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment