-
-
Save abratashov/155bcd0ea2e02940cc6157e6970e7a2b to your computer and use it in GitHub Desktop.
Add bulk import functionality to Rails Active Record (add this file to config/initializers, call <model>.import!(array-of-record-hashes))
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# frozen_string_literal: true | |
# Makes BULK INSERT for ActiveRecord rows | |
class ActiveRecord::Base | |
def import!(record_list, options = {}) | |
batch_size = options[:batch_size] || 1000 | |
raise ArgumentError "record_list not an Array of Hashes" unless valid_record_list?(record_list) | |
return record_list if record_list.empty? | |
record_list.each_slice(batch_size).each do |records| | |
key_list, value_list = convert_record_list(records) | |
sql = "INSERT IGNORE INTO #{table_name} (#{key_list.join(', ')}) VALUES #{joined_values_list(value_list)}" | |
connection.execute(sql) | |
end | |
record_list | |
end | |
private | |
def valid_record_list?(record_list) | |
record_list.is_a?(Array) && record_list.all? { |rec| rec.is_a? Hash } | |
end | |
def joined_values_list(value_list) | |
value_list.map { |rec| "(#{rec.join(', ')})" }.join(' ,') | |
end | |
def convert_record_list(record_list) | |
key_list = record_list.map(&:keys).flatten.map(&:to_s).uniq.sort | |
value_list = record_list.map do |rec| | |
list = [] | |
key_list.each { |key| list << ActiveRecord::Base.connection.quote(rec[key] || rec[key.to_sym]) } | |
list | |
end | |
time = ActiveRecord::Base.connection.quote(Time.current) | |
%w[created_at updated_at].each do |field_name| | |
if column_names.include?(field_name) && !key_list.include?(field_name) | |
key_list << field_name | |
value_list.each { |rec| rec << time } | |
end | |
end | |
[key_list, value_list] | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Improved
BULK INSERT
from https://gist.github.com/jackrg/76ade1724bd816292e4eAdded
batch_size
as parameter andIGNORE
into SQLThis gist was added because of absence the answers on the question 'Why 'activerecord-import' doesn't perform bulk insert?' was added this gist.
Also Rails 6 supports bulk insert