Created
February 26, 2019 11:55
-
-
Save carlosveucv/70af7ea1dc9f9166ee5fe87336598da4 to your computer and use it in GitHub Desktop.
Dummy data to fill up mysql on batches
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env ruby | |
require 'mysql2' | |
require 'securerandom' | |
require 'date' | |
@mysql_username = 'root' | |
@mysql_password = 'root' | |
@mysql_port = 3306 | |
@mysql_host = '1.1.1.1' | |
@database_name = 'foo_database' | |
@number_of_transactions = 1000 | |
@number_of_transactions_for_extra = 5000000 | |
@tables_for_extra_data=%w(foo_table1 foo_table2) | |
@exclude_tables=%w(foo_excluded_table1 foo_excluded_table2) | |
@batches = 500 | |
def main | |
con = Mysql2::Client.new( | |
host: @mysql_host, | |
username: @mysql_username, | |
password: @mysql_password, | |
port: @mysql_port | |
) | |
tables = con.query "SELECT table_name FROM information_schema.tables WHERE table_schema=\'#{@database_name}\'" | |
tables.each do |r| | |
table = r['table_name'] | |
p table | |
next if @exclude_tables.include?(table) | |
number_of_transactions = @number_of_transactions | |
number_of_transactions = @number_of_transactions_for_extra if @tables_for_extra_data.include?(table) | |
table_fields = con.query("DESCRIBE #{@database_name}.#{table}") | |
table_sql = "INSERT INTO #{@database_name}.#{table} (" | |
for i in 1..(number_of_transactions/@batches) | |
sql = '' | |
for j in 1..@batches | |
field_list = value_list = '' | |
table_fields.each do |field| | |
next unless valid_field?(field) | |
field_list += "`#{field['Field']}`," | |
value_list += "#{value_to_insert(field)}," | |
end | |
field_list.chop! | |
value_list.chop! | |
if j==1 | |
sql = table_sql + "#{field_list}) VALUES (#{value_list}) " | |
break if number_of_transactions < @batches | |
else | |
sql += ", (#{value_list})" | |
end | |
end | |
con.query(sql) | |
p i*@batches | |
cool_off(100000, 10) if (i*@batches)%100000 == 0 | |
cool_off(500000, 45) if (i*@batches)%500000 == 0 | |
end | |
end | |
rescue StandardError => e | |
puts e | |
ensure | |
con.close if con | |
end | |
def cool_off(cycles, sleep_time = 30) | |
p "Cool off for #{sleep_time} sec every #{cycles} cycles" | |
sleep(sleep_time) | |
end | |
def valid_field?(field) | |
field['Extra'] != 'auto_increment' | |
end | |
def value_to_insert(field) | |
field_type = field['Type'] | |
if field_type.include?('tinyint') | |
random_boolean | |
elsif field_type.include?('decimal') || field_type.include?('int') || field_type.include?('float') | |
precision = field_type.scan(/\(\d*/).join('') | |
precision[0] = '' | |
random_int(2**(precision.to_i - 1)) | |
elsif field_type.include?('date') | |
random_date | |
elsif field_type.include?('binary') || field_type.include?('char') | |
random_string(field_type.scan(/\d/).join('').to_i) | |
elsif field_type.include?('json') | |
"'{\"foo\": \"bar\"}'" | |
else | |
random_string | |
end | |
end | |
def random_int(max = 100_000_000) | |
SecureRandom.random_number(max) | |
end | |
def random_boolean | |
random_int(2) | |
end | |
def random_string(limit = 50) | |
"'#{SecureRandom.hex.slice(0, limit)}'" | |
end | |
def random_date | |
"'#{Time.at(Time.now.to_i - SecureRandom.random_number(1_000_000_000)) | |
.to_date}'" | |
end | |
main |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment