Last active
July 13, 2018 08:52
-
-
Save kikeda1104/6dfdceef30c6475da23c2711b79d9942 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def call | |
TABLES.each do |table| | |
system "mysql --defaults-file=#{conf_file_path} -h\"#{host}\" " \ | |
"-D \"#{database}\" -e \"select * from #{table} where updated_at >= \'#{@before_updated_at}\'\" " \ | |
"-s -N > #{outputs_dir}/#{table}.txt" | |
file_size = File.size("#{outputs_dir}/#{table}.txt") | |
@file_count = file_count(file_size) | |
@rows = rows(table) | |
split_files(table) | |
files = Pathname.glob(splits_dir.join("#{table}*")) | |
compress(files) | |
gzip_files = Pathname.glob(splits_dir.join("#{table}*") | |
end | |
remove_my_conf | |
end | |
private | |
def post_initialize(args) | |
@slices = ENV['slices'] || 4 | |
@before_updated_at = args[:before_updated_at] || 1.week.ago.strftime('%Y/%m/%d 00:00:00') | |
my_conf | |
remove_files | |
end | |
def file_count(file_size) | |
return @slices if file_size < LIMIT_FILESIZE | |
@slices.step(Float::INFINITY, @slices) do |i| | |
next if (file_size / i) > LIMIT_FILESIZE | |
return i | |
end | |
end | |
def rows(table) | |
`wc -l #{outputs_dir}/#{table}.txt`.match(/\s*(\d*)/)[1].to_i | |
end | |
def split_files(table) | |
if os == :macosx | |
# MAC OSでは、`-d`オプションが利用できない。使いたい場合は、brew install coreutils | |
system "gsplit -d -l #{lines} #{outputs_dir}/#{table}.txt #{splits_dir}/#{table}.txt. &>/dev/null" | |
else | |
system "split -d -l #{lines} #{outputs_dir}/#{table}.txt #{splits_dir}/#{table}.txt. &>/dev/null" | |
end | |
end | |
def compress(files) | |
files.each { |file| system "gzip #{file}" } | |
end | |
def lines | |
# NOTE: 行数がファイルの最小分割数に満たない場合は、1ファイルにする | |
return 1 if @rows < @file_count | |
(@rows / @file_count.to_f).ceil | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment