Skip to content

Instantly share code, notes, and snippets.

@kikeda1104
Last active July 13, 2018 08:52
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save kikeda1104/6dfdceef30c6475da23c2711b79d9942 to your computer and use it in GitHub Desktop.
Save kikeda1104/6dfdceef30c6475da23c2711b79d9942 to your computer and use it in GitHub Desktop.
def call
TABLES.each do |table|
system "mysql --defaults-file=#{conf_file_path} -h\"#{host}\" " \
"-D \"#{database}\" -e \"select * from #{table} where updated_at >= \'#{@before_updated_at}\'\" " \
"-s -N > #{outputs_dir}/#{table}.txt"
file_size = File.size("#{outputs_dir}/#{table}.txt")
@file_count = file_count(file_size)
@rows = rows(table)
split_files(table)
files = Pathname.glob(splits_dir.join("#{table}*"))
compress(files)
gzip_files = Pathname.glob(splits_dir.join("#{table}*")
end
remove_my_conf
end
private
def post_initialize(args)
@slices = ENV['slices'] || 4
@before_updated_at = args[:before_updated_at] || 1.week.ago.strftime('%Y/%m/%d 00:00:00')
my_conf
remove_files
end
def file_count(file_size)
return @slices if file_size < LIMIT_FILESIZE
@slices.step(Float::INFINITY, @slices) do |i|
next if (file_size / i) > LIMIT_FILESIZE
return i
end
end
def rows(table)
`wc -l #{outputs_dir}/#{table}.txt`.match(/\s*(\d*)/)[1].to_i
end
def split_files(table)
if os == :macosx
# MAC OSでは、`-d`オプションが利用できない。使いたい場合は、brew install coreutils
system "gsplit -d -l #{lines} #{outputs_dir}/#{table}.txt #{splits_dir}/#{table}.txt. &>/dev/null"
else
system "split -d -l #{lines} #{outputs_dir}/#{table}.txt #{splits_dir}/#{table}.txt. &>/dev/null"
end
end
def compress(files)
files.each { |file| system "gzip #{file}" }
end
def lines
# NOTE: 行数がファイルの最小分割数に満たない場合は、1ファイルにする
return 1 if @rows < @file_count
(@rows / @file_count.to_f).ceil
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment