Splits a CSV file into multiple, smaller chunks.

csvsplit.rb

#!/usr/bin/env ruby -w

file = "products.csv"
lines_per_file = 20000
header_lines = 1

extension = File.extname(file)
basename = File.basename(file, extension)

File.open(file) do |f|
  i = 0
  header = []
  header_lines.times { header << f.readline }
  
  lines = []
  begin
    while line = f.readline do
      lines << line
      if lines.length >= lines_per_file && line[-2..-2] == '"'
        puts "Wrote #{lines.length}"
        File.open("#{basename}#{i+=1}#{extension}", "w") do |put|
          put.write header.join + lines.join
        end
        lines = []
      end
    end
  rescue
    if lines.length > 0
      File.open("#{basename}#{i+=1}#{extension}", "w") do |put|
        put.write header.join + lines.join
      end
    end
  end
end