Buffered IO

gistfile1.txt

CHUNK_SIZE = 1024 * 1024
EMPTY_STRING = ''.freeze
NEWLINE = "\n".freeze

# line_enum('path/to/huge/file').each do |line|
#   puts line
# end
def line_enum(path)
  Enumerator.new do |y|
    File.open(path) do |io|
      carry_over = EMPTY_STRING
      while blob = io.read(CHUNK_SIZE)
        lines = (carry_over + blob).split(NEWLINE)
        carry_over = lines.pop
        lines.each do |line|
          y << line
        end
      end
      y << carry_over
    end
  end
end