buffered_read.rb

class File
  def self.buffer(path)
    n = 0
    size = File.size( path )
    chunk = 2**16
    File.open(path, "rb") do |f|
      while n < size
        n += chunk
        yield f.sysread( chunk )
      end
    end
  end 
end

file = ARGV[0]
2.times do
  lines, bytes = 0,0
  start = Time.now

  File.buffer(file) do |buf|
    lines += buf.count("\n")
    bytes += buf.size
  end

  elapsed = Time.now-start
  mb = bytes.to_f / 2**20
  puts "File.buffer read %d lines & %0.1fMB in %0.3fs (%d lines/sec, %0.1fMB/sec)" % [
    lines, mb, elapsed, lines/elapsed, mb/elapsed
  ]
end

output in gentoo linux host with 1.9.2

phiggins@lihnucks /tmp/file_read_test $ ruby run_all.rb 
ruby 1.9.2p0 (2010-08-18 revision 29036) [x86_64-linux]

read.rb
40k.txt
File.open(...) read 40000 lines & 1.9MB in 0.122s (326818 lines/sec, 15.4MB/sec)
File.open(...) read 40000 lines & 1.9MB in 0.105s (380246 lines/sec, 18.0MB/sec)
blob.txt
File.open(...) read 245561 lines & 10.1MB in 0.947s (259344 lines/sec, 10.6MB/sec)
File.open(...) read 245561 lines & 10.1MB in 0.599s (410069 lines/sec, 16.8MB/sec)

foreach.rb
40k.txt
File.foreach read 40000 lines & 1.9MB in 0.043s (939507 lines/sec, 45.2MB/sec)
File.foreach read 40000 lines & 1.9MB in 0.043s (933382 lines/sec, 45.0MB/sec)
blob.txt
File.foreach read 245561 lines & 10.3MB in 0.248s (991769 lines/sec, 41.6MB/sec)
File.foreach read 245561 lines & 10.3MB in 0.242s (1013937 lines/sec, 42.5MB/sec)

readlines.rb
40k.txt
File.open(...,'rb').readlines read 40000 lines & 1.9MB in 0.039s (1021022 lines/sec, 49.2MB/sec)
File.open(...,'rb').readlines read 40000 lines & 1.9MB in 0.042s (954558 lines/sec, 46.0MB/sec)
blob.txt
File.open(...,'rb').readlines read 245561 lines & 10.3MB in 0.217s (1129296 lines/sec, 47.4MB/sec)
File.open(...,'rb').readlines read 245561 lines & 10.3MB in 0.225s (1091649 lines/sec, 45.8MB/sec)

buffered_read.rb
40k.txt
File.buffer read 40000 lines & 1.9MB in 0.007s (5763625 lines/sec, 277.6MB/sec)
File.buffer read 40000 lines & 1.9MB in 0.006s (6223594 lines/sec, 299.8MB/sec)
blob.txt
File.buffer read 245561 lines & 10.3MB in 0.034s (7300908 lines/sec, 306.4MB/sec)
File.buffer read 245561 lines & 10.3MB in 0.032s (7707560 lines/sec, 323.4MB/sec)

readrb.rb
40k.txt
File.open(...,'rb') read 40000 lines & 1.9MB in 0.087s (458927 lines/sec, 21.7MB/sec)
File.open(...,'rb') read 40000 lines & 1.9MB in 0.086s (464021 lines/sec, 21.9MB/sec)
blob.txt
File.open(...,'rb') read 245561 lines & 10.1MB in 0.508s (483755 lines/sec, 19.8MB/sec)
File.open(...,'rb') read 245561 lines & 10.1MB in 0.494s (496913 lines/sec, 20.4MB/sec)

readlines.rb

file = ARGV[0]
2.times do
  lines, bytes = 0,0
  start = Time.now

  File.open(file, "rb").readlines.each do |line|
    lines += 1
    bytes += line.size
  end
  elapsed = Time.now-start
  mb = bytes.to_f / 2**20
  puts "File.open(...,'rb').readlines read %d lines & %0.1fMB in %0.3fs (%d lines/sec, %0.1fMB/sec)" % [
    lines, mb, elapsed, lines/elapsed, mb/elapsed
  ]
end

run_all.rb

tests = Dir["*.rb"] - [$0]

system("ruby -v")

tests.each do |test|
  puts "\n#{test}"
  Dir["*.txt"].each do |file|
    puts file
    system( "ruby #{test} #{file}" )
  end
end

windows xp guest with 1.9.1

C:\file_read_test>ruby run_all.rb
ruby 1.9.1p429 (2010-07-02 revision 28523) [i386-mingw32]

buffered_read.rb
40k.txt
File.buffer read 40000 lines & 1.9MB in 0.016s (2560000 lines/sec, 123.3MB/sec)
File.buffer read 40000 lines & 1.9MB in 0.031s (1280000 lines/sec, 61.6MB/sec)
blob.txt
File.buffer read 245561 lines & 10.3MB in 0.109s (2245129 lines/sec, 94.2MB/sec)
File.buffer read 245561 lines & 10.3MB in 0.109s (2245129 lines/sec, 94.2MB/sec)

foreach.rb
40k.txt
File.foreach read 40000 lines & 1.9MB in 0.672s (59534 lines/sec, 2.9MB/sec)
File.foreach read 40000 lines & 1.9MB in 0.672s (59534 lines/sec, 2.9MB/sec)
blob.txt
File.foreach read 245561 lines & 10.3MB in 3.500s (70160 lines/sec, 2.9MB/sec)
File.foreach read 245561 lines & 10.3MB in 3.484s (70474 lines/sec, 3.0MB/sec)

read.rb
40k.txt
File.open(...) read 40000 lines & 1.9MB in 2.156s (18550 lines/sec, 0.9MB/sec)
File.open(...) read 40000 lines & 1.9MB in 2.141s (18686 lines/sec, 0.9MB/sec)
blob.txt
File.open(...) read 245559 lines & 10.1MB in 41.797s (5875 lines/sec, 0.2MB/sec)
File.open(...) read 245559 lines & 10.1MB in 42.375s (5794 lines/sec, 0.2MB/sec)


readlines.rb
40k.txt
File.open(...,'rb').readlines read 40000 lines & 1.9MB in 0.062s (640000 lines/s ec, 30.8MB/sec)
File.open(...,'rb').readlines read 40000 lines & 1.9MB in 0.094s (426666 lines/s ec, 20.5MB/sec)
blob.txt
File.open(...,'rb').readlines read 245561 lines & 10.3MB in 0.375s (654829 lines /sec, 27.5MB/sec)
File.open(...,'rb').readlines read 245561 lines & 10.3MB in 0.609s (402971 lines /sec, 16.9MB/sec)

readrb.rb
40k.txt
File.open(...,'rb') read 40000 lines & 1.9MB in 0.109s (365714 lines/sec, 17.3MB /sec)
File.open(...,'rb') read 40000 lines & 1.9MB in 0.109s (365714 lines/sec, 17.3MB /sec)
blob.txt
File.open(...,'rb') read 245561 lines & 10.1MB in 0.672s (365486 lines/sec, 15.0 MB/sec)
File.open(...,'rb') read 245561 lines & 10.1MB in 0.656s (374188 lines/sec, 15.3 MB/sec)