Provoke an SCP error in Net::SSH 2.0.21

provoke_scp_error.rb

#!/usr/bin/env ruby
#
# Provoke the SCP error on large files.
# You must provide the remote resource location to download as SCP would accept it:
#
#   provoke_scp_error user@host:/path/to/large/file
#
require 'rubygems'
gem 'net-ssh', '2.0.21'
gem 'net-scp', '1.0.2'
require 'net/ssh'
require 'net/scp'
require 'tempfile'
require 'fileutils'

username = nil
hostname = nil
remotepath = nil
tmpfile = Tempfile.new('scp-download')
tmpref = Tempfile.new('scp-reference')
delays = [0.0, 0.01, 0.1, 0.25, 0.5]


class DelayedTester
  attr_accessor :username, :hostname, :remotepath, :localpath, :reference
  def initialize(username, hostname, remotepath, localpath, reference=nil)
    self.username = username
    self.hostname = hostname
    self.remotepath = remotepath
    self.localpath = localpath
    self.reference = reference
  end

  def test_with_delay(delay=0)
    break_at = 0
    all_of_it = 0

    starttime = Time.now
    begin
      Net::SCP.start(hostname, username) do |session|
        oldpercent = -1
        session.download(remotepath, localpath) do |ch, name, sent, total|
          break_at = sent
          all_of_it = total unless all_of_it > 0
          percent = sent * 100.0 / total
          if oldpercent < percent
            oldpercent = percent
            sleep(delay) if delay
          end
        end
      end
      duration = (Time.now - starttime) * 1000
      { :percent => 100.0,
        :downloaded => break_at,
        :total => all_of_it,
        :time => duration,
        :exception => FileUtils.compare_file(localpath, reference) ? nil : "file content mismatch",
        :delay => delay}
    rescue Exception => e
      { :percent => all_of_it > 0 ? break_at * 100.0 / all_of_it : 0.0,
        :downloaded => break_at,
        :total => all_of_it,
        :time => (Time.now - starttime) * 1000,
        :delay => delay,
        :exception => e}
    end
  end
  
end

#
begin
  puts "tempfiles:"
  puts "  download  -> #{tmpfile.path}"
  puts "  reference -> #{tmpref.path}"
  scparg = ARGV[0]
  if scparg =~ /^([\w\d]+)@([\w\d\.]+):(.+)$/
    username = $1
    hostname = $2
    remotepath = $3

    # load the reference file
    puts "downloading reference via system scp to #{tmpref.path}"
    system("scp #{scparg} #{tmpref.path}")

    tester = DelayedTester.new(username, hostname, remotepath, tmpfile.path, tmpref.path)
    puts "starting the tests using #{username}@#{hostname}:#{remotepath} ...."
    all_results = delays.map do |delay|
      puts "testing with delay=%.2f" % delay
      result = tester.test_with_delay(delay)
      puts "  -> downloaded #{result[:downloaded]} bytes (%.2f%%) of the file in #{result[:time]} ms.#{result[:percent] == 100.0 ? ' SUCCESS!' : ''}" % result[:percent]
      result
    end
    puts "And now, the results:"
    all_results.each do |r|
      puts "delay=%.2f -> downloaded #{r[:downloaded]} bytes (%.2f%%), exception: #{r[:exception] ? r[:exception] : 'none'}" % [r[:delay], r[:percent]]
    end
  else
    puts <<EOF
usage:
  #{__FILE__} user@host:/path/to/large/file
This will download the given remote file to #{localpath}.
EOF
    puts "argument given: #{ARGV[0]}"
  end
ensure
  # cleanup tempfiles
  tmpfile.close
  tmpref.close
end

Demonstrates a problem with Net::SSH/Net::SCP with large files. Run with a large file (tried on a LAN with a >100MB file, on WinXP/Ruby 1.8.6, WinXP/Cygwin/Ruby 1.8.7, and Linux/Ruby 1.8.7).
Native SCP command transfers at ~9MB/s.

Added Lighthouse ticket: http://net-ssh.lighthouseapp.com/projects/36253/tickets/12-problem-downloading-large-files-via-netscp-on-netssh#ticket-12-1