Upload a large file (> 5GB) to Rackspace Cloud Files

upload_to_cloud_files.rb

require 'rubygems'
require 'fog'

SEGMENT_LIMIT = 5368709119.0  # 5GB -1
BUFFER_SIZE = 1024 * 1024 # 1MB
CONTAINER = 'my-existing-container'
FILE_DIR = '/path/to/my/big/file'
FILE_NAME = 'huge_tarball.tgz'
RACKSPACE_USERNAME = 'my-rackspace-username'
RACKSPACE_API_KEY = 'my-rackspace-api-key'
RACKSPACE_REGION = 'ord' # ord = Chicago

service = Fog::Storage.new({
  :provider => 'Rackspace',
  :rackspace_username => RACKSPACE_USERNAME,
  :rackspace_api_key => RACKSPACE_API_KEY,
  :rackspace_region => RACKSPACE_REGION
})

File.open("#{FILE_DIR}/#{FILE_NAME}") do |file|
  segment = 0
  until file.eof?
    segment += 1
    offset = 0

    # upload segment to cloud files
    segment_suffix = segment.to_s.rjust(10, '0')
    puts "Uploading segment #{segment_suffix}..."
    service.put_object(CONTAINER, "#{FILE_NAME}/#{segment_suffix}", nil) do
      if offset <= SEGMENT_LIMIT - BUFFER_SIZE
        buf = file.read(BUFFER_SIZE).to_s
        offset += buf.size
        buf
      else
        ''
      end
    end
  end
end