Ruby script for breaking a big Salesforce metadata retrieval into smaller retrievals

retrieve_big_package.rb

# requires
require 'rexml/document'
require 'FileUtils'
include REXML

# constants
INPUT_FILE_NAME = 'bigpackage.xml'
OUTPUT_FILE_NAME = 'package.xml'
RETRIEVE_DIR = 'retrieve'
MAX_NUMBER = 100

# function definitions
def addComponentType(component_type)
  @types = Element.new("types")
  @name = Element.new("name")
  @name.add_text(component_type)
  @types.add(@name)
  @root.add(@types)
end

def doRetrieve
  output = File.open(OUTPUT_FILE_NAME, "w")
  @small_package.write(output)
  output.close
  puts "Starting retrieve #{(@count / MAX_NUMBER).to_s}"
  puts `ant retrieve`  
end

def initPackage
  @small_package = Document.new()
  @root = Element.new("Package")
  @small_package.add(@root)
  @root.add_attribute('xmlns', 'http://soap.sforce.com/2006/04/metadata')
  @version = Element.new('version')
  @version.add_text('24.0')
  @root.add(@version)
end

# increase java heap for ant
ENV["ANT_OPTS"] = "-Xmx1024m -Xms512m"

# load initial package.xml
doc = Document.new(File.new(INPUT_FILE_NAME))

# create fresh retrieve directory (wipe out old one if it exists)
FileUtils.rm_rf(RETRIEVE_DIR) if(Dir.exists?(RETRIEVE_DIR))
Dir.mkdir(RETRIEVE_DIR)

# collect types of retrieves
component_types = []
doc.each_element("/Package/types/name") do |name_node|
  component_types.push name_node.text
end

# break bigpackage.xml into smaller retrieves
@count = 0
initPackage
component_types.each do |component_type|
  addComponentType(component_type)
  doc.each_element("Package/types[name='#{component_type}']/members") do |member_node|
    if(@count % MAX_NUMBER == 0 && @count != 0)
      doRetrieve
      initPackage
      addComponentType(component_type)
    end
    @types.add_element(member_node)
    @count += 1
  end
end
doRetrieve

Excellent take on the issue with retrieving meta-data from complex orgs!