snippet.rb

require 'open-uri'
require 'thread'

# This is the index file with the filings
index_url = "http://www.sec.gov/Archives/edgar/daily-index/master.20110701.idx"

queue = Queue.new
system "mkdir -p 20110701"
Dir.chdir "20110701"

LINE_REGEX = /(\d+)\|([^\|]+)\|([^\|]+)\|(\d+)\|([^\|]+)/

threads = (0..4).to_a.map do
  Thread.new do
    while true do
      cik, file_name = queue.pop

      system "mkdir -p #{cik};cd #{cik} ; wget -cq http://www.sec.gov/Archives/#{file_name}"
    end
  end
end

open(index_url) do |index|
  index.each_line do |line|
    m = LINE_REGEX.match(line)
    next unless m
    _, cik, company_name,form_type,date_filed,file_name = m.to_a
    
    queue.push([cik, file_name])
  end
end


threads.each(&:join)