Ruby script to pull all LRMI documents from Learning Registry and save them to local files

lrmi-download.rb

require 'json'
require 'net/http'

# adds resumption token to URL if needed
def slice_url(options ={})
  resume = options[:resume]
  url = "http://node01.public.learningregistry.net/slice?any_tags=lrmi"
  if resume 
    url = "http://node01.public.learningregistry.net/slice?any_tags=lrmi&resumption_token=#{resume}"
  end
  url
end

resume = nil
counts = {:loop => 0}
x = 0
# Loop through slice API resumption tokens until none left
while true do
  url = slice_url(:resume => resume)
  stream = Net::HTTP.get(URI(url))
  File::open("test#{x}.txt", "w+") do |file|
    file.write(stream)
  end
  json = JSON.parse("{}")
  begin
    json = JSON.parse(stream)
  rescue JSON::ParserError
    puts "Rescuing parse error."
    stream += "]}"
    json = JSON.parse(stream)
  end
  resume = json["resumption_token"]
  if !resume || resume.size < 1
    puts "Resumption token empty - definitely completed successfully"
    break 
  end
  x+=1
  print "#{x} "
end