sourcicheck.rb

require 'rubygems' # or bundle something
require 'sourcify'

lamb = lambda { |workitem|

  workitem.fields['fetch_errors']       ||= []
  workitem.fields['downloaded_files']   ||= []
  workitem.fields['study_ids']          ||= Set.new

  ddi_id = workitem.fields['ddi_id']
  file_name = "#{ddi_id}.xml"

  mutex = Mutexer.wait_for_mutex(2)

  begin
    mutex.synchronize do
      # puts "\\n\n study download: downloading: #{$nesstar_server}/obj/fStudy/au.edu.anu.ada.ddi.#{ddi_id}"
      http_headers = `curl -i --compressed "#{$nesstar_server}/obj/fStudy/au.edu.anu.ada.ddi.#{ddi_id}"`
      http_headers = http_headers.split("\n")

      if http_headers.first =~ /500/
        workitem.fields['fetch_errors'] << "Error while downloading #{ddi_id}: #{http_headers.first} \n"
        Inkling::Log.create!(:category => "integration", :text =>  "HTTP 500 error downloading: #{$nesstar_server}/obj/fStudy/au.edu.anu.assda.ddi.#{ddi_id}")
        next
      end

      begin
        # puts " #{$nesstar_server}/obj/fStudy/au.edu.anu.ada.ddi.#{ddi_id}"
        `curl -o #{$studies_xml_dir}#{file_name} --compressed "#{$nesstar_server}/obj/fStudy/au.edu.anu.ada.ddi.#{ddi_id}"`
        workitem.fields['downloaded_files'] << file_name
      rescue StandardError => boom
        puts "#{boom}.to_s"
        workitem.fields['fetch_errors'] << "Error while downloading #{ddi_id}: #{boom} \n"
      end

      study_hash = Nesstar::RDF::Parser.parse("#{$studies_xml_dir}#{ddi_id}.xml")
      study = Study.store_with_fields(study_hash)

      DdiMapping.batch_create(study_hash) #create mappings entries for any DDI elements/attributes we have not yet noticed

      #find archive study integrations which need to be linked to the new study
      integrations = ArchiveStudyIntegration.find_all_by_ddi_id_and_study_id(ddi_id, nil)

      for integration in integrations
        integration.study_id = study.id
        integration.save!
      end
    end
  ensure
    ActiveRecord::Base.connection_pool.release_connection
  end
}

source = lamb.to_source

p source