nicholasf/gist:986379

## gistfile1.txt
#basically, my process def wants to keep iterating through archive_catalog_integrations, but it also wants the participant to keep
#adding items to process in this cursor block

           cursor do
             concurrent_iterator :on_field => 'archive_catalog_integrations', :to_f => "archive_catalog_integration_id" do
               participant :ref => 'download_and_convert_catalog_tree'
             end

             rewind :if => '${archive_catalog_integrations.any?}'
           end


#the participant, very messy - still working it out

      engine.register_participant 'download_and_convert_catalog_tree' do |workitem|
        archive_catalog_integration = ArchiveCatalogIntegration.find(workitem.fields['archive_catalog_integration_id'])
        workitem.fields['archive_catalog_integrations'].delete(archive_catalog_integration)

        archive = archive_catalog_integration.archive
        file = "#{$catalogs_xml_dir}#{archive.slug}/#{archive_catalog_integration.label}.xml"

        #does a parent exist for this catalog?
        parent_id = workitem.fields['children_to_parents'][archive_catalog_integration.url]

        `curl -o #{file} --compressed "#{archive_catalog_integration.url}"`
        label_hash = Nesstar::RDF::Parser.parse_catalog("#{file}")
        catalog = ArchiveCatalog.create!(:title => label_hash[:label], :archive_catalog_integration => archive_catalog_integration)

        if parent_id
          node = ArchiveCatalogNode.create!(:archive_catalog => catalog, :parent_id => parent_id)
        else
          node = ArchiveCatalogNode.create!(:archive_catalog => catalog)
        end

        children_file = "#{$catalogs_xml_dir}#{archive.slug}/#{archive_catalog_integration.label}@children.xml"
        `curl -o #{children_file} --compressed "#{archive_catalog_integration.url}@children"`
        children = Nesstar::RDF::Parser.parse_catalog_children("#{children_file}")

        for child in children
          if child[:resource] =~ /fStudy/
            study = Study.find_by_about(child[:resource])
            archive_study = study.for_archive(archive)
            node = ArchiveCatalogNode.create!(:archive_study => archive_study, :parent => node, :catalog_position => child[:position])
          else
            integration = ArchiveCatalogIntegration.create(:archive => archive, :url => child[:resource])
            workitem.fields['archive_catalog_integrations'] << integration.id
            workitem.fields['children_to_parents'][integration.url]= node.id #store the parent for future association
          end
        end

        # workitem.fields['Catalogs_oustanding'] <<
      end
	#basically, my process def wants to keep iterating through archive_catalog_integrations, but it also wants the participant to keep
	#adding items to process in this cursor block

	cursor do
	concurrent_iterator :on_field => 'archive_catalog_integrations', :to_f => "archive_catalog_integration_id" do
	participant :ref => 'download_and_convert_catalog_tree'
	end

	rewind :if => '${archive_catalog_integrations.any?}'
	end



	#the participant, very messy - still working it out

	engine.register_participant 'download_and_convert_catalog_tree' do \|workitem\|
	archive_catalog_integration = ArchiveCatalogIntegration.find(workitem.fields['archive_catalog_integration_id'])
	workitem.fields['archive_catalog_integrations'].delete(archive_catalog_integration)

	archive = archive_catalog_integration.archive
	file = "#{$catalogs_xml_dir}#{archive.slug}/#{archive_catalog_integration.label}.xml"

	#does a parent exist for this catalog?
	parent_id = workitem.fields['children_to_parents'][archive_catalog_integration.url]

	`curl -o #{file} --compressed "#{archive_catalog_integration.url}"`
	label_hash = Nesstar::RDF::Parser.parse_catalog("#{file}")
	catalog = ArchiveCatalog.create!(:title => label_hash[:label], :archive_catalog_integration => archive_catalog_integration)

	if parent_id
	node = ArchiveCatalogNode.create!(:archive_catalog => catalog, :parent_id => parent_id)
	else
	node = ArchiveCatalogNode.create!(:archive_catalog => catalog)
	end

	children_file = "#{$catalogs_xml_dir}#{archive.slug}/#{archive_catalog_integration.label}@children.xml"
	`curl -o #{children_file} --compressed "#{archive_catalog_integration.url}@children"`
	children = Nesstar::RDF::Parser.parse_catalog_children("#{children_file}")

	for child in children
	if child[:resource] =~ /fStudy/
	study = Study.find_by_about(child[:resource])
	archive_study = study.for_archive(archive)
	node = ArchiveCatalogNode.create!(:archive_study => archive_study, :parent => node, :catalog_position => child[:position])
	else
	integration = ArchiveCatalogIntegration.create(:archive => archive, :url => child[:resource])
	workitem.fields['archive_catalog_integrations'] << integration.id
	workitem.fields['children_to_parents'][integration.url]= node.id #store the parent for future association
	end
	end

	# workitem.fields['Catalogs_oustanding'] <<
	end