Illustrative example of a sync job that matches a third party data structure.

confluence_cloud.rb

class Sync::ConfluenceCloud
  
  def run_job(job) 
    case job.task
    when nil # Entry point, starts with queue_job (no params)
      fetch_spaces.each do |space|
        queue_job(task: 'crawl_space', space_id: space.id)
        queue_job(task: 'index_document', document: space.to_json)
      end
    when 'crawl_space'
      response = if job.next_page.blank?
        fetch_space_contents(job.space_id)
      else
        fetch_next_page(job.next_page)
      end
      
      queue_job(task: 'crawl_space', next_page: response.next_page) if response.next_page.present?
      response.each do |content|
        queue_job(task: 'index_document', document: content.to_json)
      end
    when 'index_document'
      job.document.raw_body = download_attachment(job.document.id) if job.document.type == 'attachment'
      index_to_elasticsearch(job.document)
    end
  end

  # Not illustrated:
  #   Auth data handling
  #   Tracking overall sync progress or completion
  #   Shared sync data cache needed for things like Google Drive document paths
  #   DLP

end