etl.rb

Vetdata::EndpointExtractor.each(:animal, :past_day) do |animal_data, installation|

  animal_data = AnimalTransform.call(animal_data, installation)
  AnimalChangesetCreator.call(animal_data)
end

class AnimalTransform
  def self.call(animal_data, installation)
    {
       unique_id: "#{installation.id}--#{animal_data[:id]}",
       name: animal_data[:name],
       ...etc
       client: ClientTransform.call(animal_data[:client])
    }
  end
end


class AnimalChangesetCreator
  def self.call(data)
    case
    when created?(data)
      Changeset.create(
         entity: 'Animal',
         event: 'animal.created',
         handler: 'AnimalCreatedChangesetExecutor',
         data: data,
         created_at: Time.now
         resolved_at: nil,
         status: :new
      )
    when deleted?(data):
      Changeset.create(
         entity: 'Animal',
         event: 'animal.deleted',
         handler: 'AnimalDeletedChangesetExecutor',
         data: { unique_id: data[:unique_id]},
         created_at: Time.now
         resolved_at: nil,
         status: :new
      )
    when modified?(data) # implicitly exists otherwise would have been found earlier
      Changeset.create(
         entity: 'Animal',
         event: 'animal.modified',
         handler: 'AnimalModifiedChangesetExecutor',
         data: data,
         created_at: Time.now
         resolved_at: nil,
         status: :new
      )
    end
  end

  def self.deleted?(data)
    data['APICreatedDate']
  end

  def self.created?(data)
    !Animal.exists?(unique_id: data[:unique_id])
  end

  def self.modified?(data)
    # something to check if any attributes are different?
    animal = Animal.find_by(unique_id: data[:unique_id])
    animal.vetdata_attributes == data
  end
end



# later...


Changeset.dependency_ordered.order(created_at: :asc).each do |changeset|
  results = changeset.handler.constantize.call(changeset.data)
  changeset.update(results: results, resolved_at: Time.now, status: :complete)

rescue => e
  changeset.update(error: e, status: :error)
  # queue up retry maybe?
  # alert us maybe?
end