Cloud-native ETL

etl-example.js

/*
data can enter the pipeline via push or pull.
the result of each intermediate step is stored (so errors can be retried)
at the end of the pipeline data is put into a document store
optionally, event handlers can listen to create/update/delete events and fan data out to other systems
*/

// example of a push handler (aka webhooks)
// src/push/shopify/order/created.js
// receive events, and returned data is stored in ClickHouse (OLAP) by the framework. 
export default function handle(req, res) {
   verifyRequest(req.headers)
   
   return {
     event: 'order/created',
     data: {
       ...req.json()
     }
   }
}

// example of pull handler (aka polling)
// src/pull/skubana/shipments.js
// can pull data on a schedule
export const schedule = '5 minutes'

export default async function handle() {
  const response = await fetch(...)
  
  return {
    event: 'shipment/created',
    data: {
      ...response.json()
    }
  }
}

// src/transformers/order/created.js
// framework calls transformer for object type
// output at each stage is stored in Mongo (document store)
//
// transformer is optional
export default function handle(order) {
  return {
    some,
    other,
    stuff(order),
    ...order
  }
} 

// src/loader/order.js
// framwork calls loader, and stores result in Mongo (document store)
export default function handle(order) {
  return {
    document: 'orders',
    id: order.id,
    data: order
  }
} 

// can handle events create/updae/delete and send data
// src/notification/order/created.js
export default function handle(order) {
  await fetch('https://api.kustomer.com/klasses/order/${order.id}', {
    method: 'POST',
    data: JSON.stringify(order)
  })
}