dgraph image importer

gistfile1.txt

/* tslint:disable:no-console */
import { TaskOptions } from 'bin/task-bin'
import { listObjects } from 'lib/apis/s3'
import { getConfig } from 'lib/config'
import { batch } from 'streaming-iterables'
import * as dgraph from 'dgraph-js'
import { getSecret } from 'lib/ssm'

export default async function({ log }: TaskOptions) {
  const dgraphClient = await getDgraphClient()
  const { typesetMediaBetaS3Bucket: s3Bucket } = await getConfig()
  console.log('starting')
  let batchCount = 0
  for await (const s3Objects of batch(1000, listObjects({ s3Bucket }))) {
    const transaction = dgraphClient.newTxn()
    const images = s3Objects.map(({ Key: s3key, Size: bytes, ETag: s3etag, LastModified: lastModified }) => ({
      _type: 'Image',
      s3key,
      bytes,
      s3etag,
      updatedAt: lastModified && new Date(lastModified),
    }))
    const mutation = new dgraph.Mutation()
    mutation.setSetJson(images)
    console.log(`committing batch ${++batchCount}`)
    await transaction.mutate(mutation)
    await transaction.commit()
  }
}

async function getDgraphClient() {
  const dgraphUrl = await getSecret('dgraphUrl')
  console.log('fetched degraphUrl', { dgraphUrl })
  const clientStub = new dgraph.DgraphClientStub(dgraphUrl)
  const dgraphClient = new dgraph.DgraphClient(clientStub)
  return dgraphClient
}