Fetch and process remote files in Gridsome

gridsome.server.js

const path = require('path')
const fs = require('fs')
const Queue = require(`better-queue`)
const child_process = require('child_process')

const remoteimageQueue = new Queue(
  (input, cb) => {
    downloadImage(input)
      .then(r => cb(null, r))
      .catch(e => cb(e))
  },
  { concurrent: 20, maxRetries: 1, retryDelay: 1000 }
)

function initialSetUp(api, options) {

  /**
   * Implements the onCreateNode hook.
   *
   * Used to manipulate the DrupalFile node to provide the correct
   * local path to the file as a new field (called localImage).
   * This is then processed in GraphQL to an image type which allows
   * it's use as the src attribute on the g-image component.
   */
  api.onCreateNode(options => {
    if (options.internal.typeName === 'DrupalFile') {
      const file = getFilenameAndExtension(options.filename)
      const processedFilename = getProcessedFilename(file)
      options.localImage = path.resolve(__dirname, `images/${processedFilename}`)
    }
  })

  api.beforeBuild(async () => {
    // Get the images for each file in the system.
    const query = '{ allDrupalFile { edges { node { id filename filesize uri { value } } } } }'
    let anyQueued = false

    await api._app.graphql(query, {})
      .then(async result => {
        try {
          console.log('Fetching remote images')
          for (let edge of result.data.allDrupalFile.edges) {
            const file = getFilenameAndExtension(edge.node.filename)
            const fileSize = edge.node.filesize
            const processedFilename = getProcessedFilename(file)
            const url = edge.node.uri.value.replace('public://', 'sites/default/files/')
            const localPath = path.resolve(__dirname, `images/${processedFilename}`)
            const remoteUrl = `${process.env.BASE_URL}${url}`
            // Check if we already have the image locally.
            if (!fs.existsSync(localPath)) {
              anyQueued = true
              remoteimageQueue.push({
                url: remoteUrl,
                path: localPath,
                size: fileSize
              })
            }
          }
          if (anyQueued) {
            await new Promise((resolve, reject) => {
              remoteimageQueue.on('drain', () => {
                console.log('Remote image fetching finished')
                resolve()
              })
            })
          }
        }
        catch (error) {
          console.log(`There were errors fetching images`)
          console.log(error)
        }

      })
  })
}

/**
 * Process the filename by replacing spaces and lowercases the extension.
 *
 * @param {object} file
 */
function getProcessedFilename(file) {
  return `${file.filename.replace(/%20/gi, '-')}${file.extension.toLowerCase()}`
}

/**
 * Returns an object with filename and extension in properties.
 *
 * @param {string} file
 */
function getFilenameAndExtension(file) {
  return {
    filename: path.parse(file).name,
    extension: path.parse(file).ext
  }
}

/**
 * Given a url and a path to save locally will download the image.
 *
 * @param {string} url
 * @param {string} localPath
 */
const downloadImage = async ({
  url,
  path,
  size
}) => {
  // Ignore certain urls as they aren't files.
  if (!url.includes('api/file/file')) {
    // Use curl to download the remote file.
    const options = "-H 'pragma: no-cache' -H 'cache-control: no-cache' -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36'"
    const random = Math.random() * 100
    const curl = `curl -s "${url}?random=${random}" ${options} -o "${path}"`;
    try {
      console.log('Fetching remote file at:', url)
      await new Promise((resolve, reject) => {
        child_process.exec(curl).on('exit', code => {
          // Confirm the downloaded file is the same size as the remote file.
          const localStats = fs.statSync(path)
          const fileSizeInBytes = localStats['size']
          // If there was an error or the file sizes are different then reject
          // so that a retry can happen.
          if (code > 0 || fileSizeInBytes !== size) {
            reject()
          }
          resolve()
        })
      })
    } catch (error) {
      console.log('File fetching error', 'filepath: ', localPath, 'url: ', url)
    }
  }
}

module.exports = initialSetUp