Extract Remote Assets Rollup Plugin

plugin-extract-remote-assets.js

import { createFilter } from '@rollup/pluginutils'
import got from 'got'
import MagicString from 'magic-string'
import path from 'node:path'
import revisionHash from 'rev-hash'

function escapeRegex(string) {
  return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}

function ensureArray(value) {
  return Array.isArray(value) ? value : [value]
}

export default function extractRemoteAssets(options = {}) {
  let isSourceMapEnabled = options.sourceMap !== false && options.sourcemap !== false
  const filter = createFilter(options.include, options.exclude)
  const patterns = ensureArray(options.source ?? []).map(source =>
    source instanceof RegExp ? source : new RegExp(escapeRegex(source), 'g')
  )
  const dir = options.dir ?? 'extracted-assets'

  if (patterns.length === 0) {
    throw new Error('One or more source string/pattern must be provided')
  }

  // Ensure that all patterns are global (otherwise an infinite loop will occur)
  if (patterns.some(pattern => !pattern.flags.includes('g'))) {
    throw new Error('Source pattern must be global')
  }

  // Create a 'got' instance that only requests the same resource once
  const gotCache = new Map()
  const gotOnce = got.extend({
    handlers: [
      (gotOptions, next) => {
        if (gotOptions.isStream) return next(gotOptions)

        const pending = gotCache.get(gotOptions.url.href)
        if (pending) return pending

        const promise = next(gotOptions)

        gotCache.set(gotOptions.url.href, promise)
        promise.finally(() => {
          gotCache.delete(gotOptions.url.href)
        })

        return promise
      }
    ]
  })

  let images = new Map()

  return {
    name: 'extract-remote-assets',

    renderChunk(code, chunk) {
      const id = chunk.fileName
      if (!filter(id)) return null
      return replaceAssetUrls(code, id)
    },

    transform(code, id) {
      if (!filter(id)) return null
      return replaceAssetUrls(code, id)
    },

    async generateBundle() {
      for (let { url, name, filename, buffer, hash } of images.values()) {
        this.emitFile({
          type: 'asset',
          name,
          fileName: path.join(dir, filename),
          source: buffer
        })
      }
    }
  }

  async function replaceAssetUrls(code, id) {
    const magicString = new MagicString(code)
    let match

    const urls = new Set()
    const replacements = []

    for (const pattern of patterns) {
      while ((match = pattern.exec(code))) {
        let url = match[0]

        urls.add(url)
        replacements.push({
          start: match.index,
          end: match.index + url.length,
          url
        })
      }
    }

    if (replacements.length === 0) return null

    let unknownUrls = [...urls].filter(url => !images.has(url))

    let resolvedImagesPromise = unknownUrls.map(async url => {
      let buffer = await gotOnce(url).buffer()
      let hash = revisionHash(buffer)
      let name = path.basename(url)
      let extension = path.extname(name)

      let filename = name.slice(0, -extension.length) + '.' + hash + extension

      return {
        url,
        name,
        filename,
        buffer,
        hash
      }
    })
    let resolvedImages = await Promise.all(resolvedImagesPromise)

    // Store data about images
    for (let entry of resolvedImages) {
      images.set(entry.url, entry)
    }

    // Replace URLs in code
    for (let { start, end, url } of replacements) {
      magicString.overwrite(start, end, `/${dir}/${images.get(url).filename}`)
    }

    // Write source map data
    const result = { code: magicString.toString() }
    if (isSourceMapEnabled) {
      result.map = magicString.generateMap({ hires: true })
    }

    return result
  }
}

plugin-extract-remote-assets.md

Extract Remote Assets Rollup Plugin

A Rollup plugin which downloads resources of bare asset URLs and replaces them with local URLs.

Attention: Remote asset detection is achieved through simple string replacement, not via the Rollup module graph.

Setup

This plugin requires Node v14+ and Rollup v1.20+.

The following dependencies need to be installed:

npm install --save-dev got@12 magic-string@0.26 rev-hash@4 @rollup/pluginutils@4

Usage

Add to rollup config:

import extractRemoteAssets from './plugin-extract-remote-assets.js'

export default {
  plugins: [
    extractRemoteAssets({
      // A string, a regex, or an array thereof
      source: /https:\/\/example.com\/([a-z0-9/._~-]+).(jpe?g|png|gif|webp)/gi
    })
  ]
}

Options

source

Type: string | RegExp | Array<string | RegExp>

The URL(s) to download and replace.

dir

Type: string Default: "extracted-assets"

Directory (relative to the output directory) to copy assets to.

exclude

Type: string | string[] Default: undefined

A picomatch pattern, or array of patterns, which specifies the files in the build the plugin should ignore. By default no files are ignored.

include

Type: string | string[] Default: undefined

A picomatch pattern, or array of patterns, which specifies the files in the build the plugin should operate on. By default all files are targeted.

sourceMap

Type: boolean Default: true

Whether to reflect source code changes in the source map.