Generate Osmium configurations and a runnable shell script to recursively cut the OSM planet file down to produce regional extracts

configure.js

const fs = require('fs')
const path = require('path')
const fetch = require('node-fetch') // npm i node-fetch@2

// Usage: node configure.js planet-v1.20.osm.pbf daylight
// A file named 'extract.sh' will be generated along with the `./configs` and `./extracts` directories

// first argument is the input .pbf file
const inputFile = process.argv[2]
if (!fs.existsSync(inputFile)) { throw new Error('input file not found') }

// second argument is the 'vintage' which is appended to output .pbf filenames
const vintage = process.argv[3] || 'latest'

// build a graph of dependencies
const graph = (json, parent) => {
  const level = {}
  json.features
    .filter(feat => (feat.properties.parent === parent))
    .map(feat => feat.properties.id)
    .forEach(dst => level[dst] = graph(json, dst))

  return level
}

// recursively write everything to disk
const write = (json, subgraph, parents) => {
  let features = json.features.filter(feat => {
    return subgraph.hasOwnProperty(feat.properties.id) && parents[parents.length-1] === feat.properties.parent
  })
  if (!features.length) { return }

  // write config file
  let config = mapper({ features }, parents)
  let configPrefix = parents.length ? parents.join('.') : 'planet'
  let configFile = path.join('./configs', `${parents.length}.${configPrefix}.config`)
  let configDir = path.dirname(configFile)
  fs.existsSync(configDir) || fs.mkdirSync(configDir, { recursive: true })
  fs.writeFileSync(configFile, JSON.stringify(config, null, 2))

  // ensure all nested subdirectories are created (else osmium will error)
  config.extracts.forEach(extract => {
    let extractDir = path.join('./extracts', path.dirname(extract.output))
    fs.existsSync(extractDir) || fs.mkdirSync(extractDir, { recursive: true })
  })

  // append command to shell script
  // https://docs.osmcode.org/osmium/latest/osmium-extract.html
  let pbfFile = `${path.join('./extracts', ...parents)}-${vintage}.osm.pbf`
  fs.appendFileSync('extract.sh', `
    osmium extract \\
      -v \\
      -c ${path.resolve(configFile)} \\
      -d ${path.resolve('extracts')} \\
      -s 'smart' \\
      --fsync \\
      ${path.resolve(parents.length ? pbfFile: inputFile)}
    `)

  // recurse
  for (let parent in subgraph) {
    write(json, subgraph[parent], parents.concat([parent]) )
  }
}

// convert inventory file to format osmium-tool accepts
const mapper = (json, parents) => {
  return {
    'extracts': json.features.map(feat => {
      return {
        'output': `${path.join(...parents, feat.properties.id)}-${vintage}.osm.pbf`,
        'output_format': 'pbf,add_metadata=false',
        'description': feat.properties.name,
        'multipolygon': feat.geometry.coordinates
      }
    })
  }
}

// patch the inventory file to ensure that subregions of
// the USA are parented by 'us' not 'north-america' as they
// are in the canonical inventory file.
const patch = (json) => {
  return {
    features: json.features.map(feat => {
      if (feat.properties.id.includes(path.sep)) {
        let segments = feat.properties.id.split(path.sep)
        if (segments.length !== 2) { throw new Error('invalid id', feat.properties) }
        feat.properties.parent = segments[0]
        feat.properties.id = segments[1]
        feat.properties.name = segments[1]
      }
      return feat
    })
  }
}

// write shell script header
fs.writeFileSync('extract.sh', `
#!/bin/bash
set -euxo pipefail
`)

// fetch inventory file from geofabrik
fetch('https://download.geofabrik.de/index-v1.json')
  .then(res => res.json())
  .then(json => patch(json))
  .then(json => write(json, graph(json), []))