Fetch and resize geojson images concurrently using async.js

process.js

"use strict";

let im = require('imagemagick');
let async = require('async');
let fs = require('fs');
let request = require('request');
let path = require('path');
let os = require('os');

let fetchAndProcess = (task, done) => {
  let parts = path.parse(task.url).dir.split('/');
  let filename = `data/images/${parts[parts.length-1]}.jpg`;
  let output = `data/images_processed/${parts[parts.length-1]}.jpg`;
  let process = () => im.convert([filename, '-resize', '1024x768', output], done);

  request
    .get(task.url)
    .pipe(fs.createWriteStream(filename))
    .on('error', done)
    .on('finish', process);
};

let data = JSON.parse(fs.readFileSync('./data/new-orleans-crosswalk-lights.geojson'));

const CONCURRENCY = os.cpus().length;
const Q = async.queue(fetchAndProcess, CONCURRENCY);

let tasks = data.features.filter((f) => f.properties.image).map((f) => {
  return { url: f.properties.image };
});

Q.push(tasks);

let progress = setInterval(() => {
  console.log(`Processing ${Q.running()} images with ${Q.length()} waiting in the queue`);
}, 2500);

Q.drain = () => {
  console.log("Q is drained");
  clearInterval(progress);
};