Skip to content

Instantly share code, notes, and snippets.

@bhelx bhelx/process.js
Last active Mar 1, 2016

Embed
What would you like to do?
Fetch and resize geojson images concurrently using async.js
"use strict";
let im = require('imagemagick');
let async = require('async');
let fs = require('fs');
let request = require('request');
let path = require('path');
let os = require('os');
let fetchAndProcess = (task, done) => {
let parts = path.parse(task.url).dir.split('/');
let filename = `data/images/${parts[parts.length-1]}.jpg`;
let output = `data/images_processed/${parts[parts.length-1]}.jpg`;
let process = () => im.convert([filename, '-resize', '1024x768', output], done);
request
.get(task.url)
.pipe(fs.createWriteStream(filename))
.on('error', done)
.on('finish', process);
};
let data = JSON.parse(fs.readFileSync('./data/new-orleans-crosswalk-lights.geojson'));
const CONCURRENCY = os.cpus().length;
const Q = async.queue(fetchAndProcess, CONCURRENCY);
let tasks = data.features.filter((f) => f.properties.image).map((f) => {
return { url: f.properties.image };
});
Q.push(tasks);
let progress = setInterval(() => {
console.log(`Processing ${Q.running()} images with ${Q.length()} waiting in the queue`);
}, 2500);
Q.drain = () => {
console.log("Q is drained");
clearInterval(progress);
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.