Skip to content

Instantly share code, notes, and snippets.

@eahrold
Last active March 30, 2020 19:28
Show Gist options
  • Save eahrold/42944ea60f7afe98f988f0ed2a9d3a4e to your computer and use it in GitHub Desktop.
Save eahrold/42944ea60f7afe98f988f0ed2a9d3a4e to your computer and use it in GitHub Desktop.
Chunkable Promises
const _ = require('lodash')
/**
* Async Waiting utility
* @param {Number} wait Number of MS to wait
* @return {Promise} A promise to await
*/
const waiter = async function(wait = 0) {
return new Promise((resolve) => {
if (wait <= 0) {
return resolve()
}
setTimeout(() => {
resolve()
}, wait)
})
}
/**
* Map a collection that requries await/async
* @param {Array} array Array of items
* @param {Function} callback [description]
* @return {Array} Mapped results
*/
const asyncMap = async function(array, callback) {
const map = []
for (var index = 0; index < array.length; index++) {
const value = await callback(array[index], index)
map.push(value)
}
return map
}
/**
* Chunk a Group of Requests and execute according to API Limitations
* @param {array<Function>} array an array of functions that return a thenable when called
* @param {Number} size The Number of items per chunk
* @param {Number} throttle The Number of ms to wait until a chunk has completed
*
* Note: the idea is that if the api can handle 6 calls/second you can chunck a
* batches of 6 api requests, that will wait up to 1000ms before the next chunk is called.
*
* @return {array} Fulfilled data from the request.
*/
const chunkRequests = async function(array, size = 3, throttle = 1000) {
const chunks = _.chunk(array, size)
const total = chunks.length
var count = 0
return await asyncMap(chunks, async (chunk) => {
console.log(
`Processing Request Chunks ${++count} of ${total} (size ${size})`
)
const start = new Date().getTime()
const results = await Promise.all(chunk.map((c) => c()))
const end = new Date().getTime()
const wait = start + throttle - end
if (wait > 0) {
await waiter(wait)
}
return results
}).then(_.flatMap)
}
const Preq = {
load: async function(item) {
return new Promise((r) => {
setTimeout(() => {
r({ b: item.a })
}, 2000)
})
},
}
const test = async function() {
var items = new Array(17).fill().map((v, a) => {
return { a }
})
const chunkables = items.map((request) => {
return () => {
return Preq.load(request).then(async (response) => {
return { ...request, ...response }
})
}
})
const results = await chunkRequests(chunkables)
console.log(results)
}
test()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment