Skip to content

Instantly share code, notes, and snippets.

@CrossEye
Last active January 27, 2023 15:13
Show Gist options
  • Save CrossEye/7f1b66570ba6e6df3fe0de686cbd1983 to your computer and use it in GitHub Desktop.
Save CrossEye/7f1b66570ba6e6df3fe0de686cbd1983 to your computer and use it in GitHub Desktop.
Simple crawler
// https://stackoverflow.com/a/75251817
// crawl :: ((a -> Promise b), (b -> [a]), (b -> c), ((Map a c) -> d), Int?) -> [a] -> Promise d
const crawl = (search, extract, convert, collect, simul = 1) => async (inits) => {
const toCheck = new Set (inits)
const found = new Map ()
while (toCheck .size !== 0) {
const nextBlock = [...toCheck .values ()] .slice (0, simul)
const results = await Promise .all (nextBlock .map ((x) => search (x)))
results .forEach ((res, i) => {
const key = nextBlock [i]
found .set (key, convert (res))
const newVals = extract (res)
newVals .forEach ((val) => found .has (val) || toCheck .add (val))
toCheck .delete (key)
})
}
return collect (found)
}
// Exmaple code
const get = ((data) => (id) => console .log (`Getting ${id}`) ||
new Promise ((res, rej) => setTimeout (() => res (data [id]), 1000))
)({ // don't look for any structure here. The fruits were more fun than "a", "b", "c", ...
1: {val: ['Apples', 'Apricots'], kids: [3, 5]},
2: {val: ['Bananas', 'Boysenberries', 'Blackberries', 'Blueberries'], kids: [3, 4]},
3: {val: ['Cherries', 'Cantaloupes', 'Clementines'], kids: []},
4: {val: ['Dates', 'Dewberries', 'Dragon Fruit'], kids: [1, 4, 5]},
5: {val: ['Elderberries', 'Eggfruit'], kids: [2]},
6: {val: ['Fig', 'Farkleberries'], kids: [6, 7]},
7: {val: ['Grapefruit', 'Grapes', 'Gooseberries', 'Guava'], kids: []},
})
const findFruit = crawl (
get,
(x) => x .kids,
(x) => x .val,
(m) => [... m .values ()],
3
)
findFruit ([1]) // or, say, [1, 6]
.then (letters => console .log ('Results:', letters))
.catch (err => console .log (`Error: ${err}`))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment