Building a Unsplash Downloader with Deno https://ryanccn.dev/posts/deno-unsplash-downloader
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { writeAll } from 'https://deno.land/std@0.103.0/io/util.ts'; | |
import pLimit from 'https://cdn.skypack.dev/p-limit?dts'; | |
const exists = async (imgId: string) => { | |
try { | |
await Deno.stat(`./downloads/${imgId}.jpg`); | |
} catch { | |
return false; | |
} | |
return true; | |
}; | |
let count = 0; | |
const imgList: string[] = JSON.parse(await Deno.readTextFile('./data.json')); | |
let filteredImgList: string[] = []; | |
for (const img of imgList) { | |
if (await exists(img)) { | |
count++; | |
} else { | |
filteredImgList = [...filteredImgList, img]; | |
} | |
} | |
/* Download a specific image */ | |
const download = async (img: string) => { | |
const url = `https://source.unsplash.com/${img}/250x250`; | |
const res = await fetch(url); | |
if (!res.ok || !res.body) { | |
const percentage = ((count / 25000) * 100).toFixed(2); | |
console.log( | |
`[${percentage}%] ${img} download error: ${res.status} ${res.statusText}` | |
); | |
count++; | |
return; | |
} | |
const file = await Deno.open(`./downloads/${img}.jpg`, { | |
create: true, | |
write: true, | |
}); | |
for await (const chunk of res.body) { | |
await writeAll(file, chunk); | |
} | |
file.close(); | |
console.log( | |
`[${((count / 25000) * 100).toFixed(2)}%] ${img} download successful` | |
); | |
count++; | |
return; | |
}; | |
/* Download through the stack with concurrency 5 */ | |
const limit = pLimit(5); | |
await Promise.all( | |
filteredImgList.map((i) => { | |
return limit(() => download(i)); | |
}) | |
); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment