Skip to content

Instantly share code, notes, and snippets.

@ryanccn
Created Aug 10, 2021
Embed
What would you like to do?
Building a Unsplash Downloader with Deno https://ryanccn.dev/posts/deno-unsplash-downloader
import { writeAll } from 'https://deno.land/std@0.103.0/io/util.ts';
import pLimit from 'https://cdn.skypack.dev/p-limit?dts';
const exists = async (imgId: string) => {
try {
await Deno.stat(`./downloads/${imgId}.jpg`);
} catch {
return false;
}
return true;
};
let count = 0;
const imgList: string[] = JSON.parse(await Deno.readTextFile('./data.json'));
let filteredImgList: string[] = [];
for (const img of imgList) {
if (await exists(img)) {
count++;
} else {
filteredImgList = [...filteredImgList, img];
}
}
/* Download a specific image */
const download = async (img: string) => {
const url = `https://source.unsplash.com/${img}/250x250`;
const res = await fetch(url);
if (!res.ok || !res.body) {
const percentage = ((count / 25000) * 100).toFixed(2);
console.log(
`[${percentage}%] ${img} download error: ${res.status} ${res.statusText}`
);
count++;
return;
}
const file = await Deno.open(`./downloads/${img}.jpg`, {
create: true,
write: true,
});
for await (const chunk of res.body) {
await writeAll(file, chunk);
}
file.close();
console.log(
`[${((count / 25000) * 100).toFixed(2)}%] ${img} download successful`
);
count++;
return;
};
/* Download through the stack with concurrency 5 */
const limit = pLimit(5);
await Promise.all(
filteredImgList.map((i) => {
return limit(() => download(i));
})
);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment