Building a Unsplash Downloader with Deno https://ryanccn.dev/posts/deno-unsplash-downloader

mod.ts

import { writeAll } from 'https://deno.land/std@0.103.0/io/util.ts';
import pLimit from 'https://cdn.skypack.dev/p-limit?dts';

const exists = async (imgId: string) => {
  try {
    await Deno.stat(`./downloads/${imgId}.jpg`);
  } catch {
    return false;
  }

  return true;
};

let count = 0;

const imgList: string[] = JSON.parse(await Deno.readTextFile('./data.json'));
let filteredImgList: string[] = [];

for (const img of imgList) {
  if (await exists(img)) {
    count++;
  } else {
    filteredImgList = [...filteredImgList, img];
  }
}

/* Download a specific image */

const download = async (img: string) => {
  const url = `https://source.unsplash.com/${img}/250x250`;

  const res = await fetch(url);

  if (!res.ok || !res.body) {
    const percentage = ((count / 25000) * 100).toFixed(2);
    console.log(
      `[${percentage}%] ${img} download error: ${res.status} ${res.statusText}`
    );

    count++;
    return;
  }

  const file = await Deno.open(`./downloads/${img}.jpg`, {
    create: true,
    write: true,
  });

  for await (const chunk of res.body) {
    await writeAll(file, chunk);
  }

  file.close();

  console.log(
    `[${((count / 25000) * 100).toFixed(2)}%] ${img} download successful`
  );
  count++;

  return;
};

/* Download through the stack with concurrency 5 */

const limit = pLimit(5);

await Promise.all(
  filteredImgList.map((i) => {
    return limit(() => download(i));
  })
);