Skip to content

Instantly share code, notes, and snippets.

@rileyjshaw
Last active March 22, 2024 11:01
Show Gist options
  • Star 5 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rileyjshaw/138e7ff15982c65fb6dd95c76c2cc813 to your computer and use it in GitHub Desktop.
Save rileyjshaw/138e7ff15982c65fb6dd95c76c2cc813 to your computer and use it in GitHub Desktop.
Download all issues of Emigre Magazine from Letterform Archive’s Online Archive
# Move the results into their own folder, drag that parent folder into imageOptim, `cd` into it, then:
for d in Issue\ */; do
cd "$d"
convert -compress jpeg -quality 33 -resize 2388x2388\> *.jpg "../${d%/}.pdf"
cd ..
done
import fs from 'fs';
import fetch from 'node-fetch';
const N_ISSUES = 69;
// We rely on a 404 to tell us when we've reached the end of an issue. We
// request multiple pages at once to speed up the process, and continue until
// one of the pages in a batch returns a 404.
const PAGE_BATCH_SIZE = 2;
const pad = (num, size) => num.toString().padStart(size, '0');
for (let issue = 1; issue <= N_ISSUES; ++issue) {
getIssue(issue);
}
async function getIssue(issue) {
// Create a folder for this issue.
const formattedIssue = pad(issue, 4);
const folderName = `Issue ${pad(issue, 2)}`;
fs.mkdirSync(folderName);
let hasMorePages = true;
let pageOffset = 0;
while (hasMorePages) {
const requests = Array.from({ length: PAGE_BATCH_SIZE }, (_, i) => {
const page = pageOffset + i + 1;
const formattedPage = pad(page, 3);
function getPage(nRetries) {
if (nRetries > 2) {
return console.error(`FAILURE! Hit the retry limit fetching page ${page} of issue ${issue}.`);
}
return fetch(
`https://oa.letterformarchive.org/oaapi/image/lfa_emigre_${formattedIssue}/lfa_emigre_${formattedIssue}_${formattedPage}`,
{
headers: {
Referer: `https://oa.letterformarchive.org/item?workID=lfa_emigre_${formattedIssue}&targPic=lfa_emigre_${formattedIssue}_${formattedPage}.jpg`,
},
}
)
.then(res => {
// A 404 implies the end of the issue.
if (res.status === 404) {
hasMorePages = false;
return;
}
// Otherwise, save the image.
res.body.pipe(fs.createWriteStream(`${folderName}/${formattedPage}.jpg`));
})
.catch(async _err => {
// Retry on error.
return await getPage(nRetries + 1);
});
}
return getPage(0);
});
await Promise.all(requests);
pageOffset += PAGE_BATCH_SIZE;
}
console.log(`Success! Got issue ${issue}.`);
}
@ciosyam
Copy link

ciosyam commented May 22, 2023

thank you so much for this !

@GabiSchaffzin
Copy link

Worked like a charm. Thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment