Skip to content

Instantly share code, notes, and snippets.

@gander
Last active December 8, 2022 13:04
Show Gist options
  • Save gander/60ff8faf48cf940b592800496da18976 to your computer and use it in GitHub Desktop.
Save gander/60ff8faf48cf940b592800496da18976 to your computer and use it in GitHub Desktop.
const axios = require("axios");
const fs = require("fs");
async function main() {
const positions = [...Array(206).keys()].map(k => k.toString().padStart(3, '0'));
for (const position of positions) {
await download(position).catch(console.error);
}
}
async function download(position) {
const output = __dirname + '/output';
const file = `concerned${position}.jpg`;
const outputFile = `${output}/${file}`;
if (await exists(outputFile)) {
return;
}
const resource = `http://www.hlcomic.com:80/comics/${file}`;
const url = `https://archive.org/wayback/available?url=${resource}&timestamp=2005`;
const {data: {archived_snapshots}} = await axios.get(url);
const {closest} = archived_snapshots;
if (!closest?.available) {
console.log(`Not exists`, resource, archived_snapshots);
return;
}
const image = `https://web.archive.org/web/${closest.timestamp}id_/${resource}`;
console.log(`Download:`, image, file);
await axios({method: 'get', url: image, responseType: 'stream'})
.then(response => response.data.pipe(fs.createWriteStream(outputFile)));
}
async function exists(file) {
return new Promise(resolve => fs.stat(file, err => resolve(!err)));
}
main().catch(console.error);
// https://archive.org/help/wayback_api.php
// https://en.wikipedia.org/wiki/Help:Using_the_Wayback_Machine#Specific_archive_copy
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment