Created
April 18, 2020 14:10
-
-
Save dotmilk/b9b5610ab065fb7aef726bd1bede934b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let axios = require("axios") | |
let cheerio = require("cheerio") | |
let download = require("download") | |
let fs = require('fs') | |
// edit these two thing only | |
let consoleName = "super-nintendo" | |
let gameRegion = "france" | |
// dont touch below here | |
let currentPage = 1 | |
let thingsToDownload = [] | |
let pageCount = 0 | |
async function fetchNewPage() { | |
let siteUrl = `https://romsmania.cc/roms/${consoleName}/search?page=${currentPage}®ion=${gameRegion}` | |
let result = await axios.get(siteUrl) | |
return cheerio.load(result.data) | |
} | |
async function fetchDownloadPage(url) { | |
let result = await axios.get(url) | |
return cheerio.load(result.data) | |
} | |
function convertLink(title){ | |
return `https://romsmania.cc/download/roms/${consoleName}/${title}` | |
} | |
async function getResults(cb) { | |
let $ = await fetchNewPage() | |
pageCount = parseInt($('.pagination__list :nth-last-child(2)').text()) | |
for (var i = 0; i < pageCount; i++) { | |
let linksOnPage = $('.table tbody tr td:first-of-type a') | |
linksOnPage.each(function(i,link){ | |
let tmpUrl = link.attribs.href | |
tmpUrl = tmpUrl.substring(tmpUrl.lastIndexOf('/') + 1) | |
thingsToDownload.push(convertLink(tmpUrl)) | |
}) | |
currentPage++ | |
console.log(`on page ${i+1} of ${pageCount}, ${thingsToDownload.length} urls so far`) | |
$ = await fetchNewPage() | |
} | |
await thingsToDownload.forEach(async (url, i) => { | |
let $ = await fetchDownloadPage(url) | |
let actualDownloadLink = $('a.wait__link') | |
actualDownloadLink.each(async function(i,theUrl){ | |
let fileUrl = theUrl.attribs.href | |
let romTitle = fileUrl.substring(fileUrl.lastIndexOf('/') + 1) | |
romTitle = decodeURI(romTitle) | |
console.log(`Downloading ${romTitle}`) | |
await download(fileUrl).pipe(fs.createWriteStream(`roms/${romTitle}`)) | |
}) | |
//console.log(actualDownloadLink) | |
}) | |
cb() | |
} | |
getResults(function(){ | |
console.log("done scraping, now downloading....be patient") | |
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "romspyder", | |
"version": "1.0.0", | |
"description": "", | |
"main": "index.js", | |
"scripts": { | |
"test": "echo \"Error: no test specified\" && exit 1" | |
}, | |
"author": "", | |
"license": "ISC", | |
"dependencies": { | |
"axios": "^0.19.2", | |
"cheerio": "^1.0.0-rc.3", | |
"download": "^8.0.0" | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment