Skip to content

Instantly share code, notes, and snippets.

@dotmilk
Created April 18, 2020 14:10
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dotmilk/b9b5610ab065fb7aef726bd1bede934b to your computer and use it in GitHub Desktop.
Save dotmilk/b9b5610ab065fb7aef726bd1bede934b to your computer and use it in GitHub Desktop.
let axios = require("axios")
let cheerio = require("cheerio")
let download = require("download")
let fs = require('fs')
// edit these two thing only
let consoleName = "super-nintendo"
let gameRegion = "france"
// dont touch below here
let currentPage = 1
let thingsToDownload = []
let pageCount = 0
async function fetchNewPage() {
let siteUrl = `https://romsmania.cc/roms/${consoleName}/search?page=${currentPage}&region=${gameRegion}`
let result = await axios.get(siteUrl)
return cheerio.load(result.data)
}
async function fetchDownloadPage(url) {
let result = await axios.get(url)
return cheerio.load(result.data)
}
function convertLink(title){
return `https://romsmania.cc/download/roms/${consoleName}/${title}`
}
async function getResults(cb) {
let $ = await fetchNewPage()
pageCount = parseInt($('.pagination__list :nth-last-child(2)').text())
for (var i = 0; i < pageCount; i++) {
let linksOnPage = $('.table tbody tr td:first-of-type a')
linksOnPage.each(function(i,link){
let tmpUrl = link.attribs.href
tmpUrl = tmpUrl.substring(tmpUrl.lastIndexOf('/') + 1)
thingsToDownload.push(convertLink(tmpUrl))
})
currentPage++
console.log(`on page ${i+1} of ${pageCount}, ${thingsToDownload.length} urls so far`)
$ = await fetchNewPage()
}
await thingsToDownload.forEach(async (url, i) => {
let $ = await fetchDownloadPage(url)
let actualDownloadLink = $('a.wait__link')
actualDownloadLink.each(async function(i,theUrl){
let fileUrl = theUrl.attribs.href
let romTitle = fileUrl.substring(fileUrl.lastIndexOf('/') + 1)
romTitle = decodeURI(romTitle)
console.log(`Downloading ${romTitle}`)
await download(fileUrl).pipe(fs.createWriteStream(`roms/${romTitle}`))
})
//console.log(actualDownloadLink)
})
cb()
}
getResults(function(){
console.log("done scraping, now downloading....be patient")
})
{
"name": "romspyder",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"axios": "^0.19.2",
"cheerio": "^1.0.0-rc.3",
"download": "^8.0.0"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment