Created
November 8, 2019 22:51
-
-
Save jonbarrow/52ea1acc8232bb5f0b67a5a1289cd52f to your computer and use it in GitHub Desktop.
Scraper for wonderfulsubs.com
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const got = require('got'); | |
const async = require('async'); | |
const URL_BASE = 'https://www.wonderfulsubs.com/api/media'; | |
const SEARCH_URL = `${URL_BASE}/search?q`; | |
const SERIES_URL = `${URL_BASE}/series?series`; | |
const STREAM_URL = `${URL_BASE}/stream?code`; | |
// Options for "got" | |
const OPTIONS = { | |
json: true, | |
headers: { | |
referer: 'https://www.wonderfulsubs.com' | |
} | |
}; | |
async function scrape(kitsuDetails, episodeNumber=1) { | |
let streams = []; | |
const titleENG = kitsuDetails.attributes.titles.en; | |
const titleJPN = kitsuDetails.attributes.titles.en_jp; | |
const title = (titleENG || titleJPN); | |
const titleEncoded = encodeURIComponent(title); | |
let response = await got(`${SEARCH_URL}=${titleEncoded}`, OPTIONS); | |
let body = response.body; | |
const searchResults = body.json.series; | |
const anime = searchResults.find(({kitsu_id}) => (kitsu_id === kitsuDetails.id)); | |
const animeID = anime.url.split('/').pop(); | |
response = await got(`${SERIES_URL}=${animeID}`, OPTIONS); | |
body = response.body; | |
// Why in gods name is this designed this way. Why not just split the seasons like a normal site | |
const seasons = body.json.seasons.ws.media; | |
const season = seasons.find(season => (season.type === 'episodes' && (season.title === title || season.japanese_title === title))); | |
const episodes = season.episodes; | |
const episode = episodes.find(episode => episode.episode_number === episodeNumber); | |
const sources = episode.sources; | |
return new Promise(resolve => { | |
async.each(sources, (source, callback) => { | |
getEpisodeStreams(source) | |
.then(scrapedStreams => { | |
if (scrapedStreams) { | |
streams = mergeArrays(streams, scrapedStreams); | |
} | |
callback(); | |
}); | |
}, () => { | |
return resolve(streams); | |
}); | |
}); | |
} | |
async function getEpisodeStreams(source) { | |
const streams = []; | |
const dubbed = source.language === 'dubs'; | |
let fileHost = source.source; | |
function addStream(stream) { | |
const obj = { | |
provider: 'WS', | |
provider_full: 'Wonderful Subs', | |
file_host: fileHost, | |
file: stream.src, | |
dubbed | |
}; | |
if (stream.label === 'Auto (HLS)') { | |
obj.m3u8 = true; | |
if (stream.captions) { | |
obj.subtitles_file = stream.captions.src; | |
} | |
} | |
streams.push(obj); | |
} | |
switch (source.source) { | |
case 'cr': | |
fileHost = 'CrunchyRoll'; | |
break; | |
case 'ka': | |
fileHost = 'Kissanime'; | |
break; | |
case 'fa': | |
fileHost = 'Funimation'; | |
break; | |
case 'na': // Skip these since idk what they are | |
return; | |
default: | |
break; | |
} | |
return new Promise(resolve => { | |
if (typeof source.retrieve_url === 'string') { | |
got(`${STREAM_URL}=${source.retrieve_url}`, OPTIONS) | |
.then(response => { | |
const body = response.body.urls; | |
for (const stream of body) { | |
addStream(stream); | |
} | |
return resolve(streams); | |
}); | |
} else { | |
const urls = source.retrieve_url; | |
async.each(urls, (url, callback) => { | |
got(`${STREAM_URL}=${url}`, OPTIONS) | |
.then(response => { | |
const body = response.body.urls; | |
for (const stream in body) { | |
addStream(stream); | |
} | |
callback(); | |
}); | |
}, () => { | |
return resolve(streams); | |
}); | |
} | |
}); | |
} | |
module.exports = scrape; | |
function mergeArrays(...arrays) { | |
return [...new Set([].concat(...arrays))]; | |
} | |
// TESTING. REMOVE THIS IN PRODUCTION | |
(async () => { | |
console.time('Scrape Time'); | |
const streams = await scrape({ // Fake Kitsu response | |
id: '42080', | |
attributes: { | |
titles: { | |
en: 'Dr. STONE', | |
en_jp: 'Dr. Stone', | |
} | |
} | |
}, 1); | |
console.timeEnd('Scrape Time'); | |
console.log(streams); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
(node:12604) UnhandledPromiseRejectionWarning: TypeError: The
GETmethod cannot be used with a body at Object.exports.normalizeRequestArguments (/Users/alexandresantos/animeflix-addon/node_modules/got/dist/source/normalize-arguments.js:263:19)