Skip to content

Instantly share code, notes, and snippets.

@ShaunLWM
Created January 4, 2019 03:56
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ShaunLWM/5337991d8c202b1f01b7567d93880d17 to your computer and use it in GitHub Desktop.
Save ShaunLWM/5337991d8c202b1f01b7567d93880d17 to your computer and use it in GitHub Desktop.
const request = require('request');
const async = require('async');
const fs = require('fs');
const download = require('download');
let page = 0;
async.whilst(
function () { return page < 5440; },
// function () { return page < 20; },
function (callback) {
console.log(`Page: ${page}`);
requestPage(`http://nokiatune.audiodraft.com/entries/toprated/${page}/`, (error, response, body) => {
if (error) {
console.log('----- ERROR ------');
console.log(`http://nokiatune.audiodraft.com/entries/toprated/${page}/`);
console.error(error);
console.log('----------------');
return callback(null);
}
if (response.statusCode !== 200) {
console.log('----- ERROR RESPONSE ------');
console.log(`http://nokiatune.audiodraft.com/entries/toprated/${page}/`);
console.error(response.statusCode);
console.log('----------------');
return callback(null);
}
fs.writeFileSync(`./pages/${page}.html`, body);
let parsed = [];
let re = new RegExp(/onclick="playAudio\('(.*?)'\);/g);
let matches;
while ((matches = re.exec(body)) != null) {
if (matches.length > 0) {
parsed.push(matches[1]);
}
}
async.eachLimit(parsed, 3, (match, callback) => {
// console.log(`Parsing: ${match}`);
getDownloadLink(match, (error, result) => {
if (result.length > 0) {
let body = JSON.parse(result);
if (body.error !== 1) {
//console.log(result);
//job.addPageLink(JSON.parse(result));
let name = body.player_text.replace(/Playing: /g, '');
try {
download(body.path).pipe(fs.createWriteStream(`./audio/${name}.mp3`));
console.log(`Downloaded: ${name}`);
writeLogs(name, 0);
} catch (error) {
writeLogs(name, error);
}
}
}
return callback();
});
}, function (err) {
console.log('Done all download link');
});
page += 20;
setTimeout(() => {
console.log('------------------------------');
return callback(null);
}, 2000);
});
},
function (err, n) {
console.log('Done');
}
);
function requestPage(url, callback) {
const options = {
url,
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
};
return request(options, callback);
}
function getDownloadLink(id, callback) {
// http://nokiatune.audiodraft.com/ajax_player/entryinfo
const options = {
url: 'http://nokiatune.audiodraft.com/ajax_player/entryinfo',
headers: {
'Accept': 'text/javascript, text/html, application/xml, text/xml, */*',
'Accept-Language': 'en-US,en-SG;q=0.9,en;q=0.8',
'Connection': 'keep-alive',
'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'DNT': 1,
'Host': 'nokiatune.audiodraft.com',
'Origin': 'http://nokiatune.audiodraft.com',
'Referer': 'http://nokiatune.audiodraft.com/entries/toprated/0',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
'X-Prototype-Version': '1.6.1',
'X-Requested-With': 'XMLHttpRequest'
},
form: {
id: id,
_: ''
}
};
request.post(options, function (error, response, body) {
if (error) {
console.log(`${id} ${error}`);
return callback(null, '')
}
if (response.statusCode !== 200) {
console.log(`${id} not 200`);
return callback(null, '');
}
// console.log('got download link');
return callback(null, body);
});
}
function writeLogs(name, error) {
let file = JSON.parse(fs.readFileSync('./links.json'));
file.push({ name, error });
fs.writeFileSync('./links.json', JSON.stringify(file, null, 2));
}
{
"name": "nokia-download",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"dependencies": {
"async": "^2.6.1",
"download": "^7.1.0",
"request": "^2.88.0"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment