Skip to content

Instantly share code, notes, and snippets.

@Sasquire
Created February 14, 2020 17:59
Show Gist options
  • Save Sasquire/5f67f603249fb73c9aabedec39d3a733 to your computer and use it in GitHub Desktop.
Save Sasquire/5f67f603249fb73c9aabedec39d3a733 to your computer and use it in GitHub Desktop.
A simple script that will download information about sets and determine the user who owns the most sets.
const fs = require('fs');
const request_main = require('request');
function request(url) {
let options = {url:url, headers: { 'User-Agent': 'Downloading user set information' }};
console.log(options);
return new Promise(function (resolve, reject) {
request_main(options, function (error, res, body) {
console.log(res.statusCode);
if (!error && res.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
}
async function start(){
for(let i = 1; i < 41; i++){
try{
const text = await request('https://e621.net/set/index.xml?page='+i);
await writefile('/path/to/file'+i+'.xml', text);
console.log('wrote '+i);
} catch(e) {
console.log(e)
}
}
}
start();
async function writefile(path, tex){
return new Promise(function(resolve, reject){
fs.writeFile(path, tex, function(err){
resolve();
})
});
}
// grep -hEi "<user_id type=\"integer\">([0-9]*)</user_id>" *.xml | grep -oEi '[0-9]+' | sort | uniq -c | sort
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment