Skip to content

Instantly share code, notes, and snippets.

@marwanbayoumi
Last active October 10, 2020 17:17
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save marwanbayoumi/e48631450f2f0d55d355f3dbe6fb9f7d to your computer and use it in GitHub Desktop.
Save marwanbayoumi/e48631450f2f0d55d355f3dbe6fb9f7d to your computer and use it in GitHub Desktop.
Async scrape then deserialize JSON
const fs = require('fs');
const process = require('process');
const spawn = require("child_process").spawn;
process.chdir('./dir/of/spider');
async function test() {
try {
await scrape();
desrialize();
} catch (error) {
console.log(error);
}
}
test();
function desrialize() {
fs.readFile('./output.file', 'utf-8', (err, data) => {
if (err) {
throw err;
} else {
JSON.parse(data).forEach(e => {
console.log(e);
});
}
});
}
function scrape() {
return new Promise((resolve, reject) => {
let tool = spawn('scrapy', [
'crawl', 'async',
'-o', 'out.json'
]);
tool.on('exit', code => {
if (code == 0) {
resolve();
} else {
reject();
}
});
}
)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment