View scrape.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const cheerio = require('cheerio'); | |
const fs = require('fs'); | |
const readline = require('readline'); | |
const get_post_data = require('./get_POST_data'); | |
const get_get_data = require('./get_GET_data'); | |
let callback_url = 'https://xyz.com'; | |
// this form data object contain the query to be sent. | |
let form_data = { |
View get_POST_data.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const https = require('https'); | |
const http = require('http'); | |
const querystring = require('querystring'); | |
const url = require('url'); | |
let get_POST_data = function(urlToPost, formData, headersToSend) { | |
return new Promise(function(resolve, reject) { | |
let parsed_url = url.parse(urlToPost); | |
let post_data = querystring.stringify(formData); | |
let secure = false; |
View get_GET_data.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const https = require('https'); | |
const http = require('http'); | |
const querystring = require('querystring'); | |
const url = require('url'); | |
let get_GET_data = function(urlToGET) { | |
return new Promise(function(resolve, reject) { | |
//console.log("get_GET_data was called"); | |
urlToGET = urlToGET + ""; | |
//console.log(urlToGET + " = url"); |