Node web scraper with axios and cheerio

web-scraper-node-js.md

Node JS Web Scraper

I this is part of the first node web scraper I created with axios and cheerio. I took out all of the logic, since I only wanted to showcase how a basic setup for a nodejs web scraper would look.

const cheerio = require('cheerio'),
      axios = require('axios'),
      url = `<url goes here>`;
      
axios.get(url)
    .then((response) => {
        let $ = cheerio.load(response.data);
        $('a').each(function (i, e) {
          let links = $(e).attr('href');      
          console.log(links);
      })
    }).catch(function (e) {
    console.log(e);
});

k

Thanks for this 🔥

Appreciate it so much!

thanks

Very nice!!

If you wonder about the axios response check here : https://axios-http.com/docs/res_schema

//crawl
//get url
var url = 'http://amazon.com';

axios.get(url)
.then((res) => {

//cheerio   
//https://axios-http.com/docs/res_schema

//body
var body = res.data;
var statusCode = res.status;
var statusText =  res.statusText;
var headers = res.headers;
var request = res.request;
var config = res.config;

//jquery
let $ = cheerio.load(body);

//variables
console.log(body);
console.log(statusCode);
console.log(statusText);
console.log(headers);
console.log(config);
console.log(request);


}).catch(function (e) {
console.log(e);
});