Scraping https://kocham.kr/theme/inet/sub/detail.php?wr_id=1

index.js

const cheerio = require("cheerio")
const axios = require("axios")
const fs = require("fs")

async function execute() {
  let final = []

  for (let i = 1; i < 2520; i++) {
    try {
      console.log("Running on item: ", i)
      const response = await axios.get(getPageUrl(i))
      const htmlRaw = response.data
      console.log("Loading page ", i)
      const query = cheerio.load(htmlRaw)
      const data = processPage(query)

      final.push(convertingCSVFormat(data))
    } catch (err) {
      console.log("Error on page : ", i, err)
      return err
    }
  }

  console.log("Done. Writing to file....")
  fs.writeFile("output.json", JSON.stringify(final), (err) => {
    console.log(err)
  })
  console.log("Writing to file done.")
}

execute().then((value) => console.log("Exiting app..."))

function getPageUrl(id) {
  return `https://kocham.kr/theme/inet/sub/detail.php?wr_id=${id}`
}

function processPage($) {
  let result = []

  $(".subtable_list")
    .find("tr")
    .each((i, row) => {
      const rowData = {}
      const cells = $(row)
        .find("td, th")
        .each((j, cell) => {
          let header = $(cell).text()
          rowData[header] = j
        })
      result.push(rowData)
    })

  return result
}

function convertingCSVFormat(data) {
  let obj = {}
  data.forEach((el) => {
    let header = Object.keys(el)[0]
    obj[header] = Object.keys(el)[1]
  })
  return obj
}