Script to scrap company stock price from ASX.

how_are_you_comps.js

/*
We want to know how are the companies doing during VOVID19 simply look at their stock price.

- scope: ASX companies. AKA comp_list.  Based on a ASX official list.
1. get the comp_list
2. get the price data by requesting yahoo API
for comp in comp_list: 
  2.1 get the before_COVID_price
  2.2 get the current_price
  3. get the price_change_percentage by compare result of 1&2
  4. store result of 1,2,3 to result_list

3. sort the result_list by price_change_percentage, descending order. And export.
3.1 get "feeling_good_list" by selecting top 10% comps, export as well
*/

const yahooFinance = require('yahoo-finance')
const moment = require('moment')
const csv = require('csvtojson')
const createCsvWriter = require('csv-writer').createObjectCsvWriter
const fs = require('fs')

const csvFilePath = './ASXListedCompanies.csv'
// change these keys base on your source data
const compNameKey = 'Company name'
const symbolKey = 'ASX code'
const groupKey = 'GICS industry group'

var compList = []
const curruptData = {}

main = async () => {
  compList = await csv().fromFile(csvFilePath) // load data
  await batchDownload(0, 50, 500, 'dl3.csv')
  fs.writeFileSync('curruptData.json', JSON.stringify(curruptData))
}

batchDownload = async (startPage, size, waitPeriod, path) => {
  for (let page = startPage; page < 2 / size; page++) {
    const compDict = await readCompDict(page, size)
    const options = {
      symbols: Object.keys(compDict),
      from: '2019-12-01',
      to: moment().format('YYYY[-]MM[-]DD'),
      period: 'm',
    }

    const results = []
    // reqeust and write into file
    yahooFinance.historical(options, (err, quotes) => {
      if (err) {
        console.log(err)
        return
      }
      for (const symbol in quotes) {
        if (quotes.hasOwnProperty(symbol)) {
          const dataList = quotes[symbol]

          const isDataCorrupt =
            !dataList ||
            !dataList[0] ||
            !dataList[0].close ||
            !dataList[dataList.length - 1] ||
            !dataList[dataList.length - 1].close

          if (isDataCorrupt) {
            curruptData[symbol] = quotes[symbol]
            compDict[symbol]['before_COVID_price'] = 0
            compDict[symbol]['current_price'] = 0
          } else {
            fs.writeFileSync('Data.json', JSON.stringify(dataList))
            compDict[symbol]['before_COVID_price'] = dataList[0].close
            compDict[symbol]['current_price'] =
              dataList[dataList.length - 1].close
          }
        }
      }

      writeCompDict(compDict, page, size, path)
    })
    await new Promise((r) => setTimeout(r, waitPeriod))
  }
}

// pagination: page start from 0
readCompDict = async (page, size) => {
  const compDict = {}
  compList
    .slice(page * size, (page + 1) * size)
    .forEach((comp) => (compDict[comp[symbolKey] + '.ax'] = comp))
  return compDict
}

writeCompDict = (compDict, page, size, path) => {
  const csvWriter = createCsvWriter({
    append: true,
    path,
    header: [
      { id: 'Company name', title: 'Company name' },
      { id: 'ASX code', title: 'ASX code' },
      { id: 'GICS industry group', title: 'GICS industry group' },
      { id: 'before_COVID_price', title: 'previous price' },
      { id: 'current_price', title: 'Current price' },
    ],
  })
  const compList = Object.values(compDict)
  csvWriter
    .writeRecords(compList)
    .then(() =>
      console.log(`${page * size + 1}-${(page + 1) * size} is writen`)
    )
}

main()

package.json

{
  "name": "au-company-price-through-covid19",
  "version": "1.0.0",
  "description": "",
  "main": "how_are_you_comps.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [
    "how_are_you_mate",
    "stay_strong",
    "CODVID19"
  ],
  "author": "Jason",
  "license": "ISC",
  "dependencies": {
    "csv-writer": "^1.6.0",
    "csvtojson": "^2.0.10",
    "json2csv": "^5.0.0",
    "moment": "^2.24.0",
    "yahoo-finance": "^0.3.6"
  }
}