Skip to content

Instantly share code, notes, and snippets.

@mikolalysenko
Created July 16, 2021 20:28
Show Gist options
  • Save mikolalysenko/dd8f1aa1334ff22b1e9162028b12fe4c to your computer and use it in GitHub Desktop.
Save mikolalysenko/dd8f1aa1334ff22b1e9162028b12fe4c to your computer and use it in GitHub Desktop.
scrapes loan data from occ
const fetch = require('node-fetch');
const fs = require('fs');
const startDate = new Date(2021, 0);
const endDate = new Date();
const msPerDay = 86400000;
const fetchDelayMS = 1000;
function pad(s, d) {
while (s.length < d) {
s = '0' + s;
}
return s;
}
function scrapeURL (date) {
const day = date.getDate();
const month = date.getMonth() + 1;
const year = date.getFullYear();
return `https://marketdata.theocc.com/stock-loan-bal-by-security?dailyDate=${pad(month + '', 2)}/${pad(day + '', 2)}/${pad(year + '', 2)}&format=csv`
}
function csvName (date) {
const day = date.getDate();
const month = date.getMonth() + 1;
const year = date.getFullYear();
return `./data/${pad(year + '', 2)}_${pad(month + '', 2)}_${pad(day + '', 2)}.csv`;
}
function sleep (ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
(async () => {
for (let d = +startDate; d < +endDate; d += msPerDay) {
const date = new Date(d);
const url = scrapeURL(date);
try {
const res = await fetch(url);
const report = await res.text();
if (report.startsWith('No record')) {
console.log('No data for: ', date);
} else {
await fs.promises.writeFile(csvName(date), report);
console.log(`Fetched data for ${date}`);
}
} catch (err) {
console.error(`Error on data for date ${date}`);
console.error(err);
}
await sleep(fetchDelayMS);
}
})()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment