GME loan scraping

analyze-security.js

const fs = require('fs');
const path = require('path');

const ticker = 'gme';

const re = new RegExp(`${ticker.toUpperCase()}\\s+,\\s*(\\d+\\.?\\d*)\\s*,\\s*(\\d+\\.?\\d*)\\s*,\\s*`);

console.log('Date,Market Loan-Loan Balance,Hedge-Loan Balance,');

(async () => {
    const files = await fs.promises.readdir('./data');
    files.sort();

    for (let i = 0; i < files.length; ++i) {
        const parsed = path.parse(files[i]);
        const tokens = parsed.name.split('_');
        const date = new Date(
            Number.parseInt(tokens[0], 10),
            Number.parseInt(tokens[1], 10) - 1,
            Number.parseInt(tokens[2]));

        const csv = await fs.promises.readFile(path.join('./data', files[i]), 'utf8');
        // console.log(csv);

        const match = re.exec(csv);
        if (match) {
            console.log(`${date.getDate()}/${date.getMonth() + 1}/${date.getFullYear()},${match[1]},${match[2]},`);
        }
    }
})();

gme-data.csv

scrape-occ.js

const fetch = require('node-fetch');
const fs = require('fs');

const startDate = new Date(2021, 0);
const endDate = new Date();
const msPerDay = 86400000;
const fetchDelayMS = 1000;

function pad(s, d) {
    while (s.length < d) {
        s = '0' + s;
    }
    return s;
}

function scrapeURL (date) {
    const day = date.getDate();
    const month = date.getMonth() + 1;
    const year = date.getFullYear();
    return `https://marketdata.theocc.com/stock-loan-bal-by-security?dailyDate=${pad(month + '', 2)}/${pad(day + '', 2)}/${pad(year + '', 2)}&format=csv`
}

function csvName (date) {
    const day = date.getDate();
    const month = date.getMonth() + 1;
    const year = date.getFullYear();
    return `./data/${pad(year + '', 2)}_${pad(month + '', 2)}_${pad(day + '', 2)}.csv`;
}

function sleep (ms) {
    return new Promise((resolve) => {
        setTimeout(resolve, ms);
    });
}

(async () => {
    for (let d = +startDate; d < +endDate; d += msPerDay) {
        const date = new Date(d);
        const url = scrapeURL(date);
        try {
            const res = await fetch(url);
            const report = await res.text();
            if (report.startsWith('No record')) {
                console.log('No data for: ', date);
            } else {
                await fs.promises.writeFile(csvName(date), report);
                console.log(`Fetched data for ${date}`);
            }
        } catch (err) {
            console.error(`Error on data for date ${date}`);
            console.error(err);
        }
        await sleep(fetchDelayMS);
    }
})()

Here's how to use this stuff:

1. Install node.js
2. Install node-fetch using npm i node-fetch
3. Run the scraper script:

node scrape-occ.js

4. Run the analysis script:

node analyze-security.js > dump.csv

You can replace ticker with whatever symbol you want to check. This gives all the loan data from the occ's stock borrow program in a timeline.