Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
GME loan scraping
const fs = require('fs');
const path = require('path');
const ticker = 'gme';
const re = new RegExp(`${ticker.toUpperCase()}\\s+,\\s*(\\d+\\.?\\d*)\\s*,\\s*(\\d+\\.?\\d*)\\s*,\\s*`);
console.log('Date,Market Loan-Loan Balance,Hedge-Loan Balance,');
(async () => {
const files = await fs.promises.readdir('./data');
files.sort();
for (let i = 0; i < files.length; ++i) {
const parsed = path.parse(files[i]);
const tokens = parsed.name.split('_');
const date = new Date(
Number.parseInt(tokens[0], 10),
Number.parseInt(tokens[1], 10) - 1,
Number.parseInt(tokens[2]));
const csv = await fs.promises.readFile(path.join('./data', files[i]), 'utf8');
// console.log(csv);
const match = re.exec(csv);
if (match) {
console.log(`${date.getDate()}/${date.getMonth() + 1}/${date.getFullYear()},${match[1]},${match[2]},`);
}
}
})();
Date Market Loan-Loan Balance Hedge-Loan Balance
4/1/2021 6300000.00 521949429.00
5/1/2021 6300000.00 543554502.00
6/1/2021 5846300.00 580727376.00
7/1/2021 8143400.00 574553466.00
8/1/2021 9093400.00 580071211.00
11/1/2021 13755000.00 627710704.00
12/1/2021 13755000.00 661710954.00
13/1/2021 21615000.00 1075260407.00
14/1/2021 26900100.00 1350479619.00
15/1/2021 15917400.00 1055906238.00
19/1/2021 7125800.00 1145882024.00
20/1/2021 6952000.00 1145034400.00
21/1/2021 10093600.00 1322935572.00
22/1/2021 15369800.00 2045028451.00
25/1/2021 36687600.00 2340909741.00
26/1/2021 79033400.00 3888263329.00
27/1/2021 142390500.00 7227744330.00
28/1/2021 79417800.00 2004265758.00
29/1/2021 33200000.00 1426181403.00
1/2/2021 0.00 1125653655.00
2/2/2021 0.00 392402882.00
3/2/2021 0.00 334107020.00
4/2/2021 0.00 163070380.00
5/2/2021 0.00 140706456.00
8/2/2021 0.00 128983932.00
9/2/2021 0.00 93822872.00
10/2/2021 0.00 106105258.00
11/2/2021 0.00 106847788.00
12/2/2021 0.00 99715644.00
16/2/2021 0.00 40145415.00
17/2/2021 0.00 35925155.00
18/2/2021 0.00 17289300.00
19/2/2021 0.00 46277700.00
22/2/2021 0.00 43326950.00
23/2/2021 0.00 27124590.00
24/2/2021 0.00 145283110.00
25/2/2021 0.00 235126965.00
26/2/2021 0.00 163182760.00
1/3/2021 0.00 204458595.00
2/3/2021 0.00 206052715.00
3/3/2021 0.00 106224705.00
4/3/2021 0.00 66163500.00
5/3/2021 0.00 62040000.00
8/3/2021 0.00 36218000.00
9/3/2021 0.00 35128800.00
10/3/2021 0.00 72275700.00
11/3/2021 0.00 109405800.00
12/3/2021 0.00 102924000.00
15/3/2021 0.00 88537500.00
16/3/2021 0.00 77681100.00
17/3/2021 0.00 76303500.00
18/3/2021 0.00 69978200.00
19/3/2021 0.00 65825500.00
22/3/2021 0.00 64575500.00
23/3/2021 0.00 33889200.00
24/3/2021 0.00 27625800.00
25/3/2021 0.00 47940000.00
26/3/2021 0.00 38184000.00
29/3/2021 0.00 41310500.00
30/3/2021 0.00 64737800.00
31/3/2021 0.00 34338600.00
1/4/2021 0.00 58461200.00
5/4/2021 0.00 81453000.00
6/4/2021 0.00 93323900.00
7/4/2021 819000.00 81639822.00
8/4/2021 783000.00 54732600.00
9/4/2021 0.00 57286800.00
12/4/2021 0.00 37392858.00
13/4/2021 0.00 51530505.00
14/4/2021 0.00 78472000.00
15/4/2021 0.00 73360000.00
16/4/2021 0.00 26717800.00
19/4/2021 0.00 55003200.00
20/4/2021 0.00 83916000.00
21/4/2021 0.00 81680400.00
22/4/2021 0.00 80883900.00
23/4/2021 0.00 81291700.00
26/4/2021 0.00 83688431.00
27/4/2021 0.00 82395844.00
28/4/2021 0.00 68663400.00
29/4/2021 0.00 92817900.00
30/4/2021 0.00 92881200.00
3/5/2021 0.00 74408300.00
4/5/2021 0.00 74462700.00
5/5/2021 0.00 91116375.00
6/5/2021 0.00 96839025.00
7/5/2021 0.00 88385550.00
10/5/2021 0.00 84053100.00
11/5/2021 0.00 90020439.00
12/5/2021 0.00 80523750.00
13/5/2021 0.00 90299500.00
14/5/2021 0.00 86667850.00
17/5/2021 0.00 91531725.00
18/5/2021 0.00 89597700.00
19/5/2021 0.00 77642200.00
20/5/2021 0.00 99762900.00
21/5/2021 0.00 104601300.00
24/5/2021 0.00 106362700.00
25/5/2021 0.00 127949800.00
26/5/2021 0.00 148178500.00
27/5/2021 0.00 166200500.00
28/5/2021 0.00 168105700.00
1/6/2021 0.00 195923500.00
2/6/2021 0.00 213471400.00
3/6/2021 0.00 192273500.00
4/6/2021 0.00 184868900.00
7/6/2021 0.00 224778700.00
8/6/2021 0.00 244460400.00
9/6/2021 0.00 266699400.00
10/6/2021 0.00 175574279.00
11/6/2021 0.00 198690600.00
14/6/2021 0.00 170788500.00
15/6/2021 0.00 165629700.00
16/6/2021 0.00 147505300.00
17/6/2021 0.00 143452500.00
18/6/2021 0.00 131362100.00
21/6/2021 0.00 121531700.00
22/6/2021 0.00 136067200.00
23/6/2021 0.00 127931500.00
24/6/2021 0.00 117488200.00
25/6/2021 0.00 123786200.00
28/6/2021 0.00 122689600.00
29/6/2021 0.00 120289400.00
30/6/2021 0.00 120690100.00
1/7/2021 0.00 113193600.00
2/7/2021 0.00 108032500.00
6/7/2021 0.00 113647600.00
7/7/2021 0.00 106001200.00
8/7/2021 0.00 101762400.00
9/7/2021 0.00 104859200.00
12/7/2021 0.00 108445200.00
13/7/2021 0.00 103682350.00
14/7/2021 0.00 106498800.00
15/7/2021 0.00 102773175.00
const fetch = require('node-fetch');
const fs = require('fs');
const startDate = new Date(2021, 0);
const endDate = new Date();
const msPerDay = 86400000;
const fetchDelayMS = 1000;
function pad(s, d) {
while (s.length < d) {
s = '0' + s;
}
return s;
}
function scrapeURL (date) {
const day = date.getDate();
const month = date.getMonth() + 1;
const year = date.getFullYear();
return `https://marketdata.theocc.com/stock-loan-bal-by-security?dailyDate=${pad(month + '', 2)}/${pad(day + '', 2)}/${pad(year + '', 2)}&format=csv`
}
function csvName (date) {
const day = date.getDate();
const month = date.getMonth() + 1;
const year = date.getFullYear();
return `./data/${pad(year + '', 2)}_${pad(month + '', 2)}_${pad(day + '', 2)}.csv`;
}
function sleep (ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
(async () => {
for (let d = +startDate; d < +endDate; d += msPerDay) {
const date = new Date(d);
const url = scrapeURL(date);
try {
const res = await fetch(url);
const report = await res.text();
if (report.startsWith('No record')) {
console.log('No data for: ', date);
} else {
await fs.promises.writeFile(csvName(date), report);
console.log(`Fetched data for ${date}`);
}
} catch (err) {
console.error(`Error on data for date ${date}`);
console.error(err);
}
await sleep(fetchDelayMS);
}
})()
@mikolalysenko

This comment has been minimized.

Copy link
Owner Author

@mikolalysenko mikolalysenko commented Jul 16, 2021

Here's how to use this stuff:

  1. Install node.js
  2. Install node-fetch using npm i node-fetch
  3. Run the scraper script:
node scrape-occ.js
  1. Run the analysis script:
node analyze-security.js > dump.csv

You can replace ticker with whatever symbol you want to check. This gives all the loan data from the occ's stock borrow program in a timeline.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment