Get the turnstile data from NYC MTA website

mta-turnstile.js

const fs = require('fs');
const axios = require('axios');
const cheerio = require('cheerio');

const downloadDir = './downloads';
const siteUrl = 'http://web.mta.info/developers/turnstile.html';

async function fetchData() {
    const result = await axios.get(siteUrl);
    return cheerio.load(result.data);
}

function sleep(ms = 0) {
    return new Promise(r => setTimeout(r, ms));
}

async function main() {
    const $ = await fetchData();
    const urls = $('#contentbox > .container > .last a').map((index, element) => {
        return $(element).attr('href');
    }).get();

    if (!fs.existsSync(downloadDir)) {
        fs.mkdirSync(downloadDir);
    }

    console.log(`found ${urls.length} files to download`);

    for (const url of urls) {
        console.log(`Downloading ${url}`);
        const response = await axios({
            method: 'get',
            url: `http://web.mta.info/developers/${url}`,
            responseType: 'stream',
        });
        const filename = url.split('/').pop();
        const stream = fs.createWriteStream(`${downloadDir}/${filename}`);
        response.data.pipe(stream);
        sleep(1000);
    } 
}

main();