github topics (https://github.com/topics/python) scraper

github-topics.js

import { assert, log } from 'console'
import fs from 'fs'
import playwright from 'playwright'

const DOWNLOAD_PATH = 'downloads'

const main = async () => {
    console.clear()

    // init download dir
    const downloadDirExists = fs.existsSync(DOWNLOAD_PATH)
    if (!downloadDirExists) {
        fs.mkdirSync(DOWNLOAD_PATH)
    }

    // get user arg
    const args = process.argv.slice(2)
    assert(args.length === 1, 'only one argument is allowed')
    const URL = args[0]
    log(`URL: ${URL}`)

    // open browser
    const browser = await playwright.chromium.launch({
        // headless: false,
        // slowMo: 1000,
    })
    const context = await browser.newContext()
    const page = await context.newPage()
    await page.goto(URL)

    const links = await page.$$eval('article div:nth-child(1) a:nth-child(2)', (anchors) => {
        const hrefs = anchors.map((anchor) => anchor.getAttribute('href'))
        const url = hrefs.map((href) => `https://github.com${href}`)
        return url
    })
    log(links)

    await browser.close()
}
main()

puppeteer.config.cjs

const { join } = require("path");

module.exports = {
  cacheDirectory: join(__dirname, ".cache", "puppeteer"),
};

rate limit:

• anonymous: 60 requests/hour (see: https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api)
• authenticated: 5000 requests/hour (see: https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api)
• web scraper: ???

rate limit bypass:

• bandwidth throttling: https://github.com/tomasbasham/ratelimit
• rotating ip addresses:
  • https://github.com/Ge0rg3/requests-ip-rotator
  • https://0xn3va.gitbook.io/cheat-sheets/web-application/improper-rate-limits#using-proxy-or-vpn
  • https://www.zenrows.com/blog/web-scraping-rate-limit
• running on multiple machines