octokit-test

main.js

import { octokit } from './index.js';
import fs from 'fs';

// if no data.json file in root, make a new one
if (!fs.existsSync('./data.json')) {
    let data = [{ query: 'defi language:javascript stars:>1000', extensions: ['js', 'ts', 'sol', 'yml'], currentPage: 0, endPage: 10, lastRunDate: null, files: [] }]
    fs.writeFileSync('./data.json', JSON.stringify(data));
}

async function searchRepo(query, page) {
    let options = {
        q: query,
        per_page: 5,
        page: page
    }
    return await octokit.request('GET /search/repositories', options)
}

// get all files in repo and filter by extension
async function getFilesInRepo(link, extensions) {
    console.log(extensions)
    let result = await octokit.request('GET /repos/' + link.split('/')[3] + '/' + link.split('/')[4] + '/contents')

    let files = []
    for (let i = 0; i < result.data.length; i++) {
        let fileExt = result.data[i].name.split('.').pop().trim()
        console.log(fileExt)
        if ((extensions.includes(fileExt))) {
            files.push(result.data[i])
        }
    }
    return files
}

// here we search for repos and get 1 page of search results
let searchResult = await searchRepo('web3 language:javascript stars:>1000', 1);
// show the URL for the first search result, the repo we are scraping
let repoURL = searchResult.data.items[0].html_url
console.log(repoURL)

// after, we get all files in repo with these specific extensions
let files = await getFilesInRepo(repoURL, ['js', 'ts', 'sol']);
console.log(files)