NodeJs Script to fetch endpoints in any url in bulk using axios npm JSDOM recon script by Chirag Artani

axios-fetch-endpoints-from-any-urls-bulk.js

const axios = require('axios');
const { JSDOM } = require('jsdom');
const fs = require('fs');
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');

// Function to fetch endpoints from a single URL
async function fetchEndpoints(url) {
    try {
        const response = await axios.get(url);
        const dom = new JSDOM(response.data);
        const scripts = dom.window.document.getElementsByTagName('script');
        const regex = /(?<=(\"|\'|\`))\/[a-zA-Z0-9_?&=\/\-\#\.]*(?=(\"|\'|\`))/g;
        const results = new Set();

        for (let i = 0; i < scripts.length; i++) {
            const scriptSrc = scripts[i].getAttribute('src');
            if (scriptSrc) {
                results.add(scriptSrc);
                try {
                    const scriptResponse = await axios.get(scriptSrc);
                    const scriptText = scriptResponse.data;
                    const matches = scriptText.matchAll(regex);
                    for (const match of matches) {
                        results.add(match[0]);
                    }
                } catch (error) {
                    if (error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND' || error.code === 'ERR_TLS_CERT_ALTNAME_INVALID') {
                        console.log(`Skipping script ${scriptSrc} due to SSL error:`, error.message);
                        continue;
                    } else {
                        console.log(`An error occurred while fetching script ${scriptSrc}:`, error);
                    }
                }
            }
        }

        const pageContent = dom.serialize();
        const matches = pageContent.matchAll(regex);
        for (const match of matches) {
            results.add(match[0]);
        }

        return Array.from(results);
    } catch (error) {
        if (error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND' || error.code === 'ERR_TLS_CERT_ALTNAME_INVALID') {
            console.log(`Skipping URL ${url} due to SSL error:`, error.message);
            return [];
        } else {
            console.error(`Error fetching ${url}:`, error);
            return [];
        }
    }
}


// Function to process URLs from a list of URLs
async function processURLs(urls) {
    const promises = urls.map(url => fetchEndpoints(url));
    const results = await Promise.all(promises);
    return results.flat();
}

// Main function
async function main() {
    const inputFileName = 'YourFile.txt'; // Change Name of the file containing URLs
    const outputFileName = 'endpoints.txt'; // Name of the file to save endpoints

    if (isMainThread) {
        // Read URLs from file
        const urls = fs.readFileSync(inputFileName, 'utf8').split('\n').filter(Boolean);

        // Split URLs into chunks for multi-threading
        const chunkSize = Math.ceil(urls.length / 10); // Adjust the number of threads here
        const urlChunks = [];
        for (let i = 0; i < urls.length; i += chunkSize) {
            urlChunks.push(urls.slice(i, i + chunkSize));
        }

        // Create worker threads
        const workers = urlChunks.map(chunk => new Worker(__filename, { workerData: chunk }));

        // Collect endpoints from each worker
        const endpointResults = await Promise.all(workers.map(worker => {
            return new Promise((resolve, reject) => {
                worker.on('message', resolve);
                worker.on('error', reject);
                worker.on('exit', code => {
                    if (code !== 0) reject(new Error(`Worker stopped with exit code ${code}`));
                });
            });
        }));

        // Flatten endpoint results from workers
        const endpoints = endpointResults.flat();

        // Write endpoints to file
        fs.writeFileSync(outputFileName, endpoints.join('\n'), 'utf8');

        console.log('Endpoints saved to', outputFileName);
    } else {
        // Worker thread
        const urls = workerData;
        const endpoints = await processURLs(urls);
        parentPort.postMessage(endpoints);
    }
}

main();