test mvt using node

test.js

// api in question is a mapbox vector tiles

// Path: test.js

const SphericalMercator = require("@mapbox/sphericalmercator");
const merc = new SphericalMercator({ size: 256 });
const axios = require("axios");
const fs = require("fs").promises;
const { Console } = require("console");
const http = require('http');
const async = require('async');

const {
    Worker, isMainThread, parentPort, workerData,
} = require('node:worker_threads');
const tileTree = (boundingBox, zoomLevels) => {
    const tiles = [];
    for (let zoom of zoomLevels) {
        const tileRange = merc.xyz(boundingBox, zoom);
        for (let x = tileRange.minX; x <= tileRange.maxX; x++) {
            for (let y = tileRange.minY; y <= tileRange.maxY; y++) {
                tiles.push({ zoom, x, y }); // Push each task to the queue
            }
        }
    }
    return tiles;
}

const testUrl = async (mbtilesLocation, { zoom, x, y }) => {
    const url = tileUrl(mbtilesLocation, { zoom, x, y });

    const startTime = process.hrtime.bigint();
    try {
        const response = await axios.get(`${'http://127.0.0.1:42785/'}${url}`);
        const endTime = process.hrtime.bigint();
        return {
            zoom, x, y,
            size: response.data.length,
            status: response.status,
            responseTime: Number(endTime - startTime) / 1000000
        };
    } catch (error) {
        const endTime = Date.now();
        console.error(`Error for URL ${url}:`, error.message);
        return { error: error.message, zoom, x, y, responseTime: Number(endTime - startTime) / 1000000 };
    }
};
const requestQueue = async.queue(async (task) => {
    return await testUrl(task.mbtilesLocation, task.tile);
}, 32);

// based on the tile tree i want to create urls and check the response size and status code
const mbtilesLocation = process.argv[2];

const tileUrl = (mbtilesLocation, { zoom, x, y }) => `${mbtilesLocation}/${zoom}/${x}/${y}`;


const testTilesInWorker = (tiles) => {
    return new Promise((resolve, reject) => {
        const worker = new Worker(__filename, { workerData: { tiles, mbtilesLocation } });
        worker.on('message', resolve);
        worker.on('error', reject);
        worker.on('exit', (code) => {
            if (code !== 0) reject(new Error(`Worker stopped with exit code ${code}`));
        });
    });
};

const calculateStats = (results) => {
    let statsByZoomLevel = {};

    results.forEach(result => {
        if (!statsByZoomLevel[result.zoom]) {
            statsByZoomLevel[result.zoom] = {
                successCount: 0,
                errorCount: 0,
                responseTimes: [],
                minResponseTime: Infinity,
                maxResponseTime: -Infinity
            };
        }

        let zoomStats = statsByZoomLevel[result.zoom];

        if (result.status === 200 || result.status === 204) {
            zoomStats.successCount++;
            zoomStats.responseTimes.push(result.responseTime);
            zoomStats.minResponseTime = Math.min(zoomStats.minResponseTime, result.responseTime);
            zoomStats.maxResponseTime = Math.max(zoomStats.maxResponseTime, result.responseTime);
        } else {
            console.error(`Error for tile ${result.zoom}/${result.x}/${result.y}: ${result.status}`);
            zoomStats.errorCount++;
        }
    });

    // Calculate median and percentiles for each zoom level
    Object.keys(statsByZoomLevel).forEach(zoom => {
        let zoomStats = statsByZoomLevel[zoom];
        zoomStats.responseTimes.sort((a, b) => a - b);
        zoomStats.medianResponseTime = calculateMedian(zoomStats.responseTimes);
        zoomStats.percentiles = calculatePercentiles(zoomStats.responseTimes);
        zoomStats.count = { total: zoomStats.successCount + zoomStats.errorCount, success: zoomStats.successCount, error: zoomStats.errorCount }
        delete zoomStats.responseTimes;
    });

    return statsByZoomLevel;
};

const calculateMedian = (arr) => {
    const mid = Math.floor(arr.length / 2);
    return arr.length % 2 !== 0 ? arr[mid] : (arr[mid - 1] + arr[mid]) / 2;
};

const calculatePercentiles = (arr) => {
    return {
        p25: percentile(arr, 25),
        p50: percentile(arr, 50), // same as median
        p75: percentile(arr, 75),
        p90: percentile(arr, 90),
        p95: percentile(arr, 95),
        p99: percentile(arr, 99)
    };
};

const percentile = (arr, p) => {
    if (arr.length === 0) return 0;
    if (p === 100) return arr[arr.length - 1];

    let index = (arr.length - 1) * p / 100;
    let lower = Math.floor(index);
    let upper = lower + 1;
    let weight = index % 1;

    if (upper >= arr.length) return arr[lower];
    return arr[lower] * (1 - weight) + arr[upper] * weight;
};

const testTiles = async (mbtilesLocation, tiles) => {
    return new Promise((resolve, reject) => {
        const results = [];
        requestQueue.drain(() => resolve(results));

        for (let tile of tiles) {
            requestQueue.push({ mbtilesLocation, tile }, (err, result) => {
                if (err) reject(err);
                else results.push(result);
            });
        }
    });
}

const testBoundingBox = async (boundingBox, zoomLevels) => {
    const numWorkers = 4; // for example, adjust as needed
    const promises = [];


    console.time("getTileTree")
    const tiles = tileTree(boundingBox, zoomLevels);
    console.timeEnd("getTileTree")
    for (let i = 0; i < numWorkers; i++) {
        // Divide tiles among workers
        const tilesForWorker = tiles.slice(i * tiles.length / numWorkers, (i + 1) * tiles.length / numWorkers);
        promises.push(testTilesInWorker(tilesForWorker));
    }

    console.time("getTiles")

    const results = (await Promise.all(promises)).flat();

    console.timeEnd("getTiles")
    console.time("calcStats")

    const stats = calculateStats(results);
    console.timeEnd("calcStats")

    return { stats, results };
}

const boundingBox = [68.1766451354, 7.96553477623, 97.4025614766, 35.4940095078];
const zoomLevels = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14];



if (isMainThread) {

    testBoundingBox(boundingBox, zoomLevels)
        .then(async ({ results, stats }) => {
            console.log("Aggregate Stats:", stats);
            await fs.writeFile(`./${mbtilesLocation}.results.json`, JSON.stringify({ stats }, null, 2));
        })
        .catch((error) => {
            console.error(error);
        });
} else {
    const tiles = workerData.tiles;
    const mbtilesLocation = workerData.mbtilesLocation;
    testTiles(mbtilesLocation, tiles).then(results => parentPort.postMessage(results));
}