KafkaJS topic metadata prefetch performance impact measurement

test-prefetch-impact.js

import { Kafka } from 'kafkajs';
import { hrtime } from 'node:process';

// This script measures the performance impact of prefetching topic metadata before sending events.
// More specifically, the script measures how long each `send` call takes while it iterates through thousands of topics and sends an event to each.
// If PREFETCH is set to true, the script prefetches the metadata for each topic the cluster hosts before sending the events.
// The measurement results are output in CSV format and can be easily visualized with e.g. rawgraphs.io

// ---

const CLUSTER_URL = 'localhost:9092';
const TOPIC_COUNT = 5000;
const PREFETCH = true;

// --- 

const kafka = new Kafka({
    clientId: 'mkocsar-kafkajs-test',
    brokers: [ CLUSTER_URL ]
});

const producer = kafka.producer();
await producer.connect();

if (PREFETCH) {
    const admin = kafka.admin();
    const topics = await admin.listTopics();
    await admin.disconnect();

    // This requires changing KafkaJS code so that producer exposes the underlying cluster object
    producer.cluster.addMultipleTargetTopics(topics);
}

console.log('index , durationMs');

for (let index = 0; index < TOPIC_COUNT; ++index) {
    const topic = `mkocsar.kafkajs.test-${index}`;

    const start = hrtime.bigint();
    
    await producer.send({
        topic,
        messages: [{
            value: 'dummy-value'
        }]
    });

    const durationMs = Number(hrtime.bigint() - start) / 1_000_000;
    console.log(`${index} , ${durationMs}`);
}

await producer.disconnect();