testgen-kafka.py

#!/usr/bin/env python

from __future__ import print_function
import json
import sys

def add_test():
    driver = {
        'name': 'Kafka',
        'driverClass': 'io.openmessaging.benchmark.driver.kafka.KafkaBenchmarkDriver',
        'replicationFactor': 3,

        'commonConfig':
          "bootstrap.servers=127.0.0.1",

        'topicConfig':
          "min.insync.replicas=2\n"
          # Uncomment below flags to enable sync mode in Kafka
          #"flush.messages=1\n"
          #"flush.ms=0\n"
          ,

        'producerConfig':
          "acks=all\n"
          "linger.ms=1\n"
          "batch.size=131072\n"
          # This is for transaction
          # "enableTransaction=True\n"
          # "eventPerTransaction=100\n"
          # "enable.idempotence=true"
          ,

        'consumerConfig':
          "auto.offset.reset=earliest\n"
          "enable.auto.commit=false",
      }

    workload = {
        'messageSize': messageSize,
        'topics':  topics,
        'partitionsPerTopic': partitionsPerTopic,
        'subscriptionsPerTopic': subscriptionsPerTopic,
        'consumerPerSubscription': consumerPerSubscription,
        'producersPerTopic': producersPerTopic,
        'producerRate': producerRateEventsPerSec,
        'consumerBacklogSizeGB': consumerBacklogSizeGB,
        'testDurationMinutes': testDurationMinutes,
        'keyDistributor': 'RANDOM_NANO',
    }
    t = dict(
        test='openmessaging-benchmark',
        max_test_attempts=1,
        driver=driver,
        workload=workload,
        numWorkers=numWorkers,
        localWorker=localWorker,
        tarball=tarball,
        build=build,
        undeploy=True,
    )
    test_list.append(t)


test_list = []

localWorker = False
tarball = 'package/target/openmessaging-benchmark-0.0.1-SNAPSHOT-bin.tar.gz'
build = False

# Message size 100 B
for repeat in range(1):
    for producerWorkers in [1]:
        numWorkers = 0 if localWorker else producerWorkers*2
        for testDurationMinutes in [4]:
            for messageSize in [100]:
                for producerRateEventsPerSec in [1e2, 1e3, 5e3, 1e4, 5e4, 1e5, 2e5, 3e5, 5e5, 8e5, -1]:
                    for topics in [1]:
                        for partitionsPerTopic in [1, 16]:
                            for producersPerWorker in [1]:
                                producersPerTopic = int(producersPerWorker * producerWorkers)
                                for consumerBacklogSizeGB in [0]:
                                    for subscriptionsPerTopic in [1]:
                                        for consumerPerSubscription in [partitionsPerTopic]:
                                            for includeTimestampInEvent in [True]:
                                                add_test()

# Message size 10 KB
for repeat in range(1):
    for producerWorkers in [1]:
        numWorkers = 0 if localWorker else producerWorkers*2
        for testDurationMinutes in [4]:
            for messageSize in [10000]:
                for producerRateEventsPerSec in [1e3, 3e3, 6e3, 9e3, 12e3, 15e3, 20e3, 25e3, 30e3, 35e3]:
                    for topics in [1]:
                        for partitionsPerTopic in [1, 16]:
                            for producersPerWorker in [1]:
                                producersPerTopic = int(producersPerWorker * producerWorkers)
                                for consumerBacklogSizeGB in [0]:
                                    for subscriptionsPerTopic in [1]:
                                        for consumerPerSubscription in [partitionsPerTopic]:
                                            for includeTimestampInEvent in [True]:
                                                add_test()

                                                
print(json.dumps(test_list, sort_keys=True, indent=4, ensure_ascii=False))
print('Number of tests generated: %d' % len(test_list), file=sys.stderr)