nats-streams-compression.sh

#!/bin/bash
set -ex

nats stream del NO_COMPRESSION --force > /dev/null || true
nats stream del WITH_COMPRESSION --force > /dev/null || true

nats stream add NO_COMPRESSION \
--subjects NO_COMPRESSION \
--replicas 1 \
--storage=file \
--defaults > /dev/null

nats stream add WITH_COMPRESSION \
--subjects WITH_COMPRESSION \
--replicas 1 \
--storage=file \
--compression s2 \
--defaults > /dev/null

## Please set the JS_STORAGE_DIR to the location of the JetStream storage
EXPORT JS_STORAGE_DIR=/TODO

echo "MESSAGES_WITH_COMPRESSION;MESSAGES_NO_COMPRESSION;CLI_STREAM_BYTES_WITH_COMPRESSION;CLI_STREAM_BYTES_NO_COMPRESSION;DU_WITH_COMPRESSION_KILOBYTES;DU_NO_COMPRESSION_KILOBYTES"
function print_size() {
    sleep 2
    clistreambyteswithcomp=$(nats stream info -j WITH_COMPRESSION | jq '.state.bytes');
    clistreambytesnocomp=$(nats stream info -j NO_COMPRESSION | jq '.state.bytes');
    messageswithcomp=$(nats stream info -j WITH_COMPRESSION | jq '.state.messages');
    messagesnocomp=$(nats stream info -j NO_COMPRESSION | jq '.state.messages');
    withcomp=$(du -c $JS_STORAGE_DIR/\$G/streams/WITH_COMPRESSION | awk 'END{print $1}')
    nocomp=$(du -c $JS_STORAGE_DIR/\$G/streams/NO_COMPRESSION | awk 'END{print $1}')
    echo "$messageswithcomp;$messagesnocomp;$clistreambyteswithcomp;$clistreambytesnocomp;$withcomp;$nocomp"
}

## 5k messages
nats pub WITH_COMPRESSION --count 5000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 5000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size


## 10k messages
nats pub WITH_COMPRESSION --count 5000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 5000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 20k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 30k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 40k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 50k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 60k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 70k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 80k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 90k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 100k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 110k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 120k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 130k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 140k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 150k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 160k messages
nats pub WITH_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 10000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 320k messages
nats pub WITH_COMPRESSION --count 160000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 160000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 640k messages
nats pub WITH_COMPRESSION --count 320000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 320000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size


## 1280k messages
nats pub WITH_COMPRESSION --count 640000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 640000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size

## 2560k messages
nats pub WITH_COMPRESSION --count 1280000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null
nats pub NO_COMPRESSION --count 1280000 "4KJZE2Nx6IbTsEo8lvGJIg1XDz2njdLF" > /dev/null

print_size