Cassandra Compression Block Size Testing

README.md

Compile with:

javac -cp '/path/to/cassandra/lib/*' CompressionTest.java

Run with:

java -cp '.:/path/to/cassandra/lib/*' CompressionTest <filename>

Note that it doens't make sense to go below 4kb as most Linux filesystems default to 4kb blocks.

CompressionTest.java

import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;

public class CompressionTest
{
  public static void main(String args[]) throws Exception
    {
        System.out.println("Observing compression ratios of LZ4 on: " + args[0]);
        String filename = args[0];
        Path fileLocation = Paths.get(filename);

        byte[] buf = Files.readAllBytes(fileLocation);
        ByteBuffer corpus = ByteBuffer.allocateDirect(buf.length);
        corpus.put(buf);
        corpus.flip();

        LZ4Factory factory = LZ4Factory.fastestInstance();
        LZ4Compressor compressor = factory.fastCompressor();
        ByteBuffer output = ByteBuffer.allocateDirect(compressor.maxCompressedLength(1024 * 1024 * 64));

        int sizes[] = new int[] { 1024 * 4, 1024 * 8, 1024 * 16, 1024 * 32, 1024 * 64 };
        for (int size : sizes)
        {
            long compressedSize = 0;
            corpus.clear();
            while (corpus.hasRemaining())
            {
                int read = Math.min(size, corpus.remaining()); 
                compressedSize += compressor.compress(corpus, 0, read, output, 0, output.capacity());
                corpus.position(corpus.position() + read);
            }
            System.out.printf("Chunk size %d, ratio %f%n", size, compressedSize / (double)(buf.length));
        }
    }
}