Apex Compression ZipWriter Performance Test

Apex_Compression_ZipWriter_Performance_Test.java

/**
 * @description Class with methods to test the performance of the ZipWriter
 *              methods.
 * @author      Justus van den Berg (jfwberg@gmail.com)
 * @data        May 2024
 * @note        File sizes are as close as possible to a whole number, 
 *              but due to heap size can't be perfect in all cases. In these
 *              cases they have been pushed to the largest possible
 * @note        CPU times are the ZipWriter.getArchive() / ZipWriter.addEntry()
 *              times
 * @url         https://medium.com/@justusvandenberg/apex-zip-support-performance-test-03bef1539ed6
 * @result      See below test cases + result
 * @manual      Use "AsyncZipWriterPerformanceTest.runTests();" to execute the tests
 *
 * ----------|-----------|----------| ---------| ----------- | -------------
 * Num Files | File size | Num Rows | Num Cols | Size(Bytes) | CPU Time (ms)
 * ----------|-----------|------- --| -------- | ----------- | -------------
 *         2 | 12.000 MB |    14980 |       25 |  11,984,000 | 114 /   34
 *         4 |  6.000 MB |     7500 |       25 |   6.000.000 | 114 /   42
 *         8 |  3.000 MB |     3735 |       25 |   2,988,000 | 111 /   40
 *        16 |  1.500 MB |     1875 |       25 |   1,500,000 | 123 /   60
 *        32 |  0.750 MB |      938 |       25 |     750,400 | 134 /  112
 *        64 |  0.375 MB |      469 |       25 |     375,200 | 138 /  150
 *       128 |  0.188 MB |      235 |       25 |     188,000 | 179 /  228
 *       256 |  0.096 MB |      120 |       25 |      96.000 | 236 /  393
 *       512 |  0.047 MB |       59 |       25 |      47,200 | 385 /  770
 *      1024 |  0.024 MB |       30 |       25 |      24,000 | 628 / 1581
 * ----------|-----------|----------| -------- |------------ |-------------
 */
@SuppressWarnings('PMD.ExcessiveParameterList')
public with sharing class AsyncZipWriterPerformanceTest{

    // Counters for the CPU time it takes to add data to archives
    private static integer fileGenerationTime = 0;
    private static integer addToArchiveTime   = 0;
    

    /**
     * @description Method to run the tests cases, run this to initiate all the test
     */
    public static void runTests(){
        createTestZipFile(Datetime.now(),    2, 14980, 25);
        createTestZipFile(Datetime.now(),    4,  7500, 25);
        createTestZipFile(Datetime.now(),    8,  3735, 25);
        createTestZipFile(Datetime.now(),   16,  1875, 25);
        createTestZipFile(Datetime.now(),   32,   938, 25);
        createTestZipFile(Datetime.now(),   64,   469, 25);
        createTestZipFile(Datetime.now(),  128,   235, 25);
        createTestZipFile(Datetime.now(),  256,   120, 25);
        createTestZipFile(Datetime.now(),  512,    59, 25);
        createTestZipFile(Datetime.now(), 1024,    30, 25);
    }


    /**
     * @description Method that creates a test zip file based on a number of rows and columns
     *              that are passed to a random CSV file Blob generator
     * @param  zw            The Compression.ZipWriter class instance
     * @param  dt            The datetime for generating the timestamp
     * @param  numberOfFiles The number of files   for the test data generation
     * @param  numRows       The number of rows    for the test data generation
     * @param  numColumns    The number of columns for the test data generation
     * @future      Future method in order to test Blob size between 6MB and 12MB
     */
    @future
    public static void createTestZipFile(Datetime dt, Integer numberOfFiles, Integer numberOfRows, Integer numberOfColumns){

        // Change the compression for various test       
        Compression.ZipWriter zw = new Compression.ZipWriter();
        zw.setLevel(Compression.Level.BEST_COMPRESSION);

        Integer st = Limits.getCpuTime();

        // Insert a new file
        insert as user new ContentVersion(
            Title        = dt.format('yyyy-MM-dd HH:mm:ss') + ' - Test Data ('  + numberOfFiles + ') ZIP',
            Description  = 'A ZIP file containing test data to test ZIP Archive limits',
            PathOnClient = 'A_' + dt.format('yyyyMMdd_HHmmss') + '_Test_Data (' + numberOfFiles + ').zip',
            VersionData  = createZipFileBody(zw, dt, numberOfFiles, numberOfRows, numberOfColumns)
        );

        // Calculate the file insertion time, minus the file generation
        // this is not a perfect method and we have a few ms deviation, but it's close enough imho
        System.debug('Number of files / rows / columns : ' + numberOfFiles + ' - ' + numberOfRows + ' - ' + numberOfColumns);
        System.debug('File generation  time: ' + fileGenerationTime);
        System.debug('Add To Archive   time: ' + addToArchiveTime  );
        System.debug('getArchive       time: ' + ((Limits.getCpuTime() - st) - fileGenerationTime - addToArchiveTime));
    }


    /**
     * @description Method to output the size of a blob based on the number of rows and columns
     *              to generate the test data
     * @param  numRows    The number of rows    for the test data generation
     * @param  numColumns The number of columns for the test data generation
     * @future      Future method in order to test Blob size between 6MB and 12MB
     */
    @future
    public static void getBlobSize(Integer numRows, Integer numColumns){
        System.debug('Blob size: ' + generateBlob(new XmlStreamWriter(), numRows, numColumns).size());
    }


    /**
     * @description Method to output the size of a blob based on the number of rows and columns
     *              to generate the test data
     * @param zw            The Compression.ZipWriter class instance
     * @param dt            The datetime for generating the timestamp
     * @param numberOfFiles The number of files   for the test data generation
     * @param numRows       The number of rows    for the test data generation
     * @param numColumns    The number of columns for the test data generation
     */
    private static Blob createZipFileBody(Compression.ZipWriter zw, Datetime dt, Integer numberOfFiles, Integer numberOfRows, Integer numberOfColumns){
        for(Integer i=1;i<=numberOfFiles;i++){

            // Create a file and update the file generation time variable
            Integer fgLimitStart = Limits.getCpuTime();
            Blob data = generateBlob(new XmlStreamWriter(), numberOfRows, numberOfColumns);
            fileGenerationTime = fileGenerationTime + (Limits.getCpuTime() - fgLimitStart);

            // Add the data to archive Update the add to archive measure time variable
            Integer ataLimitStart = Limits.getCpuTime();
            addFileToZipArchive(
                zw,
                'A_' + dt.format('yyyyMMdd_HHmmss') + '_Test_Data_' + i + '_' + numberOfFiles + '.csv',
                data
            );
            addToArchiveTime = addToArchiveTime + (Limits.getCpuTime() - ataLimitStart);
        }
        
        // return the archive
        return zw.getArchive();
    }


    /**
     * @description Method to heap efficiently add a file to a zip writer class
     * @param zw        The Compression.ZipWriter class instance
     * @param fileName  The path of the file
     * @param fileBody  The file body content
     */
    private static void addFileToZipArchive(Compression.ZipWriter zw, String fileName, Blob fileBody){
        zw.addEntry(
            fileName,
            fileBody
        );
    }


    /**
     * @description Method to generate a Blob with CSV Test Data (No CSV header row)
     *              To prevent hitting the heap limits you might need to adjust the numbers
     *               7500 * 25 =  6.000.000 (Max blob size synchronous )
     *              15000 * 25 = 12.000.000 (Max blob size asynchronous)
     * @param numRows    The number of rows    for the test data generation
     * @param numColumns The number of columns for the test data generation
     * @return      A blob value of the user specified size        
     */
    private static Blob generateBlob(XmlStreamWriter xsw, Integer numRows, Integer numColumns){
        for(Integer rowI=0; rowI < numRows; rowI++){
            for(Integer colI=0; colI < numColumns; colI++){
                xsw.writeCharacters('ABCDEFGHIJKLMNOPQRSTUVWXYZ01234'.escapeCsv());
                if(colI < 24){xsw.writeCharacters(',');}
            }
            xsw.writeCharacters('\n');
        }
        return Blob.valueOf(xsw.getXmlString());
    }
}