zstd : Unexpected error for huge data

zstd-huge-data-error.c

#include <assert.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "zstd.h"

static uint64_t xorshift(uint64_t y) {
    y ^= y << 1;
    y ^= y >> 1;
    y ^= y << 54;
    return y;
}

static size_t test(size_t srcSize) {
    size_t result = 0;

    size_t          cmpSize     = 0;
    const size_t    cmpBufSize  = ZSTD_compressBound(srcSize);
    char*           cmpPtr      = (char*) malloc(cmpBufSize);
    char*           srcPtr      = (char*) malloc(srcSize);
    char*           decPtr      = NULL;
    size_t          decSize     = 0;

    if(!cmpPtr || !srcPtr) {
        printf("Out of memory\n");
        result = (size_t) -1002;
        goto end;
    }

    {
        size_t i = 0;
        uint64_t r = srcSize;
        for(i = 0; i < srcSize; i++) {
            r = xorshift(r);
            srcPtr[i] = (char) r;
        }
    }
    printf("source data       : %lld bytes\n", srcSize);

    cmpSize = ZSTD_compress(cmpPtr, cmpBufSize, srcPtr, srcSize);
    printf("zstd compressed   : %lld bytes\n", cmpSize);

    if(ZSTD_isError(cmpSize)) {
        result = cmpSize;
        goto end;
    }

    free(srcPtr);
    srcPtr = NULL;

    decPtr = (char*) malloc(srcSize);
    decSize = ZSTD_decompress(decPtr, srcSize, cmpPtr, cmpSize);
    printf("zstd decompressed : %lld bytes\n", decSize);

    if(ZSTD_isError(decSize)) {
        result = decSize;
        goto end;
    }

    if(decSize != srcSize) {
        printf("size mismatch (%lld != %lld)\n", decSize, srcSize);
        result = (size_t) -1000;
        goto end;
    }

    {
        size_t i = 0;
        uint64_t r = srcSize;
        for(i = 0; i < srcSize; i++) {
            r = xorshift(r);
            if(decPtr[i] != (char) r) {
                printf("Data error : offset @0x%llx\n", i);
                result = (size_t) -1001;
                goto end;
            }
        }
    }

    printf("OK\n");

end:
    if(ZSTD_isError(result)) {
        printf("ZSTD Error : %s\n", ZSTD_getErrorName(result));
    }

    if(srcPtr) free(srcPtr);
    if(cmpPtr) free(cmpPtr);
    if(decPtr) free(decPtr);

    return result;
}


int main(int argc, char* argv[]) {
//  const size_t srcSize = (4096ULL << 20);             // OK
    const size_t srcSize = (4096ULL << 20) + 32*1024;   // Error
    assert(sizeof(size_t) == 8);
    test(srcSize);
    return 0;
}

This test requires 8GB of memory...

Test : my system refuses to allocate that much memory. So long...

Nevertheless, I guess I understand what's going on, so I'll fix it in "blind" mode.