-
-
Save t-mat/a7e93d4767b991e191ea to your computer and use it in GitHub Desktop.
zstd : Unexpected error for huge data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <assert.h> | |
#include <stdint.h> | |
#include <stdlib.h> | |
#include <stdio.h> | |
#include <string.h> | |
#include "zstd.h" | |
static uint64_t xorshift(uint64_t y) { | |
y ^= y << 1; | |
y ^= y >> 1; | |
y ^= y << 54; | |
return y; | |
} | |
static size_t test(size_t srcSize) { | |
size_t result = 0; | |
size_t cmpSize = 0; | |
const size_t cmpBufSize = ZSTD_compressBound(srcSize); | |
char* cmpPtr = (char*) malloc(cmpBufSize); | |
char* srcPtr = (char*) malloc(srcSize); | |
char* decPtr = NULL; | |
size_t decSize = 0; | |
if(!cmpPtr || !srcPtr) { | |
printf("Out of memory\n"); | |
result = (size_t) -1002; | |
goto end; | |
} | |
{ | |
size_t i = 0; | |
uint64_t r = srcSize; | |
for(i = 0; i < srcSize; i++) { | |
r = xorshift(r); | |
srcPtr[i] = (char) r; | |
} | |
} | |
printf("source data : %lld bytes\n", srcSize); | |
cmpSize = ZSTD_compress(cmpPtr, cmpBufSize, srcPtr, srcSize); | |
printf("zstd compressed : %lld bytes\n", cmpSize); | |
if(ZSTD_isError(cmpSize)) { | |
result = cmpSize; | |
goto end; | |
} | |
free(srcPtr); | |
srcPtr = NULL; | |
decPtr = (char*) malloc(srcSize); | |
decSize = ZSTD_decompress(decPtr, srcSize, cmpPtr, cmpSize); | |
printf("zstd decompressed : %lld bytes\n", decSize); | |
if(ZSTD_isError(decSize)) { | |
result = decSize; | |
goto end; | |
} | |
if(decSize != srcSize) { | |
printf("size mismatch (%lld != %lld)\n", decSize, srcSize); | |
result = (size_t) -1000; | |
goto end; | |
} | |
{ | |
size_t i = 0; | |
uint64_t r = srcSize; | |
for(i = 0; i < srcSize; i++) { | |
r = xorshift(r); | |
if(decPtr[i] != (char) r) { | |
printf("Data error : offset @0x%llx\n", i); | |
result = (size_t) -1001; | |
goto end; | |
} | |
} | |
} | |
printf("OK\n"); | |
end: | |
if(ZSTD_isError(result)) { | |
printf("ZSTD Error : %s\n", ZSTD_getErrorName(result)); | |
} | |
if(srcPtr) free(srcPtr); | |
if(cmpPtr) free(cmpPtr); | |
if(decPtr) free(decPtr); | |
return result; | |
} | |
int main(int argc, char* argv[]) { | |
// const size_t srcSize = (4096ULL << 20); // OK | |
const size_t srcSize = (4096ULL << 20) + 32*1024; // Error | |
assert(sizeof(size_t) == 8); | |
test(srcSize); | |
return 0; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This test requires 8GB of memory...
Test : my system refuses to allocate that much memory. So long...
Nevertheless, I guess I understand what's going on, so I'll fix it in "blind" mode.