Created
December 8, 2016 10:24
-
-
Save c4pt0r/64c26e86b807e36d0ce4b2da9426f854 to your computer and use it in GitHub Desktop.
compression.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
inline bool Zlib_Compress(const CompressionOptions& opts, | |
uint32_t compress_format_version, const char* input, | |
size_t length, ::std::string* output, | |
const Slice& compression_dict = Slice()) { | |
#ifdef ZLIB | |
if (length > std::numeric_limits<uint32_t>::max()) { | |
// Can't compress more than 4GB | |
return false; | |
} | |
size_t output_header_len = 0; | |
if (compress_format_version == 2) { | |
output_header_len = compression::PutDecompressedSizeInfo( | |
output, static_cast<uint32_t>(length)); | |
} | |
// Resize output to be the plain data length. | |
// This may not be big enough if the compression actually expands data. | |
output->resize(output_header_len + length); | |
// The memLevel parameter specifies how much memory should be allocated for | |
// the internal compression state. | |
// memLevel=1 uses minimum memory but is slow and reduces compression ratio. | |
// memLevel=9 uses maximum memory for optimal speed. | |
// The default value is 8. See zconf.h for more details. | |
static const int memLevel = 8; | |
z_stream _stream; | |
memset(&_stream, 0, sizeof(z_stream)); | |
int st = deflateInit2(&_stream, opts.level, Z_DEFLATED, opts.window_bits, | |
memLevel, opts.strategy); | |
if (st != Z_OK) { | |
return false; | |
} | |
if (compression_dict.size()) { | |
// Initialize the compression library's dictionary | |
st = deflateSetDictionary( | |
&_stream, reinterpret_cast<const Bytef*>(compression_dict.data()), | |
static_cast<unsigned int>(compression_dict.size())); | |
if (st != Z_OK) { | |
deflateEnd(&_stream); | |
return false; | |
} | |
} | |
// Compress the input, and put compressed data in output. | |
_stream.next_in = (Bytef *)input; | |
_stream.avail_in = static_cast<unsigned int>(length); | |
// Initialize the output size. | |
_stream.avail_out = static_cast<unsigned int>(length); | |
_stream.next_out = reinterpret_cast<Bytef*>(&(*output)[output_header_len]); | |
bool compressed = false; | |
st = deflate(&_stream, Z_FINISH); | |
if (st == Z_STREAM_END) { | |
compressed = true; | |
output->resize(output->size() - _stream.avail_out); | |
} | |
// The only return value we really care about is Z_STREAM_END. | |
// Z_OK means insufficient output space. This means the compression is | |
// bigger than decompressed size. Just fail the compression in that case. | |
deflateEnd(&_stream); | |
return compressed; | |
#endif | |
return false; | |
} | |
// compress_format_version == 1 -- decompressed size is not included in the | |
// block header | |
// compress_format_version == 2 -- decompressed size is included in the block | |
// header in varint32 format | |
// @param compression_dict Data for presetting the compression library's | |
// dictionary. | |
inline char* Zlib_Uncompress(const char* input_data, size_t input_length, | |
int* decompress_size, | |
uint32_t compress_format_version, | |
const Slice& compression_dict = Slice(), | |
int windowBits = -14) { | |
#ifdef ZLIB | |
uint32_t output_len = 0; | |
if (compress_format_version == 2) { | |
if (!compression::GetDecompressedSizeInfo(&input_data, &input_length, | |
&output_len)) { | |
return nullptr; | |
} | |
} else { | |
// Assume the decompressed data size will 5x of compressed size, but round | |
// to the page size | |
size_t proposed_output_len = ((input_length * 5) & (~(4096 - 1))) + 4096; | |
output_len = static_cast<uint32_t>( | |
std::min(proposed_output_len, | |
static_cast<size_t>(std::numeric_limits<uint32_t>::max()))); | |
} | |
z_stream _stream; | |
memset(&_stream, 0, sizeof(z_stream)); | |
// For raw inflate, the windowBits should be -8..-15. | |
// If windowBits is bigger than zero, it will use either zlib | |
// header or gzip header. Adding 32 to it will do automatic detection. | |
int st = inflateInit2(&_stream, | |
windowBits > 0 ? windowBits + 32 : windowBits); | |
if (st != Z_OK) { | |
return nullptr; | |
} | |
if (compression_dict.size()) { | |
// Initialize the compression library's dictionary | |
st = inflateSetDictionary( | |
&_stream, reinterpret_cast<const Bytef*>(compression_dict.data()), | |
static_cast<unsigned int>(compression_dict.size())); | |
if (st != Z_OK) { | |
return nullptr; | |
} | |
} | |
_stream.next_in = (Bytef *)input_data; | |
_stream.avail_in = static_cast<unsigned int>(input_length); | |
char* output = new char[output_len]; | |
_stream.next_out = (Bytef *)output; | |
_stream.avail_out = static_cast<unsigned int>(output_len); | |
bool done = false; | |
while (!done) { | |
st = inflate(&_stream, Z_SYNC_FLUSH); | |
switch (st) { | |
case Z_STREAM_END: | |
done = true; | |
break; | |
case Z_OK: { | |
// No output space. Increase the output space by 20%. | |
// We should never run out of output space if | |
// compress_format_version == 2 | |
assert(compress_format_version != 2); | |
size_t old_sz = output_len; | |
uint32_t output_len_delta = output_len/5; | |
output_len += output_len_delta < 10 ? 10 : output_len_delta; | |
char* tmp = new char[output_len]; | |
memcpy(tmp, output, old_sz); | |
delete[] output; | |
output = tmp; | |
// Set more output. | |
_stream.next_out = (Bytef *)(output + old_sz); | |
_stream.avail_out = static_cast<unsigned int>(output_len - old_sz); | |
break; | |
} | |
case Z_BUF_ERROR: | |
default: | |
delete[] output; | |
inflateEnd(&_stream); | |
return nullptr; | |
} | |
} | |
// If we encoded decompressed block size, we should have no bytes left | |
assert(compress_format_version != 2 || _stream.avail_out == 0); | |
*decompress_size = static_cast<int>(output_len - _stream.avail_out); | |
inflateEnd(&_stream); | |
return output; | |
#endif | |
return nullptr; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment