Skip to content

Instantly share code, notes, and snippets.

@viewpointsa
Created January 23, 2020 13:55
Show Gist options
  • Save viewpointsa/b6be251d26af18456ad23841e56ddcb8 to your computer and use it in GitHub Desktop.
Save viewpointsa/b6be251d26af18456ad23841e56ddcb8 to your computer and use it in GitHub Desktop.
import struct, time, zlib
def generate_gzip():
# Migration Python2 to 3 https://stackoverflow.com/a/44387566/2137364
# Yield a gzip file header first.
yield (
b'\037\213\010\000' + # Gzip file, deflate, no filename
struct.pack('<L', int(time.time())) + # compression start time
b'\002\377' # maximum compression, no OS specified
)
# bookkeeping: the compression state, running CRC and total length
compressor = zlib.compressobj(
9, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0)
crc = zlib.crc32(b'')
length = 0
for x in range(10000):
data = "this is my line: {}\n".format(x).encode()
chunk = compressor.compress(data)
if chunk:
yield chunk
crc = zlib.crc32(data, crc) & 0xffffffff
length += len(data)
# Finishing off, send remainder of the compressed data, and CRC and length
yield compressor.flush()
yield struct.pack("<2L", crc, length & 0xffffffff)
with open('out.gz', 'wb') as f:
for data in generate_gzip():
f.write( data )
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment