Skip to content

Instantly share code, notes, and snippets.

@andrewrk
Created September 29, 2014 04:19
Show Gist options
  • Save andrewrk/cab1bca9b98ec23e1aee to your computer and use it in GitHub Desktop.
Save andrewrk/cab1bca9b98ec23e1aee to your computer and use it in GitHub Desktop.
var http = require('http');
var fs = require("fs");
var Transform = require("stream").Transform;
var PassThrough = require("stream").PassThrough;
var zlib = require("zlib");
var util = require("util");
var EventEmitter = require("events").EventEmitter;
var server = http.createServer(function(req, resp) {
console.log("got request");
req.on('close', cleanupEverything);
resp.setHeader("Content-Type", "application/zip");
resp.setHeader("Content-Disposition", "attachment; filename=songs.zip");
var zipfile = new ZipFile();
zipfile.on('error', function(err) {
log.error("zipfile Error while sending zip of files:", err.stack);
cleanupEverything();
});
var files = ['danse1.ogg', 'danse2.ogg'];
files.forEach(function(file) {
zipfile.addFile(file, file, {compress: false});
});
zipfile.end(function(finalSize) {
resp.setHeader("Content-Length", finalSize.toString());
zipfile.outputStream.pipe(resp);
});
function cleanupEverything() {
resp.end();
}
});
var port = 14797;
server.listen(port, "0.0.0.0", function() {
console.log("curl http://demo.groovebasin.com:" + port + "/");
});
var CRC_TABLE = [
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
0x2d02ef8d
];
if (typeof Int32Array !== 'undefined')
CRC_TABLE = new Int32Array(CRC_TABLE);
function bufferizeInt(num) {
var tmp = Buffer(4);
tmp.writeInt32BE(num, 0);
return tmp;
}
function _crc32(buf, previous) {
if (!Buffer.isBuffer(buf)) {
buf = Buffer(buf);
}
if (Buffer.isBuffer(previous)) {
previous = previous.readUInt32BE(0);
}
var crc = ~~previous ^ -1;
for (var n = 0; n < buf.length; n++) {
crc = CRC_TABLE[(crc ^ buf[n]) & 0xff] ^ (crc >>> 8);
}
return (crc ^ -1);
}
function crc32() {
return bufferizeInt(_crc32.apply(null, arguments));
}
crc32.signed = function () {
return _crc32.apply(null, arguments);
};
crc32.unsigned = function () {
return _crc32.apply(null, arguments) >>> 0;
};
util.inherits(ZipFile, EventEmitter);
function ZipFile() {
this.outputStream = new PassThrough();
this.entries = [];
this.outputStreamCursor = 0;
this.ended = false;
}
ZipFile.prototype.addFile = function(realPath, metadataPath, options) {
var self = this;
validateMetadataPath(metadataPath);
if (options == null) options = {};
var entry = new Entry(metadataPath, options);
self.entries.push(entry);
fs.stat(realPath, function(err, stats) {
if (err) return self.emit("error", err);
if (!stats.isFile()) return self.emit("error", new Error("not a file: " + realPath));
entry.uncompressedSize = stats.size;
if (entry.lastModFileTime == null || entry.lastModFileDate == null) entry.setLastModDate(stats.mtime);
if (entry.externalFileAttributes == null) entry.setFileAttributesMode(stats.mode);
entry.setFileDataPumpFunction(function() {
var readStream = fs.createReadStream(realPath);
entry.state = Entry.FILE_DATA_IN_PROGRESS;
readStream.on("error", function(err) {
self.emit("error", err);
});
pumpFileDataReadStream(self, entry, readStream);
});
pumpEntries(self);
});
};
ZipFile.prototype.addReadStream = function(readStream, metadataPath, options) {
var self = this;
validateMetadataPath(metadataPath);
if (options == null) options = {};
var entry = new Entry(metadataPath, options);
validateFilelessEntryProperties(entry);
self.entries.push(entry);
entry.setFileDataPumpFunction(function() {
entry.state = Entry.FILE_DATA_IN_PROGRESS;
pumpFileDataReadStream(self, entry, readStream);
pumpEntries(self);
});
pumpEntries(self);
};
ZipFile.prototype.addBuffer = function(buffer, metadataPath, options) {
var self = this;
validateMetadataPath(metadataPath);
if (options == null) options = {};
var entry = new Entry(metadataPath, options);
validateFilelessEntryProperties(entry, buffer.length);
entry.uncompressedSize = buffer.length;
entry.crc32 = crc32.unsigned(buffer);
self.entries.push(entry);
if (!entry.compress) {
setCompressedBuffer(buffer);
} else {
zlib.deflateRaw(buffer, function(err, compressedBuffer) {
setCompressedBuffer(compressedBuffer);
});
}
function setCompressedBuffer(compressedBuffer) {
entry.compressedSize = compressedBuffer.length;
entry.setFileDataPumpFunction(function() {
writeToOutputStream(self, compressedBuffer);
writeToOutputStream(self, entry.getFileDescriptor());
entry.state = Entry.FILE_DATA_DONE;
pumpEntries(self);
});
pumpEntries(self);
}
};
ZipFile.prototype.end = function(finalSizeCallback) {
if (this.ended) return;
this.ended = true;
this.finalSizeCallback = finalSizeCallback;
pumpEntries(this);
};
function writeToOutputStream(self, buffer) {
self.outputStream.write(buffer);
self.outputStreamCursor += buffer.length;
}
function pumpFileDataReadStream(self, entry, readStream) {
var crc32Watcher = new Crc32Watcher();
var uncompressedSizeCounter = new ByteCounter();
var compressor = entry.compress ? new zlib.DeflateRaw() : new PassThrough();
var compressedSizeCounter = new ByteCounter();
readStream.pipe(crc32Watcher)
.pipe(uncompressedSizeCounter)
.pipe(compressor)
.pipe(compressedSizeCounter)
.pipe(self.outputStream, {end: false});
compressedSizeCounter.on("finish", function() {
entry.crc32 = crc32Watcher.crc32;
if (entry.uncompressedSize == null) {
entry.uncompressedSize = uncompressedSizeCounter.byteCount;
} else {
if (entry.uncompressedSize !== uncompressedSizeCounter.byteCount) return self.emit("error", new Error("file data stream has unexpected number of bytes"));
}
entry.compressedSize = compressedSizeCounter.byteCount;
self.outputStreamCursor += entry.compressedSize;
writeToOutputStream(self, entry.getFileDescriptor());
entry.state = Entry.FILE_DATA_DONE;
pumpEntries(self);
});
}
function pumpEntries(self) {
// first check if finalSize is finally known
if (self.ended && self.finalSizeCallback != null) {
var finalSize = calculateFinalSize(self);
if (finalSize != null) {
// we have an answer
self.finalSizeCallback(finalSize);
self.finalSizeCallback = null;
}
}
// pump entries
var entry = getFirstNotDoneEntry();
function getFirstNotDoneEntry() {
for (var i = 0; i < self.entries.length; i++) {
var entry = self.entries[i];
if (entry.state < Entry.FILE_DATA_DONE) return entry;
}
return null;
}
if (entry != null) {
// this entry is not done yet
if (entry.state < Entry.READY_TO_PUMP_FILE_DATA) return; // input file not open yet
if (entry.state === Entry.FILE_DATA_IN_PROGRESS) return; // we'll get there
// start with local file header
entry.relativeOffsetOfLocalHeader = self.outputStreamCursor;
var localFileHeader = entry.getLocalFileHeader();
writeToOutputStream(self, localFileHeader);
entry.doFileDataPump();
} else {
// all cought up on writing entries
if (self.ended) {
// head for the exit
self.offsetOfStartOfCentralDirectory = self.outputStreamCursor;
self.entries.forEach(function(entry) {
var centralDirectoryRecord = entry.getCentralDirectoryRecord();
writeToOutputStream(self, centralDirectoryRecord);
});
writeToOutputStream(self, getEndOfCentralDirectoryRecord(self));
self.outputStream.end();
}
}
}
function calculateFinalSize(self) {
var result = 0;
for (var i = 0; i < self.entries.length; i++) {
var entry = self.entries[i];
// compression is too hard to predict
if (entry.compress) return -1;
if (entry.state >= Entry.READY_TO_PUMP_FILE_DATA) {
// if addReadStream was called without providing the size, we can't predict the final size
if (entry.uncompressedSize == null) return -1;
} else {
// if we're still waiting for fs.stat, we might learn the size someday
if (entry.uncompressedSize == null) return null;
}
result += LOCAL_FILE_HEADER_FIXED_SIZE + entry.utf8FileName.length +
entry.uncompressedSize +
FILE_DESCRIPTOR_SIZE +
CENTRAL_DIRECTORY_RECORD_FIXED_SIZE + entry.utf8FileName.length;
}
result += END_OF_CENTRAL_DIRECTORY_RECORD_SIZE;
return result;
}
var END_OF_CENTRAL_DIRECTORY_RECORD_SIZE = 22;
function getEndOfCentralDirectoryRecord(self) {
var buffer = new Buffer(END_OF_CENTRAL_DIRECTORY_RECORD_SIZE);
buffer.writeUInt32LE(0x06054b50, 0); // end of central dir signature 4 bytes (0x06054b50)
buffer.writeUInt16LE(0, 4); // number of this disk 2 bytes
buffer.writeUInt16LE(0, 6); // number of the disk with the start of the central directory 2 bytes
buffer.writeUInt16LE(self.entries.length, 8); // total number of entries in the central directory on this disk 2 bytes
buffer.writeUInt16LE(self.entries.length, 10); // total number of entries in the central directory 2 bytes
buffer.writeUInt32LE(self.outputStreamCursor - self.offsetOfStartOfCentralDirectory, 12); // size of the central directory 4 bytes
buffer.writeUInt32LE(self.offsetOfStartOfCentralDirectory, 16); // offset of start of central directory with respect to the starting disk number 4 bytes
buffer.writeUInt16LE(0, 20); // .ZIP file comment length 2 bytes
/* no comment */ // .ZIP file comment (variable size)
return buffer;
}
function validateMetadataPath(metadataPath) {
if (metadataPath.indexOf("\\") !== -1) throw new Error("invalid characters in path: " + metadataPath);
if (/^[a-zA-Z]:/.test(metadataPath) || /^\//.test(metadataPath)) throw new Error("absolute path: " + metadataPath);
if (metadataPath.split("/").indexOf("..") !== -1) throw new Error("invalid relative path: " + metadataPath);
}
function validateFilelessEntryProperties(entry, length) {
if (entry.lastModFileTime == null || entry.lastModFileDate == null) throw new Error("missing options.mtime");
if (entry.externalFileAttributes == null) throw new Error("missing options.mode");
if (entry.uncompressedSize != null && length != null && entry.uncompressedSize !== length) throw new Error("invalid options.size");
}
// this class is not part of the public API
function Entry(metadataPath, options) {
this.utf8FileName = new Buffer(metadataPath);
if (this.utf8FileName.length > 0xffff) throw new Error("utf8 file name too long. " + utf8FileName.length + " > " + 0xffff);
this.state = Entry.WAITING_FOR_METADATA;
if (options.mtime != null) this.setLastModDate(options.mtime);
if (options.mode != null) this.setFileAttributesMode(options.mode);
this.uncompressedSize = null; // unknown
if (options.size != null) this.uncompressedSize = options.size;
this.compress = true; // default
if (options.compress != null) this.compress = !!options.compress;
}
Entry.WAITING_FOR_METADATA = 0;
Entry.READY_TO_PUMP_FILE_DATA = 1;
Entry.FILE_DATA_IN_PROGRESS = 2;
Entry.FILE_DATA_DONE = 3;
Entry.prototype.setLastModDate = function(date) {
var dosDateTime = dateToDosDateTime(date);
this.lastModFileTime = dosDateTime.time;
this.lastModFileDate = dosDateTime.date;
};
Entry.prototype.setFileAttributesMode = function(mode) {
if ((mode & 0xffff) !== mode) throw new Error("invalid mode. expected: 0 <= " + mode + " <= " + 0xffff);
// http://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute/14727#14727
this.externalFileAttributes = (mode << 16) >>> 0;
};
Entry.prototype.setFileDataPumpFunction = function(doFileDataPump) {
this.doFileDataPump = doFileDataPump;
this.state = Entry.READY_TO_PUMP_FILE_DATA;
};
var LOCAL_FILE_HEADER_FIXED_SIZE = 30;
// this version enables utf8 filename encoding
var VERSION_NEEDED_TO_EXTRACT = 0x0014;
// this is the "version made by" reported by linux info-zip.
var VERSION_MADE_BY_INFO_ZIP = 0x031e;
var FILE_NAME_IS_UTF8 = 1 << 11;
var UNKNOWN_CRC32_AND_FILE_SIZES = 1 << 3;
Entry.prototype.getLocalFileHeader = function() {
var fixedSizeStuff = new Buffer(LOCAL_FILE_HEADER_FIXED_SIZE);
var generalPurposeBitFlag = UNKNOWN_CRC32_AND_FILE_SIZES | FILE_NAME_IS_UTF8;
fixedSizeStuff.writeUInt32LE(0x04034b50, 0); // local file header signature 4 bytes (0x04034b50)
fixedSizeStuff.writeUInt16LE(VERSION_NEEDED_TO_EXTRACT, 4); // version needed to extract 2 bytes
fixedSizeStuff.writeUInt16LE(generalPurposeBitFlag, 6); // general purpose bit flag 2 bytes
fixedSizeStuff.writeUInt16LE(this.getCompressionMethod(), 8); // compression method 2 bytes
fixedSizeStuff.writeUInt16LE(this.lastModFileTime, 10); // last mod file time 2 bytes
fixedSizeStuff.writeUInt16LE(this.lastModFileDate, 12); // last mod file date 2 bytes
fixedSizeStuff.writeUInt32LE(0, 14); // crc-32 4 bytes
fixedSizeStuff.writeUInt32LE(0, 18); // compressed size 4 bytes
fixedSizeStuff.writeUInt32LE(0, 22); // uncompressed size 4 bytes
fixedSizeStuff.writeUInt16LE(this.utf8FileName.length, 26); // file name length 2 bytes
fixedSizeStuff.writeUInt16LE(0, 28); // extra field length 2 bytes
return Buffer.concat([
fixedSizeStuff,
this.utf8FileName, // file name (variable size)
/* no extra fields */ // extra field (variable size)
]);
};
var FILE_DESCRIPTOR_SIZE = 12
Entry.prototype.getFileDescriptor = function() {
var buffer = new Buffer(FILE_DESCRIPTOR_SIZE);
buffer.writeUInt32LE(this.crc32, 0); // crc-32 4 bytes
buffer.writeUInt32LE(this.compressedSize, 4); // compressed size 4 bytes
buffer.writeUInt32LE(this.uncompressedSize, 8); // uncompressed size 4 bytes
return buffer;
}
var CENTRAL_DIRECTORY_RECORD_FIXED_SIZE = 46;
Entry.prototype.getCentralDirectoryRecord = function() {
var fixedSizeStuff = new Buffer(CENTRAL_DIRECTORY_RECORD_FIXED_SIZE);
fixedSizeStuff.writeUInt32LE(0x02014b50, 0); // central file header signature 4 bytes (0x02014b50)
fixedSizeStuff.writeUInt16LE(VERSION_MADE_BY_INFO_ZIP, 4); // version made by 2 bytes
fixedSizeStuff.writeUInt16LE(VERSION_NEEDED_TO_EXTRACT, 6); // version needed to extract 2 bytes
fixedSizeStuff.writeUInt16LE(FILE_NAME_IS_UTF8, 8); // general purpose bit flag 2 bytes
fixedSizeStuff.writeUInt16LE(this.getCompressionMethod(), 10); // compression method 2 bytes
fixedSizeStuff.writeUInt16LE(this.lastModFileTime, 12); // last mod file time 2 bytes
fixedSizeStuff.writeUInt16LE(this.lastModFileDate, 14); // last mod file date 2 bytes
fixedSizeStuff.writeUInt32LE(this.crc32, 16); // crc-32 4 bytes
fixedSizeStuff.writeUInt32LE(this.compressedSize, 20); // compressed size 4 bytes
fixedSizeStuff.writeUInt32LE(this.uncompressedSize, 24); // uncompressed size 4 bytes
fixedSizeStuff.writeUInt16LE(this.utf8FileName.length, 28); // file name length 2 bytes
fixedSizeStuff.writeUInt16LE(0, 30); // extra field length 2 bytes
fixedSizeStuff.writeUInt16LE(0, 32); // file comment length 2 bytes
fixedSizeStuff.writeUInt16LE(0, 34); // disk number start 2 bytes
fixedSizeStuff.writeUInt16LE(0, 36); // internal file attributes 2 bytes
fixedSizeStuff.writeUInt32LE(this.externalFileAttributes, 38); // external file attributes 4 bytes
fixedSizeStuff.writeUInt32LE(this.relativeOffsetOfLocalHeader, 42); // relative offset of local header 4 bytes
return Buffer.concat([
fixedSizeStuff,
this.utf8FileName, // file name (variable size)
/* no extra fields */ // extra field (variable size)
/* empty comment */ // file comment (variable size)
]);
};
Entry.prototype.getCompressionMethod = function() {
var NO_COMPRESSION = 0;
var DEFLATE_COMPRESSION = 8;
return this.compress ? DEFLATE_COMPRESSION : NO_COMPRESSION;
};
function dateToDosDateTime(jsDate) {
var date = 0;
date |= jsDate.getDate() & 0x1f; // 1-31
date |= ((jsDate.getMonth() + 1) & 0xf) << 5; // 0-11, 1-12
date |= ((jsDate.getFullYear() - 1980) & 0x7f) << 9; // 0-128, 1980-2108
var time = 0;
time |= Math.floor(jsDate.getSeconds() / 2); // 0-59, 0-29 (lose odd numbers)
time |= (jsDate.getMinutes() & 0x3f) << 5; // 0-59
time |= (jsDate.getHours() & 0x1f) << 11; // 0-23
return {date: date, time: time};
}
function defaultCallback(err) {
if (err) throw err;
}
util.inherits(ByteCounter, Transform);
function ByteCounter(options) {
Transform.call(this, options);
this.byteCount = 0;
}
ByteCounter.prototype._transform = function(chunk, encoding, cb) {
this.byteCount += chunk.length;
cb(null, chunk);
};
util.inherits(Crc32Watcher, Transform);
function Crc32Watcher(options) {
Transform.call(this, options);
this.crc32 = 0;
}
Crc32Watcher.prototype._transform = function(chunk, encoding, cb) {
this.crc32 = crc32.unsigned(chunk, this.crc32);
cb(null, chunk);
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment