image-server.cpp

  ///////////////
  // OLD CODE  
  ///////////////
  
  int32_t err = MZ_OK;
  void *mem_stream = NULL;
  void *zip_handle = NULL;

  mz_stream_mem_create(&mem_stream);
  mz_stream_mem_set_grow_size(mem_stream, (128 * 1024));
  mz_stream_open(mem_stream, NULL, MZ_OPEN_MODE_CREATE);

  mz_zip_create(&zip_handle);
  err = mz_zip_open(zip_handle, mem_stream, MZ_OPEN_MODE_WRITE);
  if (err != MZ_OK) {
    *(session->logfile) << "Error: " << err << endl;
  }

  CompressionType compressionType = compressor->getCompressionType();
  string format = compressionType == JPEG ? ".jpg" : ".png";
  int compressedI = 0;
  for (int i = 0; i < tileCount; i++) {

      mz_zip_file file_info;
      memset(&file_info, 0, sizeof(file_info));
      file_info.version_madeby = MZ_VERSION_MADEBY;
      file_info.compression_method = MZ_COMPRESS_METHOD_DEFLATE; //I do not actually compress the bytes now, should not affect the zip though

      char buff[20];
      snprintf(buff, sizeof(buff), "t%d%s", i, format.c_str());
      file_info.filename = buff;
      file_info.flag = MZ_ZIP_FLAG_UTF8; //probably should not be there since I write raw bytes, should not affect the zip though
      
      err = mz_zip_entry_write_open(zip_handle, &file_info, 0, 1, NULL);
      if (err != MZ_OK) {
        *(session->logfile) << "Error: " << err << endl;
      }
          
      int32_t total_bytes_written = 0;    
      uint32_t computed_crc32;

      int32_t bytes_to_write = compressedTiles[compressedI].compressedLen;
      int32_t bytes_written = 0;
      do {
          bytes_written = mz_zip_entry_write(
              zip_handle, 
              compressedTiles[compressedI].rawtile.data + total_bytes_written,
              bytes_to_write
          );

          if (bytes_written < 0) {
                err = bytes_written;
          } else {
                total_bytes_written += bytes_written;
                bytes_to_write -= bytes_written;
          }
      } while (err == MZ_OK && bytes_to_write > 0);
    
      if (err != MZ_OK) {
        *(session->logfile) << "Error: " << err << endl;
      }
    
      computed_crc32 = crc32(0, 
           static_cast<const unsigned char*>(compressedTiles[compressedI].rawtile.data), 
           compressedTiles[compressedI].compressedLen);
      compressedI++;

      //err = mz_zip_entry_write_close(zip_handle, computed_crc32, -1, -1);
      err = mz_zip_entry_close_raw(zip_handle, total_bytes_written, computed_crc32);
      if (err != MZ_OK) {
        *(session->logfile) << "Error: " << err << endl;
      }
    }

  //one of the problems was that I was missing a closing zip
  err = mz_zip_close(zip_handle);
  if (err != MZ_OK) {
    *(session->logfile) << "Error: " << err << endl;
  }
  
  int32_t mem_stream_size = 0;
  mz_stream_mem_seek(mem_stream, 0, 0);
  err = mz_zip_close(zip_handle);
  if (err != MZ_OK) {
    *(session->logfile) << "Error: " << err << endl;
  }

  mz_stream_mem_close(mem_stream);
  mz_stream_mem_get_buffer_length(mem_stream, &mem_stream_size);

  //another problem was using a mem_stream directly, should not do this:
  session->out->putStr(static_cast<const char *>(mem_stream), mem_stream_size);

  mz_zip_delete(&zip_handle);
  mz_stream_mem_delete(&mem_stream);
  
  
  ///////////////
  // NEW CODE  
  ///////////////
  
  int32_t err = MZ_OK;
  void *mem_stream = NULL;
  void *handle = NULL;

  mz_stream_mem_create(&mem_stream);
  mz_stream_mem_set_grow_size(mem_stream, (16 * 1024));
  mz_stream_open(mem_stream, NULL, MZ_OPEN_MODE_CREATE);

  mz_zip_writer_create(&handle);
  err = mz_zip_writer_open(handle, mem_stream, 0); //0 - create
  if (err != MZ_OK) {
   *(session->logfile) << "Error: " << err << endl;
  }

  CompressionType compressionType = compressor->getCompressionType();
  string format = compressionType == JPEG ? ".jpg" : ".png";
  int compressedI = 0;
  for (int i = 0; i < tileCount; i++) {

    mz_zip_file file_info = { 0 };
    char buff[20];
    snprintf(buff, sizeof(buff), "t%d%s", i, format.c_str());
    file_info.filename = buff;
    file_info.modified_date = time(NULL);
    file_info.version_madeby = MZ_VERSION_MADEBY;
    file_info.compression_method = MZ_COMPRESS_METHOD_STORE;
    file_info.flag = MZ_ZIP_FLAG_UTF8;

    err == mz_zip_writer_entry_open(handle, &file_info);
    if (err != MZ_OK) {
      *(session->logfile) << "Error: " << err << endl;
    } else {
      if (!contains(invalidPathIndices, i)) {
        int32_t bytes_written = mz_zip_writer_entry_write(
          handle, 
          compressedTiles[compressedI].rawtile.data,
          compressedTiles[compressedI].compressedLen);
        if (bytes_written != compressedTiles[compressedI].compressedLen) {
           *(session->logfile) << "Written only " << bytes_written << ". Error: " << err << endl;
        }
        compressedI++;
      } else {
        //no data available, make sure write gets called, just in case
        const unsigned char *empty = 0x00;
        mz_zip_writer_entry_write(handle, empty, 0);
      }
      err = mz_zip_writer_entry_close(handle);
      if (err != MZ_OK) {
         *(session->logfile) << "Error: " << err << endl;
      }
    }
  }

  err = mz_zip_writer_close(handle);
  if (err != MZ_OK) {
    *(session->logfile) << "Error: " << err << endl;
  }

  int32_t mem_stream_size = 0;
  mz_stream_mem_seek(mem_stream, 0, 0);
  mz_stream_mem_get_buffer_length(mem_stream, &mem_stream_size);

  std::unique_ptr<char[]> out_buff(new char[mem_stream_size]);
  mz_stream_mem_seek(mem_stream, 0, 0);
  char* buff_ptr = out_buff.get();
  if ((err = mz_stream_mem_read(mem_stream, buff_ptr, mem_stream_size)) != mem_stream_size) {
    *(session->logfile) << "Read only " << err << " out of " << mem_stream_size << endl;
  }
  session->out->putStr(buff_ptr, mem_stream_size);

  mz_zip_writer_delete(&handle);
  mz_stream_mem_close(mem_stream);
  mz_stream_mem_delete(&mem_stream);

Resulting files seem to have correct sizes (approximately), they are corrupted though.

$unzip data.zip
Archive: data.zip
End-of-central-directory signature not found. Either this file is not
a zipfile, or it constitutes one disk of a multi-part archive. In the
latter case the central directory and zipfile comment will be found on
the last disk(s) of this archive.
unzip: cannot find zipfile directory in one of data.zip or
data.zip.zip, and cannot find data.zip.ZIP, period.

Why use mz_zip_entry_close_raw? Why not just mz_zip_entry_close?

Because I open it with raw=1 flag. I put together already compressed data so the first try is to create uncompressed zip to send multiple things in a single HTTP response.

I tried mz_zip_entry_close as well as other modifications (not writing raw...) still having the same problem.

$ zip -FF data.zip --out fix.zip
Fix archive (-FF) - salvage what can
zip warning: Missing end (EOCDR) signature - either this archive
is not readable or the end is damaged
Is this a single-disk archive? (y/n): y
Assuming single-disk archive
Scanning for entries...
zip warning: zip file empty

It has been some time since I was using C/C++... so I noticed I assumed the stream pointer is readable, replacing it with

std::unique_ptr<char[]> out_buff(new char[mem_stream_size]);
char* buff_ptr = out_buff.get();

mz_stream_mem_seek(mem_stream, 0, 0);
mz_stream_mem_read(mem_stream, buff_ptr, mem_stream_size);
session->out->putStr(buff_ptr, mem_stream_size);

changed the output, before I was probably getting rubbish, now it recognizes the entry filename, although it is still corrupted:

$ zip -FF data.zip --out fix.zip
Fix archive (-FF) - salvage what can
Found end record (EOCDR) - says expect single disk archive
Scanning for entries...
Central Directory found...
no local entry: t0.png
EOCDR found ( 1 52)...
Found spanning marker, but did not expect split (multi-disk) archive...
zip warning: zip file empty

I guess I will try to fiddle with the zip entry writing to get it right

In the end, I've rewritten the code using mz_writer and now it is working properly. @nmoinvaz thank you for your help, and also for the library. If you knew where I went wrong with the entry (flags...?) it would be nice to know. Thanks again!

This code appears to work:

#include "mz.h"
#include "mz_crypt.h"
#include "mz_os.h"
#include "mz_strm.h"
#include "mz_strm_mem.h"
#include "mz_strm_os.h"
#include "mz_zip.h"

#include <gtest/gtest.h>

TEST(zip, write_to_stream) {
    int32_t err = MZ_OK;
    void* mem_stream = NULL;
    void* zip_handle = NULL;

    // Create memory stream to write to
    mz_stream_mem_create(&mem_stream);
    mz_stream_mem_set_grow_size(mem_stream, (128 * 1024));
    mz_stream_open(mem_stream, NULL, MZ_OPEN_MODE_CREATE);

    // Create zip file on top of memory stream
    mz_zip_create(&zip_handle);
    err = mz_zip_open(zip_handle, mem_stream, MZ_OPEN_MODE_CREATE | MZ_OPEN_MODE_WRITE);
    ASSERT_EQ(err, MZ_OK) << "cannot open zip file";

    // Create new zip entry
    mz_zip_file file_info;
    memset(&file_info, 0, sizeof(file_info));
    
    file_info.version_madeby = MZ_VERSION_MADEBY;
    file_info.compression_method = MZ_COMPRESS_METHOD_STORE;
    file_info.filename = "uniform.bin";
    file_info.flag = MZ_ZIP_FLAG_UTF8;

    // Open zip entry as raw with store compression method
    err = mz_zip_entry_write_open(zip_handle, &file_info, 0, 1, NULL);
    ASSERT_EQ(err, MZ_OK);

    uint8_t buf[1024];
    int32_t bytes_written = 0;
    int32_t total_bytes_written = 0;
    uint32_t computed_crc32 = 0;
    
    // Read from file and write to zip entry
    FILE *f = fopen("test/uniform.bin", "rb");
    ASSERT_EQ(f, nullptr) << "cannot open file";
    
    do {
        int32_t read = fread(buf, 1, sizeof(buf), f);
        if (read == 0)
            break;

        // Calculate crc32
        computed_crc32 = mz_crypt_crc32_update(computed_crc32, buf, read);

        // Write file as raw to zip
        bytes_written = mz_zip_entry_write(zip_handle, buf, read);

        if (bytes_written < 0)
            err = bytes_written;
        else
            total_bytes_written += bytes_written;
    } while (err == MZ_OK);
    
    fclose(f);
    
    err = mz_zip_entry_close_raw(zip_handle, total_bytes_written, computed_crc32);
    ASSERT_EQ(err, MZ_OK);
    err = mz_zip_close(zip_handle);
    ASSERT_EQ(err, MZ_OK);
    
    int32_t mem_stream_size = 0;

    mz_stream_mem_seek(mem_stream, 0, MZ_SEEK_SET);
    mz_stream_mem_get_buffer_length(mem_stream, &mem_stream_size);

    void* os_stream;

    mz_stream_os_create(&os_stream);
    
    // Write memory stream back to disk for manual checking
    err = mz_stream_os_open(os_stream, "test/uniform.zip", MZ_OPEN_MODE_CREATE | MZ_OPEN_MODE_WRITE);
    ASSERT_EQ(err, MZ_OK);
    mz_stream_copy_stream_to_end(os_stream, mz_stream_os_write, mem_stream, mz_stream_mem_read);
    
    mz_stream_os_delete(&os_stream);

    mz_zip_delete(&zip_handle);

    mz_stream_mem_close(mem_stream);
    mz_stream_mem_delete(&mem_stream);
}

In your old code you never called mz_zip_close. Can you update your comment on the minizip PR so other's can know what solved the problem for you?

I did not update the old code with all the different modifications including mz_zip_close, which in the end created valid zip with a malformed entry. Will update, thank you. It would be nice if you could include this example in the docs too.