Zip all the google cloud storage files with a prefix, and upload it back into the prefix.

google_cloud_storage_multiple_file_zip.go

var StorageBucket string // google storage bucket name
var PrefixToZip string // the prefix you want to zip
var ArchiveZipFilename string // the ultimate name of the zipfile, e.g. archive.zip

var GoogleCloudClientOption option.ClientOption // can load creds from a json file with this, e.g. GoogleCloudClientOption = option.WithCredentialsFile(jsonFile)

var LogContext string // additional logging string

// Download all files & zip (named ArchiveZipFilename) from gcloud prefix (PrefixToZip)
func main() {
	ctx := &context.Background()
	client, serr := storage.NewClient(ctx)
	if GoogleCloudClientOption != nil {
		client, serr = storage.NewClient(ctx, GoogleCloudClientOption)
	}
	if serr != nil {
		reportError(serr)
		os.Exit(1)
	}
  
	objects := []*storage.ObjectAttrs{}
	it := client.Bucket(StorageBucket).Objects(*ctx,
		&storage.Query{
			Prefix: PrefixToZip, // no delim, to pickup sub folders
		},
	)
	for {
		objectAttrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			reportError(err)
			os.Exit(1)
		}
		objects = append(objects, objectAttrs)
	}

	if len(objects) < 1 {
		reportError(fmt.Errorf("No objects found in storage folder bucket(%s)/%v", StorageBucket, PrefixToZip))
		os.Exit(1)
	}

	// create storage file for writing
	zipFileURI := PrefixToZip + "/" + ArchiveZipFilename
	storageWriter := client.Bucket(StorageBucket).Object(zipFileURI).NewWriter(*ctx)
	storageWriter.ContentType = "application/zip"
	defer storageWriter.Close()

	// Create a new zip archive to memory buffer
	zipWriter := zip.NewWriter(storageWriter)

	// go through each file in the prefix
	for _, obj := range objects {
		log.Printf("Packing file %v of size %v to zip file", obj.Name, obj.Size)

		storageReader, rerr := client.Bucket(StorageBucket).Object(obj.Name).NewReader(*ctx)
		if rerr != nil {
			reportError(fmt.Errorf("unable to open file from bucket %s, file %q: `%+v`", StorageBucket, obj.Name, rerr))
			continue
		}

		// make all paths relative
		relativeFilename, rerr := filepath.Rel("/"+PrefixToZip, "/"+obj.Name)
		if rerr != nil {
			reportError(rerr)
			storageReader.Close()
			continue
		}

		// add filename to zip
		zipFile, zerr := zipWriter.Create(relativeFilename)
		if zerr != nil {
			reportError(zerr)
			storageReader.Close()
			continue
		}

		// copy from storage reader to zip writer
		_, cerr := io.Copy(zipFile, storageReader)
		if cerr != nil {
			reportError(cerr)
			storageReader.Close()
			continue
		}

		storageReader.Close()
	}

	err := zipWriter.Close()
	if err != nil {
		reportError(fmt.Errorf("Packing failed to close zip file writer from bucket %q file %q : %v", StorageBucket, zipFileURI, err))
		storageWriter.Close()
		os.Exit(1)
	}

	storageWriter.Close() // we should be uploaded by here
}


func reportError(err error) { // I typically notify other services here, hence the convenience function...
	errorMsg := fmt.Sprintf("{\"error\": \"%v\", \"trace\":\"%v\",\"context\":\"%s\"}", err, debug.Stack(), LogContext)
	log.Printf("[ERROR] %+v\n", errorMsg)
}