model_gcs_shared.go

func openFileReader(ctx context.Context, bucketName, fileName string) (*storage.Reader, error) {
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "Error getting GCS Client: %s", err)
		return nil, err
	}
	defer client.Close()

	bucket := client.Bucket(bucketName)

	obj, err := bucket.Object(fileName).NewReader(ctx)
	if err != nil {
		log.Errorf(ctx, "Error opening reader for file on GCS (gs://%s/%s): %s", bucketName, fileName, err)
		return nil, err
	}

	return obj, nil
}

func openFileWriter(ctx context.Context, bucketName, fileName string) (*storage.Writer, error) {
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "Error getting GCS Client: %s", err)
		return nil, err
	}
	defer client.Close()

	bucket := client.Bucket(bucketName)

	obj := bucket.Object(fileName).NewWriter(ctx)
	if err != nil {
		log.Errorf(ctx, "Error opening writer for file on GCS (gs://%s/%s): %s", bucketName, fileName, err)
		return nil, err
	}

	return obj, nil
}

zip_generate_excerpt.go

  snapshotBucket, err := snapshotGetBucket(ctx)
	if err != nil {
		error500Handler(w, r)
		return
	}
	snapshotDir, zipFileName := snapshotGetPrefix(reqID), snapshotGetZIPFile(reqID)

	// Delete the file if it already exists - ignore any errors.
	deleteFile(ctx, snapshotBucket, zipFileName)

	log.Infof(ctx, "Opening GCS writer...")
	zipFileHandle, err := openFileWriter(ctx, snapshotBucket, zipFileName)
	if err != nil {
		error500Handler(w, r)
		return
	}
	log.Infof(ctx, "Opening ZIP writer...")
	zipWriter := zip.NewWriter(zipFileHandle)

	// Now grab the files from the snapshot and add them into the ZIP
	log.Infof(ctx, "Listing snapshot directory...")
	snapshotDirListing, err := directoryListing(ctx, snapshotBucket, snapshotDir)
	if err != nil {
		error500Handler(w, r)
		return
	}

	for _, v := range snapshotDirListing {
		shortname := v.Name[len(snapshotDir):]
		if shortname == "" {
			continue
		}

		log.Infof(ctx, "opening file reader...")
		fReader, err := openFileReader(ctx, snapshotBucket, v.Name)
		if err != nil {
			error500Handler(w, r)
			return
		}
		log.Infof(ctx, "opening file writer inside zip...")
		zWriter, err := zipWriter.Create(shortname)
		if err != nil {
			log.Errorf(ctx, "Failed to create file in ZIP: %s", err)
			error500Handler(w, r)
			return
		}

		log.Infof(ctx, "copying data into zip...")
		if _, err := io.Copy(zWriter, fReader); err != nil {
			log.Errorf(ctx, "Failed to write file contents to ZIP (%s): %s", shortname, err)
			error500Handler(w, r)
			return
		}

		log.Infof(ctx, "closing reader...")
		if err := fReader.Close(); err != nil {
			log.Errorf(ctx, "Failed to close reader of file to ZIP (%s): %s", shortname, err)
			error500Handler(w, r)
			return
		}
		log.Infof(ctx, "add file done...")
	}

	// Now grab the files from the snapshot and add them into the ZIP
	supportingBucket, err := supportingGetBucket(ctx)
	if err != nil {
		error500Handler(w, r)
		return
	}
	supportingDir := supportingGetPrefix(reqID)

	log.Infof(ctx, "Listing supporting directory...")
	supportingDirListing, err := directoryListing(ctx, supportingBucket, supportingDir)
	if err != nil {
		error500Handler(w, r)
		return
	}

	for _, v := range supportingDirListing {
		shortname := v.Name[len(supportingDir):]
		if shortname == "" {
			continue
		}

		log.Infof(ctx, "opening file reader...")
		fReader, err := openFileReader(ctx, supportingBucket, v.Name)
		if err != nil {
			error500Handler(w, r)
			return
		}
		log.Infof(ctx, "opening file writer inside zip...")
		zWriter, err := zipWriter.Create(shortname)
		if err != nil {
			log.Errorf(ctx, "Failed to create file in ZIP: %s", err)
			error500Handler(w, r)
			return
		}

		log.Infof(ctx, "copying data into zip...")
		if _, err := io.Copy(zWriter, fReader); err != nil {
			log.Errorf(ctx, "Failed to write file contents to ZIP (%s): %s", shortname, err)
			error500Handler(w, r)
			return
		}

		log.Infof(ctx, "closing reader...")
		if err := fReader.Close(); err != nil {
			log.Errorf(ctx, "Failed to close reader of file to ZIP (%s): %s", shortname, err)
			error500Handler(w, r)
			return
		}
		log.Infof(ctx, "add file done...")
	}


	// Close the ZIP handle to complete the GCS upload.
	log.Infof(ctx, "closing zip handle...")
	if err := zipWriter.Close(); err != nil {
		log.Errorf(ctx, "Error closing ZIP: %s", err)
		error500Handler(w, r)
		return
	}
	log.Infof(ctx, "closing gcs file handle...")
	if err := zipFileHandle.Close(); err != nil {
		log.Errorf(ctx, "Error closing ZIP on GCS: %s", err)
		error500Handler(w, r)
		return
	}