poc-gcs-upload-bug.js

const fs = require('fs');
const stream = require('stream');
const path = require('path');
const util = require('util');

const gcs = require('google-cloud')({projectId: process.env.PROJECT_ID, keyFilename: './keyfile.json'}).storage();
const bucketName = process.env.BUCKET_NAME;

const concurrency = +process.env.CONCURRENCY || 100;

/**
 * make sure this file exists and is a few bytes
 * $ cat test-smallfile.csv
 * test, test, test
 * test, test, test
 * test, test, test
 */
const fileToUpload = process.env.FILENAME || 'test-smallfile.csv';

function uploadFileToCloud(fileToUpload, index) {
  return new Promise(function (resolve, reject) {
    let extname = path.extname(fileToUpload);
    let fileName = `${path.basename(fileToUpload, extname)}-${index}${extname}`;

    gcs.getBuckets(function (err, result) {
      if (err) {
        console.error({ err }, 'Error getting the bucket' + err);
        reject(err);
        return;
      } else {
        let targetBucket = gcs.bucket(bucketName);
        let targetFile = targetBucket.file(fileName);
        fs.createReadStream(fileToUpload)
        .pipe(new stream.Transform({transform(chunk, enc, done) {
          this.push(chunk);
          // make the file unique, note that this does NOT reproduce
          // the issue in and of itself
          this.push(`${index}`);
          done();
        }}))
        .pipe(targetFile.createWriteStream({ validation: 'md5' }))
        .on('error', function (err) {
          console.error(err, 'Error while uploading target file ' + err);
          reject(err);
          return;
        })
        .on('finish', function () {
          resolve(url);
        });
      }
    });
  });
}

function doUpload(i){
  let name = `test-smallfile-${i}`;
  process.stdout.write('='); // write a `=` every upload start
  uploadFileToCloud(fileToUpload, i).then(url => {
    process.stdout.write('\b \b'); // delete a `=` every upload complete
    let jitter = Math.floor(Math.random() * 1000);
    // start another upload after `jitter` ms
    setTimeout(() => doUpload(i+concurrency), jitter);
  }).catch(err => {
    // finally, we got an error
    console.log(`filename: ${name}.csv`)
    console.error(err);
    console.log(err);
    process.exit(1);
  });
}

process.stdout.write('\n');
for(let i=0;i<concurrency;i++){
  let jitter = Math.floor(Math.random() * 1000);
  setTimeout(() => doUpload(i), jitter);
}