stream-upload-to-s3.js

var AWS = require('aws-sdk'),
    fs = require('fs'),
    awsCredentialsPath = './aws.credentials.json',
    sqsQueueUrl = 'SQS_QUEUE_URL',
    http = require('http'),
    sqs;

  // Load credentials from local json file
  AWS.config.loadFromPath(awsCredentialsPath);

  function retrieveFile(filename, callback) {
    var s3Stream = require('s3-upload-stream')(new AWS.S3());

    var upload = s3Stream.upload({
      "Bucket": "BUCKET_NAME",
      "Key": filename
    });

    upload.on('error', function (error) {
      console.log(error);
    });

    upload.on('part', function (details) {
      callback();
    });

    // get the given filename from the path given
    http.get({
      hostname: 'cdn.gea.esac.esa.int',
      port: 80,
      path: `/Gaia/gaia_source/csv/${filename}`,
      agent: false  // create a new agent just for this one request
    }, (res) => {
      // stream-upload the file to Amazon S3 as it comes in
      res.pipe(upload);
    });

  }

module.exports = retrieveFile;