Skip to content

Instantly share code, notes, and snippets.

@christophercliff
Last active February 24, 2019 23:30
Show Gist options
  • Save christophercliff/7723712 to your computer and use it in GitHub Desktop.
Save christophercliff/7723712 to your computer and use it in GitHub Desktop.
Stream email attachments to S3 with Mailgun, node.js.

Stream email attachments to S3 using Mailgun, node.js

Using:

1. Mailgun setup

First define a Mailgun Route using the store action:

Filter Expression: match_recipient("save@YOUR_DOMAIN")
Action: store(notify="YOUR_ROUTE")

We could forward the email directly to the handler, however this approach allows us to verify the email prior to doing anything with the attachments.

2. Express setup

See email.js.

var _ = require('underscore')
var assert = require('assert')
var async = require('async')
var connectBusboy = require('connect-busboy')
var express = require('express')
var request = require('request')
var Uploader = require('s3-streaming-upload').Uploader
var server = module.exports = express()
var parser = connectBusboy({
immediate: true,
files: 0
})
server.post(YOUR_ROUTE, [parser], postEmail)
function postEmail(req, res) {
parse(req.busboy, function(err, _res, body){
if (err) throw err
if (_res.statusCode !== 200) throw new Error('unable to retrieve message')
var msg = JSON.parse(body)
console.log('saving work for ' + msg.sender + '...')
upload(msg, function(err, attachments){
if (err) throw err
console.log('work saved')
return res.send({ attachments: _.map(attachments, function(a){ return a.location }) })
})
})
}
function parse(busboy, callback) {
var url
busboy
.on('error', function(err){
return callback(err)
})
.on('field', function(name, value){
if (name !== 'message-url') return
url = value
})
.on('end', function(){
if (!url) return callback(new Error('message url does not exist'))
return request.get(url, callback).auth('api', YOUR_MAILGUN_API_KEY, false)
})
}
function upload(msg, callback) {
var tasks
try {
assert(_.isObject(msg), 'message must be an object')
assert(_.isArray(msg.attachments), 'message must have an attachments array')
tasks = _.map(msg.attachments, function(attachment){
return async.apply(uploadAttachment, attachment)
})
} catch (ex) {
return callback(ex)
}
return async.parallel(tasks, callback)
}
function uploadAttachment(attachment, callback) {
var readStream
var options
try {
assert(_.isObject(attachment), 'must have an attachment')
assert(_.isString(attachment.url), 'attachment must have a url')
assert(_.isString(attachment['content-type']), 'attachment must have a content type')
assert(_.isString(attachment.name), 'attachment must have a name')
readStream = request.get(attachment['url']).auth('api', YOUR_MAILGUN_API_KEY, false)
options = {
accessKey: YOUR_AWS_KEY,
secretKey: YOUR_AWS_SECRET,
bucket: YOUR_AWS_BUCKET,
objectName: attachment.name,
objectParams: {
ACL: 'public-read',
ContentType: attachment['content-type']
},
stream: readStream
}
} catch (ex) {
return callback(ex)
}
new Uploader(options)
.on('completed', callback)
.on('failed', callback)
}
@amitaibu
Copy link

amitaibu commented May 1, 2014

Thanks for that! any chance you have a git repo with the entire setup? :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment