description.md

Stream email attachments to S3 using Mailgun, node.js

Using:

• Mailgun
• express
• request
• busboy
• s3-streaming-upload

1. Mailgun setup

First define a Mailgun Route using the store action:

Filter Expression: match_recipient("save@YOUR_DOMAIN")
Action: store(notify="YOUR_ROUTE")

We could forward the email directly to the handler, however this approach allows us to verify the email prior to doing anything with the attachments.

2. Express setup

See email.js.

email.js

var _ = require('underscore')
var assert = require('assert')
var async = require('async')
var connectBusboy = require('connect-busboy')
var express = require('express')
var request = require('request')
var Uploader = require('s3-streaming-upload').Uploader

var server = module.exports = express()
var parser = connectBusboy({
    immediate: true,
    files: 0
})

server.post(YOUR_ROUTE, [parser], postEmail)

function postEmail(req, res) {
    parse(req.busboy, function(err, _res, body){
        if (err) throw err
        if (_res.statusCode !== 200) throw new Error('unable to retrieve message')
        var msg = JSON.parse(body)
        console.log('saving work for ' + msg.sender + '...')
        upload(msg, function(err, attachments){
            if (err) throw err
            console.log('work saved')
            return res.send({ attachments: _.map(attachments, function(a){ return a.location }) })
        })
    })
}

function parse(busboy, callback) {
    var url
    busboy
        .on('error', function(err){
            return callback(err)
        })
        .on('field', function(name, value){
            if (name !== 'message-url') return
            url = value
        })
        .on('end', function(){
            if (!url) return callback(new Error('message url does not exist'))
            return request.get(url, callback).auth('api', YOUR_MAILGUN_API_KEY, false)
        })
}

function upload(msg, callback) {
    var tasks
    try {
        assert(_.isObject(msg), 'message must be an object')
        assert(_.isArray(msg.attachments), 'message must have an attachments array')
        tasks = _.map(msg.attachments, function(attachment){
            return async.apply(uploadAttachment, attachment)
        })
    } catch (ex) {
        return callback(ex)
    }
    return async.parallel(tasks, callback)
}

function uploadAttachment(attachment, callback) {
    var readStream
    var options
    try {
        assert(_.isObject(attachment), 'must have an attachment')
        assert(_.isString(attachment.url), 'attachment must have a url')
        assert(_.isString(attachment['content-type']), 'attachment must have a content type')
        assert(_.isString(attachment.name), 'attachment must have a name')
        readStream = request.get(attachment['url']).auth('api', YOUR_MAILGUN_API_KEY, false)
        options = {
            accessKey: YOUR_AWS_KEY,
            secretKey: YOUR_AWS_SECRET,
            bucket: YOUR_AWS_BUCKET,
            objectName: attachment.name,
            objectParams: {
                ACL: 'public-read',
                ContentType: attachment['content-type']
            },
            stream: readStream
        }
    } catch (ex) {
        return callback(ex)
    }
    new Uploader(options)
        .on('completed', callback)
        .on('failed', callback)
}