A Rack middleware (at this point, for Rails) to handle Nginx mail authentication proxy.

address.rb

#
# == Mail::Address extensions
#
# Add support for local extension: user+extension@domain.tld
#
module Mail
  class Address
    def local_user
      local.to_s.split('+', 2).first
    end
    def local_extension
      local.to_s.split('+', 2).last
    end
    def canonical
      local_user << '@' << domain
    end
  end
end

nginx_auth.rb

require 'mail/address'

module Rack
  #
  ## = Nginx Mail Authentication Middleware
  #
  # Rack middleware to respond to Nginx Mail Proxy requests for authentication.
  #
  # Nginx wants to know if the remote user is authorized to use the mail service
  # and if so, where to forward the connection to.
  #
  # Rack::NginxAuth intercepts the request and provides a suitable response.
  # It relies on a model to authenticate the remote user and identify 
  # the server address for that user.
  #
  # === Nginx Mail Proxy Authentication Protocol
  #
  # Nginx sends an HTTP/1.0 GET to /auth
  # It contains authentication and context data in specific headers.
  #
  # It expects a successful (200), emtpy response with headers specifying
  # the result of the authentication, along with the backend mail server
  # address and port if successful.
  #
  # @see http://wiki.nginx.org/MailCoreModule#Authentication
  #
  # === License
  #
  # Copyright 2010 Hellekin O. Wolf
  #
  #  This program is free software: you can redistribute it and/or modify
  #  it under the terms of the GNU Affero General Public License as
  #  published by the Free Software Foundation, either version 3 of the
  #  License, or (at your option) any later version.
  #              
  #  This program is distributed in the hope that it will be useful,
  #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  #  GNU Affero General Public License for more details.
  #
  #  You should have received a copy of the GNU Affero General Public License
  #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  #
  # TODO: something with the CLIENT_IP and Auth-Login-Attempt
  # TODO: pop3 support
  #
  class NginxAuth
    def initialize(app)
      @app = app
    end
    
    #
    # == Nginx Authentication Request
    #
    # Nginx sends an HTTP/1.0 GET request to /auth with specific headers.
    #
    def call(env)
      nginx_auth_request?(env) ? process_request(env) : @app.call(env)
    end
    
    private
    
    def nginx_auth_request?(env)
      'GET' == env['REQUEST_METHOD'] &&
      '/auth' == env['PATH_INFO'] &&
      %w(imap smtp).include?(env['HTTP_AUTH_PROTOCOL'])
    end
    
    def process_request(env)
      headers = send(:"process_#{env['HTTP_AUTH_PROTOCOL']}_request", env)
      auth_response env, headers
    end
    
    #
    # == Rack::NginxAuth Response
    #
    # Nginx expects a successful, empty response with specific headers.
    #
    # @see: #auth_failure, #auth_success, #common_headers
    #
    def auth_response(env, headers)
      [200, headers.merge(common_headers(env)), []]
    end

    #
    # == Nginx IMAP Proxy Request
    #
    # We simply authenticate the user.
    #
    # @see: #auth_success
    # @see: #auth_failure
    #
    def process_imap_request(env)
      headers_for_authentication(env)
    end

    #
    # == Nginx SMTP Proxy Request
    #
    # It's a bit more interesting than IMAP: here you have access to the mail message context:
    #
    # - HTTP_AUTH_SMTP_HELO
    # - HTTP_AUTH_SMTP_FROM
    # - HTTP_AUTH_SMTP_TO
    #
    def process_smtp_request(env)
      ehlo, from, to = env['HTTP_AUTH_SMTP_HELO'], env['HTTP_AUTH_SMTP_FROM'], env['HTTP_AUTH_SMTP_TO']
      if ehlo.nil? and from.nil? and to.nil? and 'none' != env['HTTP_AUTH_METHOD']
        headers_for_authentication(env) # Empty SMTP request with auth method
      elsif smtp_ehlo_ok?(ehlo) and smtp_recipient_ok?(to) and 'none' == env['HTTP_AUTH_METHOD']
        headers_for_regular_smtp        # Regular unauthenticated SMTP request
      elsif not smtp_ehlo_ok?(ehlo) or not smtp_sender_ok?(from) or not smtp_recipient_ok?(to)
        auth_failure(env)               # Bad request?
      else
        headers_for_authentication(env) # Another authenticated request?
      end
    end

    #
    # == Supporting Model
    #
    # The User model must implement #service_host, #service_port and #identifier,
    # which hold the backend's address and port, and the authenticated username.
    #
    def headers_for_authentication(env)
      account = User.authenticate(env['HTTP_AUTH_USER'], env['HTTP_AUTH_PASSWORD'], 
                                  env['HTTP_AUTH_PROTOCOL'], env['HTTP_AUTH_METHOD'])
      account ? auth_success(account) : auth_failure(env)
    end

    def headers_for_regular_smtp
      {
        'Auth-Status'    => 'OK',
        'Auth-Server'    => ::ActionMailer::Base.smtp_settings[:address],
        'Auth-Port'      => ::ActionMailer::Base.smtp_settings[:port].to_s
      }
    end

    def auth_success(account)
      {
        "Auth-Status"    => "OK",
        "Auth-Server"    => account.service_host,
        "Auth-Port"      => account.service_port.to_s,
        "Auth-User"      => account.identifier
      }
    end

    def auth_failure(env)
      {
        "Auth-Status"    => "Invalid login or password",
        "Auth-Wait"      => '3',
        "Auth-User"      => env['HTTP_AUTH_USER'],
        "Auth-Pass"      => env['HTTP_AUTH_PASS']
      }
    end

    def common_headers(env)
      {
        'Host'             => env['HTTP_SERVER_HOST'],
        'Port'             => env['HTTP_SERVER_PORT'],
        'Auth-Protocol'    => env['HTTP_AUTH_PROTOCOL']
      }
    end
    
    #
    ## SMTP checks
    #

    # Generic case, allow missing EHLO for authenticating cases.
    # Postfix will take care of them.
    def smtp_ehlo_ok?(header_ehlo)
      (header_ehlo.nil? or header_ehlo.empty? or
         !!(header_ehlo =~ /\A(\[[0-9\.]+\]|[a-z][\w\.-]+\.[a-z][\w\.-]+)\z/))
    end

    # Model-dependent.  Certainly slower than Postfix.
    def smtp_recipient_ok?(header_to)
      return true if header_to.nil?
      from = Mail::Address.new(header_to.to_s.split(':', 2).last)
      cred = Credential.by_strategy(:imap).by_identifier(from.canonical).first
      cred && cred.user.can_receive_email?
    rescue ::Exception => e
      header_to.blank? # if it's empty, it's a login (it will be rejected anyway by postfix)
    end

    # To implement a blacklist?
    def smtp_sender_ok?(header_from)
      true
    end

  end
end