mTLS with Warp filters and hyper-rustls

ca-config.json

{
  "signing": {
    "default": {
      "expiry": "8760h"
    },
    "profiles": {
      "server": {
        "usages": ["signing", "key encipherment", "server auth"],
        "expiry": "8760h"
      },
      "client": {
        "usages": ["signing","key encipherment","client auth"],
        "expiry": "8760h"
      }
    }
  }
}

ca-csr.json

{
  "CN": "Local Test CA",
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
      "C": "US"
    }
  ]
}

client-csr.json

{
  "CN": "Client",
  "hosts": [""],
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
      "C": "US"
    }
  ]
}

main.rs

// This is an example of getting mutual TLS to work with Warp filters.
// It stores the client certificate in the request extension field.
//
// It serves on https://localhost:8433 the routes:
//   /subject
//   /issuer
//   /noauth
//
// The first two routes require a client certificate in the request.
//
// Cargo.toml deps:
//
// [dependencies]
// futures = "0.3"
// x509-parser = "0.11.0"
// warp = "0.3.1"
// hyper = { version =  "0.14.12", features = ["full"]}
// hyper-rustls = "0.22.1"
// rustls = "0.19.1"
// tokio = { version = "1", features = ["full"] }
// tokio-rustls = "0.22.0"
use futures::TryFutureExt;
use hyper::{service, service::Service};
use rustls::{internal::pemfile, Session};
use std::{fs, io, sync, vec::Vec};
use tokio::net;
use warp::Filter;
use x509_parser::{certificate, traits::FromDer};

async fn serve_mtls_connection<F>(
    listener: &net::TcpListener,
    tls_acceptor: &tokio_rustls::TlsAcceptor,
    warp_filter: F,
) -> io::Result<()>
where
    F: warp::Filter + Clone + Send + Sync + 'static,
    <F::Future as futures::TryFuture>::Ok: warp::Reply,
{
    // Wait for an incoming TCP connection
    let (socket, client_addr) = listener.accept().await?;
    println!("Received connection from {}", client_addr);

    // Interpret data coming through the TCP stream as a TLS stream
    let stream = tls_acceptor
        .accept(socket)
        .map_err(|err| {
            io::Error::new(
                io::ErrorKind::Other,
                format!("Problem accepting TLS connection: {:?}", err),
            )
        })
        .await?;

    // Hand off actual request handling to a new tokio task
    tokio::task::spawn(async move {
        // Pull the client certificate out of the TLS session
        let (_, session) = stream.get_ref();
        let client_cert = session.get_peer_certificates().and_then(|certs| {
            if certs.is_empty() {
                None
            } else {
                Some(certs[0].clone())
            }
        });

        // Turn the warp filter into a service, but instead of using that
        // service directly as usual, we wrap it around another service
        // so that we can modify the request and inject the client certificate
        // into the request extentions before it goes into the filter.
        let mut svc = warp::service(warp_filter.clone());
        let service = service::service_fn(move |mut req| {
            if let Some(cert) = client_cert.to_owned() {
                req.extensions_mut().insert(cert);
            }
            svc.call(req)
        });
        if let Err(e) = hyper::server::conn::Http::new()
            .serve_connection(stream, service)
            .await
        {
            eprintln!("Error handling request: {}", e);
        }
    });

    return Ok(());
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    // Listen for TCP connection
    let svr_addr = "localhost:8433";
    let listener = net::TcpListener::bind(&svr_addr).await?;

    // Build TLS configuration.
    let tls_config = {
        // Load server key pair
        let (certs, key) = load_keypair("server.pem", "server-key.pem")?;

        // Load root CA key(s) used to verify client certificate
        let rootstore = load_root_store("ca.pem")?;

        let mut config = rustls::ServerConfig::new(
            rustls::AllowAnyAnonymousOrAuthenticatedClient::new(rootstore),
        );
        // Select a certificate to use.
        config
            .set_single_cert(certs, key)
            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{}", err)))?;
        sync::Arc::new(config)
    };

    // A warp filter where we assume that a client certificate will be stored
    // in the request extensions if the request is made through a TLS connection

    let subject_route = warp::path("subject")
        .and(warp::ext::get::<rustls::Certificate>())
        .map(|cert: rustls::Certificate| {
            if let Ok((_, parsed_cert)) = certificate::X509Certificate::from_der(&cert.0) {
                format!("Got a client certificate with subject: {}", parsed_cert.subject())
            } else {
                format!("Got something in the certificate extension but cannot parse it\n")
            }
        });

    let issuer_route = warp::path("issuer")
        .and(warp::ext::get::<rustls::Certificate>())
        .map(|cert: rustls::Certificate| {
            if let Ok((_, parsed_cert)) = certificate::X509Certificate::from_der(&cert.0) {
                format!("Got a client certificate from issuer: {}\n", parsed_cert.issuer())
            } else {
                format!("Got something in the certificate extension but cannot parse it\n")
            }
        });

    let without_cert_route = warp::path("noauth")
        .map(|| {
               format!("This route does not require certs\n")
        });


    let mtls_route = 
        subject_route
        .or(issuer_route)
        .or(without_cert_route);

    let tls_acceptor = tokio_rustls::TlsAcceptor::from(tls_config);

    loop {
        if let Err(e) = serve_mtls_connection(&listener, &tls_acceptor, mtls_route).await {
            eprintln!("Problem accepting TLS connection: {}", e);
        }
    }
}

fn load_keypair(
    certfile: &str,
    keyfile: &str,
) -> io::Result<(Vec<rustls::Certificate>, rustls::PrivateKey)> {
    let file = fs::File::open(certfile)?;
    let mut reader = io::BufReader::new(file);

    let certs = pemfile::certs(&mut reader).map_err(|_err| {
        io::Error::new(
            io::ErrorKind::Other,
            format!("Cannot load certificate from {}", certfile),
        )
    })?;

    let file = fs::File::open(keyfile)?;
    let mut reader = io::BufReader::new(file);

    // Load and return a single private key.
    let keys = pemfile::rsa_private_keys(&mut reader).map_err(|_err| {
        io::Error::new(
            io::ErrorKind::Other,
            format!("Cannot load private key from {}", keyfile),
        )
    })?;

    let key = keys.first().ok_or(io::Error::new(
        io::ErrorKind::Other,
        format!("No keys found in the private key file {}", keyfile),
    ))?;

    Ok((certs, key.clone()))
}

fn load_root_store(filename: &str) -> io::Result<rustls::RootCertStore> {
    let file = fs::File::open(filename)?;
    let mut reader = io::BufReader::new(file);

    let mut store = rustls::RootCertStore::empty();
    store.add_pem_file(&mut reader).map_err(|_err| {
        io::Error::new(
            io::ErrorKind::Other,
            format!("Failed to load root cert store from {}", filename),
        )
    })?;

    Ok(store)
}

server-csr.json

{
  "CN": "Server",
  "hosts": ["localhost", "127.0.0.1"],
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
      "C": "US"
    }
  ]
}

@darwindarak interesting, I noticed in the hyper docs you linked to that this line is outside of the loop instead of inside
let mut tcp_listener = TcpListener::bind(addr).await?;
So I tried testing out your code with that change and it seems to work fine now

@howhelan Can you please be more explicit on what you changed? let mut tcp_listener = TcpListener::bind(addr).await?; is already out of the loop. Or am I missing something?
@darwindarak A good tool to do this is JMeter, and it is pretty straightforward to use too.

I have yet another question. Wouldn't it be better if we also push

    // Interpret data coming through the TCP stream as a TLS stream
    let stream = tls_acceptor
        .accept(socket)
        .map_err(|err| {
            io::Error::new(
                io::ErrorKind::Other,
                format!("Problem accepting TLS connection: {:?}", err),
            )
        })
        .await?;

inside the new Thread?

Yes, clearly I've been looking at this for too long.. I must've forgotten changing that in my code and assumed it was the same here. Sorry for the confusion!

I'll try some basic load testing on this and post the results here when I get around to it this week.

Great
Looking forward to your results.

Hello Guys.
@howhelan Any updates on this? I have been making a couple of tests too.
@darwindarak you mentioned an issue with line 149 and the .await clause. What do you mean by removing it?

And I am having problems moving this:

    // Interpret data coming through the TCP stream as a TLS stream
    let stream = tls_acceptor
        .accept(socket)
        .map_err(|err| {
            io::Error::new(
                io::ErrorKind::Other,
                format!("Problem accepting TLS connection: {:?}", err),
            )
        })
        .await?;

Inside the new thread. I mean it should make sense that we wait for the connection in the loop and we validate the TLS part inside the thread so that we do not lose any valuable time on waiting for other connections.

The problem is that I cannot insert the above code inside the Thread, since I get several issues such as moving values etc. :).

Best Regards.

@renis1235 I've run some very non-scientific tests with our application using this code and found that I can get close to 150 request per second before I start seeing errors. That being said, there are a lot of other dependencies in the application that could be the bottleneck there, so I'm still intending to test the bare-bones code here to get a more targeted result. I've just hit a couple issues getting the mutual TLS working with Jmeter that I need to figure out. Sorry for the delay!

I can certainly help you with configuring JMeter though.
Here are the steps:

1. First create a .p12 file from your certificate and key file. It is pretty straightforward:

openssl pkcs12 -export -out keyStore.p12 -inkey myKey.pem -in certs.pem

2. Start JMeter and click on options
   

3. Select your certificate (.p12) file and input the password whenever it is required.

I hope this helps. @howhelan
If you need anything else
I will gladly assist :)