mTLS with Warp filters and hyper-rustls

ca-config.json

{
  "signing": {
    "default": {
      "expiry": "8760h"
    },
    "profiles": {
      "server": {
        "usages": ["signing", "key encipherment", "server auth"],
        "expiry": "8760h"
      },
      "client": {
        "usages": ["signing","key encipherment","client auth"],
        "expiry": "8760h"
      }
    }
  }
}

ca-csr.json

{
  "CN": "Local Test CA",
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
      "C": "US"
    }
  ]
}

client-csr.json

{
  "CN": "Client",
  "hosts": [""],
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
      "C": "US"
    }
  ]
}

main.rs

// This is an example of getting mutual TLS to work with Warp filters.
// It stores the client certificate in the request extension field.
//
// It serves on https://localhost:8433 the routes:
//   /subject
//   /issuer
//   /noauth
//
// The first two routes require a client certificate in the request.
//
// Cargo.toml deps:
//
// [dependencies]
// futures = "0.3"
// x509-parser = "0.11.0"
// warp = "0.3.1"
// hyper = { version =  "0.14.12", features = ["full"]}
// hyper-rustls = "0.22.1"
// rustls = "0.19.1"
// tokio = { version = "1", features = ["full"] }
// tokio-rustls = "0.22.0"
use futures::TryFutureExt;
use hyper::{service, service::Service};
use rustls::{internal::pemfile, Session};
use std::{fs, io, sync, vec::Vec};
use tokio::net;
use warp::Filter;
use x509_parser::{certificate, traits::FromDer};

async fn serve_mtls_connection<F>(
    listener: &net::TcpListener,
    tls_acceptor: &tokio_rustls::TlsAcceptor,
    warp_filter: F,
) -> io::Result<()>
where
    F: warp::Filter + Clone + Send + Sync + 'static,
    <F::Future as futures::TryFuture>::Ok: warp::Reply,
{
    // Wait for an incoming TCP connection
    let (socket, client_addr) = listener.accept().await?;
    println!("Received connection from {}", client_addr);

    // Interpret data coming through the TCP stream as a TLS stream
    let stream = tls_acceptor
        .accept(socket)
        .map_err(|err| {
            io::Error::new(
                io::ErrorKind::Other,
                format!("Problem accepting TLS connection: {:?}", err),
            )
        })
        .await?;

    // Hand off actual request handling to a new tokio task
    tokio::task::spawn(async move {
        // Pull the client certificate out of the TLS session
        let (_, session) = stream.get_ref();
        let client_cert = session.get_peer_certificates().and_then(|certs| {
            if certs.is_empty() {
                None
            } else {
                Some(certs[0].clone())
            }
        });

        // Turn the warp filter into a service, but instead of using that
        // service directly as usual, we wrap it around another service
        // so that we can modify the request and inject the client certificate
        // into the request extentions before it goes into the filter.
        let mut svc = warp::service(warp_filter.clone());
        let service = service::service_fn(move |mut req| {
            if let Some(cert) = client_cert.to_owned() {
                req.extensions_mut().insert(cert);
            }
            svc.call(req)
        });
        if let Err(e) = hyper::server::conn::Http::new()
            .serve_connection(stream, service)
            .await
        {
            eprintln!("Error handling request: {}", e);
        }
    });

    return Ok(());
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    // Listen for TCP connection
    let svr_addr = "localhost:8433";
    let listener = net::TcpListener::bind(&svr_addr).await?;

    // Build TLS configuration.
    let tls_config = {
        // Load server key pair
        let (certs, key) = load_keypair("server.pem", "server-key.pem")?;

        // Load root CA key(s) used to verify client certificate
        let rootstore = load_root_store("ca.pem")?;

        let mut config = rustls::ServerConfig::new(
            rustls::AllowAnyAnonymousOrAuthenticatedClient::new(rootstore),
        );
        // Select a certificate to use.
        config
            .set_single_cert(certs, key)
            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{}", err)))?;
        sync::Arc::new(config)
    };

    // A warp filter where we assume that a client certificate will be stored
    // in the request extensions if the request is made through a TLS connection

    let subject_route = warp::path("subject")
        .and(warp::ext::get::<rustls::Certificate>())
        .map(|cert: rustls::Certificate| {
            if let Ok((_, parsed_cert)) = certificate::X509Certificate::from_der(&cert.0) {
                format!("Got a client certificate with subject: {}", parsed_cert.subject())
            } else {
                format!("Got something in the certificate extension but cannot parse it\n")
            }
        });

    let issuer_route = warp::path("issuer")
        .and(warp::ext::get::<rustls::Certificate>())
        .map(|cert: rustls::Certificate| {
            if let Ok((_, parsed_cert)) = certificate::X509Certificate::from_der(&cert.0) {
                format!("Got a client certificate from issuer: {}\n", parsed_cert.issuer())
            } else {
                format!("Got something in the certificate extension but cannot parse it\n")
            }
        });

    let without_cert_route = warp::path("noauth")
        .map(|| {
               format!("This route does not require certs\n")
        });


    let mtls_route = 
        subject_route
        .or(issuer_route)
        .or(without_cert_route);

    let tls_acceptor = tokio_rustls::TlsAcceptor::from(tls_config);

    loop {
        if let Err(e) = serve_mtls_connection(&listener, &tls_acceptor, mtls_route).await {
            eprintln!("Problem accepting TLS connection: {}", e);
        }
    }
}

fn load_keypair(
    certfile: &str,
    keyfile: &str,
) -> io::Result<(Vec<rustls::Certificate>, rustls::PrivateKey)> {
    let file = fs::File::open(certfile)?;
    let mut reader = io::BufReader::new(file);

    let certs = pemfile::certs(&mut reader).map_err(|_err| {
        io::Error::new(
            io::ErrorKind::Other,
            format!("Cannot load certificate from {}", certfile),
        )
    })?;

    let file = fs::File::open(keyfile)?;
    let mut reader = io::BufReader::new(file);

    // Load and return a single private key.
    let keys = pemfile::rsa_private_keys(&mut reader).map_err(|_err| {
        io::Error::new(
            io::ErrorKind::Other,
            format!("Cannot load private key from {}", keyfile),
        )
    })?;

    let key = keys.first().ok_or(io::Error::new(
        io::ErrorKind::Other,
        format!("No keys found in the private key file {}", keyfile),
    ))?;

    Ok((certs, key.clone()))
}

fn load_root_store(filename: &str) -> io::Result<rustls::RootCertStore> {
    let file = fs::File::open(filename)?;
    let mut reader = io::BufReader::new(file);

    let mut store = rustls::RootCertStore::empty();
    store.add_pem_file(&mut reader).map_err(|_err| {
        io::Error::new(
            io::ErrorKind::Other,
            format!("Failed to load root cert store from {}", filename),
        )
    })?;

    Ok(store)
}

server-csr.json

{
  "CN": "Server",
  "hosts": ["localhost", "127.0.0.1"],
  "key": {
    "algo": "rsa",
    "size": 4096
  },
  "names": [
    {
      "C": "US"
    }
  ]
}

Hey

Thank you so much for this!

I would like to use this snippet in an existing project. Can I pass several routes to serve_mtls_connection? If so, how?

What would be the correct way to handle bad requests (e.g without a certificate, wrong certificate and so on.

How would performing a cURL to your implementation look like? With Certs and all.

I would really appreciate it if you would reply.

Best regards,
Renis.

Hello, sorry for writing without getting a reply first, but my lack of knowledge on Rust/Warp has made this even more complicated for me.

I have tried sending different requests with different clients to your implementation. Of course all of them with a certificate inside. Still, the app (your implementation) is not able to get any certificate. What kind of requests have you tested? With what library/software?

Thank you :)

@renis1235 I've updated main.rs to have three different routes, subject and issuer require a client certificate and noauth does not. I'm also pretty unfamiliar with rust and warp so this is most likely not the intended way to do this. It basically comes down to messing around with the filters in Warp. I've also added files that would be helpful in generating local certificates for testing. It can get a little tricky/annoying to get right using openssl, to I'm using the cfssl tool as follows:

# Generate the root CA
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
# Generate server key pair
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server-csr.json | cfssljson -bare server
# Generate client key pair
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssljson -bare client

Testing this in with curl comes down to

# Should suceed
curl --cert client.pem --key client-key.pem --cacert ca.pem https://localhost:8433/subject
curl --cert client.pem --key client-key.pem --cacert ca.pem https://localhost:8433/issuer
curl --cert client.pem --key client-key.pem --cacert ca.pem https://localhost:8433/noauth
curl --cacert ca.pem https://localhost:8433/noauth

# Should fail
curl --cacert ca.pem https://localhost:8433/subject
curl --cacert ca.pem https://localhost:8433/issuer

Hope this helps

Thanks so much for taking the time to reply and to change the implementation for me.
I got it to work with several routes and certificates.

My question is regarding the infinite loop, would that be the correct way to serve the connection?
What happens when a new connection comes and the loop will be doing something else while that connection comes?
Than the server would have to drop it, right?

It is clear to me that when a connection is created, a tunnel is created and that connection stays alive. I am talking only about extreme cases where several thousand connections try to connect with the app in < 5s.

The await keyword on line 149 makes it so that the loop doesn't really block when serving a connection. You might want to take a look at Rust's async/await functionality. I'm definitely not the right person to ask about this since I'm not really familiar with Rust in general, but here's the way I think it works:

Async functions in Rust are scheduled and run by executors (in this case it's Tokio). The executor can ask an async function for its result, and the function and either return the result or tell the executor that it's not ready and how/when it can check back. That way, the executor can jump around multiple tasks, incrementally driving them to return their results instead of waiting on any one of them. The loop inside the main function looks like it blocks waiting for serve_mtls_connection, but if you look in serve_mtls_connection, you should see that it actually spawns off another async task to actually handle the connection without an await keyword to block it.
After adding the actual handler task to the Tokio scheduler, it simply returns Ok(()).

You might want to try adding a route where all it does is sleep for 5s. If you do, make sure you use the sleep function provided by Tokio instead of the system sleep. The system sleep function blocks the entire thread, which also stops the executor, whereas Tokio's sleep function basically tells the executor that it doesn't need to check back on the task for a while.

Ah
Now I understood everything. So basically this would not have any disadvantages to warp's .server().run function, right? I guess warp works the same way since it also opens a new (TCP) connection for every new client.

I can't thank you enough, this has helped a lot for my Master's Thesis.

@darwindarak thanks for the workaround here. I think @renis1235 is onto something in terms of the loop not being the correct way to serve the connections. When I make 10-20 requests simultaneously about half of them get dropped, probably because the Tokio task does not get spawned immediately at the start of the loop. Perhaps if the code between the start of the loop and the start of the Tokio task are executing when a request comes in, the request is not processed?

In my case the easiest solution here may be to put NGINX in front of the warp server, terminate TLS there, and then pass the client cert to warp as a header.

@howhelan Wouldn't it be easier to work with the code and not with NGINX?

@howhelan thanks for catching that. I didn't know that NGINX can wrap the client cert into the header, cool! That might be the most appropriate way to do this on production code until support is added into the hyper/warp ecosystem.

I don't think the problem is in using a loop, since I modeled this after the example on the hyper::server::conn docs. I'm almost sure the problem is with the counterproductive .await inside the loop on line 149. Do you mind removing that and testing to see that that fixes it?

Can you recommend a simple tool to generate/test simultaneous requests?

@renis1235 it doesn't seem trivial to implement with Warp. Here is the issue in Warp which has been open for more than two years. In my case it should work just as well to use nginx for this purpose.

@darwindarak interesting, I noticed in the hyper docs you linked to that this line is outside of the loop instead of inside
let mut tcp_listener = TcpListener::bind(addr).await?;
So I tried testing out your code with that change and it seems to work fine now

@howhelan Can you please be more explicit on what you changed? let mut tcp_listener = TcpListener::bind(addr).await?; is already out of the loop. Or am I missing something?
@darwindarak A good tool to do this is JMeter, and it is pretty straightforward to use too.

I have yet another question. Wouldn't it be better if we also push

    // Interpret data coming through the TCP stream as a TLS stream
    let stream = tls_acceptor
        .accept(socket)
        .map_err(|err| {
            io::Error::new(
                io::ErrorKind::Other,
                format!("Problem accepting TLS connection: {:?}", err),
            )
        })
        .await?;

inside the new Thread?

Yes, clearly I've been looking at this for too long.. I must've forgotten changing that in my code and assumed it was the same here. Sorry for the confusion!

I'll try some basic load testing on this and post the results here when I get around to it this week.

Great
Looking forward to your results.

Hello Guys.
@howhelan Any updates on this? I have been making a couple of tests too.
@darwindarak you mentioned an issue with line 149 and the .await clause. What do you mean by removing it?

And I am having problems moving this:

    // Interpret data coming through the TCP stream as a TLS stream
    let stream = tls_acceptor
        .accept(socket)
        .map_err(|err| {
            io::Error::new(
                io::ErrorKind::Other,
                format!("Problem accepting TLS connection: {:?}", err),
            )
        })
        .await?;

Inside the new thread. I mean it should make sense that we wait for the connection in the loop and we validate the TLS part inside the thread so that we do not lose any valuable time on waiting for other connections.

The problem is that I cannot insert the above code inside the Thread, since I get several issues such as moving values etc. :).

Best Regards.

@renis1235 I've run some very non-scientific tests with our application using this code and found that I can get close to 150 request per second before I start seeing errors. That being said, there are a lot of other dependencies in the application that could be the bottleneck there, so I'm still intending to test the bare-bones code here to get a more targeted result. I've just hit a couple issues getting the mutual TLS working with Jmeter that I need to figure out. Sorry for the delay!

I can certainly help you with configuring JMeter though.
Here are the steps:

1. First create a .p12 file from your certificate and key file. It is pretty straightforward:

openssl pkcs12 -export -out keyStore.p12 -inkey myKey.pem -in certs.pem

2. Start JMeter and click on options
   

3. Select your certificate (.p12) file and input the password whenever it is required.

I hope this helps. @howhelan
If you need anything else
I will gladly assist :)