Jersey/JAX-RS protobuf produce and consume

README.md

Using Jersey/JAX-RS with protobuf

I had a requirement to produce and consume protobuf messages through a Jersey web service, either as their binary representation, or as JSON. Helpfully, the protobuf-java-util artefact contains the JsonFormat class that handles the proto-to-JSON and JSON-to-proto side of things.

An elegant approach when using Jersey is to create (and register) MessageBodyWriter and MessageBodyReader providers. The appropriate implementation will be used based on the Accept and Content-type header the HTTP client sends.

Your service code is as simple as:

@GET
@Path("/hello")
public Common.Event test() {
    return Common.Event.newBuilder().setIdentifier("Hello World").build();
}

...and, to consume

@POST
@Path("/hello")
public Common.Event testPost(Common.Event input) {
    return Common.Event.newBuilder().setIdentifier("You said: " + input.getIdentifier()).build();
}

Registering the providers with Jersey looks like the below.

ResourceConfig rc = new ResourceConfig();
// Resources can produce either application/x-protobuf or application/json (Accept header)
rc.register(ProtobufBinaryMessageWriter.class);
rc.register(ProtobufJsonMessageWriter.class);
// Resource that take a body can consume application/json
rc.register(ProtobufJsonMessageReader.class); 

An HTTP header, x-proto-type is added to the response. This is the fully qualified name of the message, which would be used by the client to look up a descriptor in order to parse the response. As JSON responses are self-describing this isn't really required.

proto3 has an Any type that allows messages of, well, any type, to be embedded. In order for the JSON serializer to work, you need to register the descriptors of possibly enclosed message types. See line 30 of ProtobufJsonMessageWriter.java

The classes within this gist are adapted from various older (possibly proto2) implementations that you'll probably have also found by Googling this.

Note that I have only implemented consumption of JSON as I had no requirement to accept a binary protobuf through a POST request at this point.

How am I using this?

I have messages on Kafka topics as protobufs. I am using the interactive queries feature in Kafka Streams to expose a Kafka Streams state store as a REST service. Browsers may access this REST service so need JSON. Due to the architecture of Kafka Streams IQ, I also need to support service-to-service communication so that a request for a key held on another instances' state store can be proxied back to the caller. protobuf seemed to be the natural serialization to use here instead of JSON. If you're interested in Kafka Streams interactive queries, see here: https://kafka.apache.org/20/documentation/streams/developer-guide/interactive-queries.html

ProtobufBinaryMessageWriter.java

package com.github.alexjreid;

import com.google.protobuf.Message;

import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyWriter;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;

/**
 * Writes a protobuf message, with the type specified in annotated method signature, to the response.
 * Adds an HTTP header, x-proto-type, to denote the proto used.
 */
@Provider
@Produces("application/x-protobuf")
public class ProtobufBinaryMessageWriter implements MessageBodyWriter {
    public boolean isWriteable(Class type, Type genericType, Annotation[] annotations, MediaType mediaType) {
        return Message.class.isAssignableFrom(type);
    }

    @Override
    @SuppressWarnings("unchecked")
    public void writeTo(Object o, Class aClass, Type type, Annotation[] annotations, MediaType mediaType, MultivaluedMap headers, OutputStream outputStream) throws IOException, WebApplicationException {
        headers.add("x-proto-type", ((Message) o).getDescriptorForType().getFullName());
        outputStream.write(((Message) o).toByteArray());
    }

    @Override
    public long getSize(Object o, Class type, Type genericType, Annotation[] annotations, MediaType mediaType) {
        return ((Message) o).getSerializedSize();
    }
}

ProtobufJsonMessageReader.java

package com.github.alexjreid;

import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message.Builder;
import com.google.protobuf.util.JsonFormat;

import javax.ws.rs.Consumes;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyReader;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Type;


/**
 * Reads a stream of JSON and populates a protobuf, type specified in the signature of the annotated method.
 * Uses proto3's in-built support for JSON.
 */
@Provider
@Consumes("application/json")
public class ProtobufJsonMessageReader implements MessageBodyReader {
    private final static JsonFormat.TypeRegistry TYPE_REGISTRY = JsonFormat.TypeRegistry.newBuilder()
            //.add(MyProto.getDescriptor().getMessageTypes()) // Register types to be understood by 'Any'
            .build();

    private JsonFormat.Parser protoParser = null;

    public ProtobufJsonMessageReader() {
        protoParser = JsonFormat.parser()
                .usingTypeRegistry(TYPE_REGISTRY)
                .ignoringUnknownFields();
    }

    @Override
    public boolean isReadable(Class aClass, Type type, Annotation[] annotations, MediaType mediaType) {
        return mediaType.isCompatible(MediaType.APPLICATION_JSON_TYPE);
    }

    @Override
    @SuppressWarnings("unchecked")
    public Object readFrom(Class clazz, Type type, Annotation[] annotations, MediaType mediaType, MultivaluedMap headers, InputStream inputStream) throws IOException, WebApplicationException {
        try {
            Method newBuilder = clazz.getMethod("newBuilder");
            Builder builder = (Builder) newBuilder.invoke(type);
            protoParser.merge(new InputStreamReader(inputStream), builder);

            return builder.build();
        } catch (InvalidProtocolBufferException e) {
            throw new WebApplicationException(e);
        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException reflectionException) {
            throw new WebApplicationException(reflectionException);
        }
    }
}

ProtobufJsonMessageWriter.java

package com.github.alexjreid;

import com.google.protobuf.Message;
import com.google.protobuf.util.JsonFormat;

import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyWriter;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;

/**
 * Writes a protobuf message as JSON, with the type specified in annotated method signature, to the response.
 * Adds an HTTP header, x-proto-type, to denote the proto used.
 * Uses proto3's in-built support for JSON.
 */
@Provider
@Produces("application/json")
public class ProtobufJsonMessageWriter implements MessageBodyWriter {
    public boolean isWriteable(Class type, Type genericType, Annotation[] annotations,
                               MediaType mediaType) {
        return mediaType.isCompatible(MediaType.APPLICATION_JSON_TYPE);
    }

    @Override
    @SuppressWarnings("unchecked")
    public void writeTo(Object o, Class aClass, Type type, Annotation[] annotations, MediaType mediaType, MultivaluedMap headers, OutputStream outputStream) throws IOException, WebApplicationException {
        headers.add("x-proto-type", ((Message) o).getDescriptorForType().getFullName());
        outputStream.write(JsonFormat.printer().print((Message) o).getBytes());
    }

    @Override
    public long getSize(Object o, Class type, Type genericType, Annotation[] annotations, MediaType mediaType) {
        return -1;
    }
}