Kafka SSL 2-ways setup

kafka-ssl-2-ways.md

Kafka SSL 2-ways

Certificates

CA certificate

Create CA key and certificate

openssl req -new -x509 -newkey 'rsa:4096' -nodes -keyout 'ca.key' -out 'ca.crt' -days '365' -subj '/CN=My Root CA/OU=My Unit/O=My Company/C=US'

Import CA certificate into client and server truststores

keytool -importcert -keystore 'kafka.client.truststore.jks' -storepass 'changeit' -alias 'my-root-ca' -file 'ca.crt' -trustcacerts -noprompt
keytool -list -v -keystore 'kafka.client.truststore.jks' -storepass 'changeit'
keytool -importcert -keystore 'kafka.server.truststore.jks' -storepass 'changeit' -alias 'my-root-ca' -file 'ca.crt' -trustcacerts -noprompt
keytool -list -v -keystore 'kafka.server.truststore.jks' -storepass 'changeit'

Server certificate

Generate Kafka server key

kafka_broker_hostname=kafka.example.com
keytool -genkey -keystore 'kafka.server.keystore.jks' -storepass 'changeit' -keyalg 'RSA' -alias 'kafka-broker' -keypass 'changeit' -dname "CN=${kafka_broker_hostname}, OU=My Unit, O=My Company, C=US" -ext "SAN=DNS:${kafka_broker_hostname}" -validity '365' -noprompt
keytool -list -v -keystore 'kafka.server.keystore.jks' -storepass 'changeit'

Generate Kafka server CSR

keytool -certreq -keystore 'kafka.server.keystore.jks' -storepass 'changeit' -alias 'kafka-broker' -file 'kafka.server.csr'

Generate Kafka server certificate

openssl x509 -req -CA 'ca.crt' -CAkey 'ca.key' -in 'kafka.server.csr' -out 'kafka.server.crt' -days 365 -CAcreateserial

Import CA and Kafka server certificate into Kafka server keystore

keytool -importcert -keystore 'kafka.server.keystore.jks' -storepass 'changeit' -alias 'my-root-ca' -file 'ca.crt' -noprompt
keytool -importcert -keystore 'kafka.server.keystore.jks' -storepass 'changeit' -alias 'kafka-broker' -file 'kafka.server.crt' -noprompt

Link Kafka server truststore and keystore

sudo mkdir -vp /etc/ssl/private/
sudo ln -vsnf $PWD/kafka.server.truststore.jks /etc/ssl/private/
sudo ln -vsnf $PWD/kafka.server.keystore.jks /etc/ssl/private/

Client certificate (for SSL 2-ways only)

Generate Kafka client key

kafka_client_hostname=kafka.example.com
keytool -genkey -keystore 'kafka.client.keystore.jks' -storepass 'changeit' -keyalg 'RSA' -alias 'kafka-client' -keypass 'changeit' -dname "CN=${kafka_client_hostname}, OU=My Unit, O=My Company, C=US" -ext "SAN=DNS:${kafka_client_hostname}" -validity '365' -noprompt
keytool -list -v -keystore 'kafka.client.keystore.jks' -storepass 'changeit'

Generate Kafka client CSR

keytool -certreq -keystore 'kafka.client.keystore.jks' -storepass 'changeit' -alias 'kafka-client' -file 'kafka.client.csr'

Generate Kafka client certificate

openssl x509 -req -CA 'ca.crt' -CAkey 'ca.key' -in 'kafka.client.csr' -out 'kafka.client.crt' -days 365 -CAcreateserial

Import CA and Kafka certificate into Kafka client keystore

keytool -importcert -keystore 'kafka.client.keystore.jks' -storepass 'changeit' -alias 'my-root-ca' -file 'ca.crt' -noprompt
keytool -importcert -keystore 'kafka.client.keystore.jks' -storepass 'changeit' -alias 'kafka-client' -file 'kafka.client.crt' -noprompt

Link Kafka client truststore and keystore

sudo mkdir -vp /etc/ssl/private/
sudo ln -vsnf $PWD/kafka.client.truststore.jks /etc/ssl/private/
sudo ln -vsnf $PWD/kafka.client.keystore.jks /etc/ssl/private/

Start Zookeeper server

kafka/bin/zookeeper-server-start.sh kafka/config/zookeeper.properties

Configure Kafka server

Copy server.properties.

cp kafka/config/server.properties .

Add the following to server.properties:

# SSL
ssl.truststore.location=/etc/ssl/private/kafka.server.truststore.jks
ssl.truststore.password=changeit
ssl.keystore.location=/etc/ssl/private/kafka.server.keystore.jks
ssl.keystore.password=changeit
ssl.key.password=changeit
listeners=SSL://:9093
security.inter.broker.protocol=SSL

# For SSL 2-ways only
ssl.client.auth=required

Configure Kafka client

Create a client.properties file:

security.protocol=SSL
ssl.truststore.location=/etc/ssl/private/kafka.client.truststore.jks
ssl.truststore.password=changeit

# For SSL 2-ways only
ssl.keystore.location=/etc/ssl/private/kafka.client.keystore.jks
ssl.keystore.password=changeit

Start Kafka server

kafka/bin/kafka-server-start.sh server.properties

Test

Create topic

kafka_client_hostname=kafka.example.com
kafka/bin/kafka-topics.sh --bootstrap-server "${kafka_client_hostname}:9093" --command-config 'client.properties' --create --if-not-exists --topic 'my-topic'

Show topic

kafka_client_hostname=kafka.example.com
kafka/bin/kafka-topics.sh --bootstrap-server "${kafka_client_hostname}:9093" --command-config 'client.properties' --describe --topic 'my-topic'

Produce to topic

kafka_client_hostname=kafka.example.com
date | kafka/bin/kafka-console-producer.sh --bootstrap-server "${kafka_client_hostname}:9093" --producer.config 'client.properties' --topic 'my-topic'

Consume from topic

kafka_client_hostname=kafka.example.com
kafka/bin/kafka-console-consumer.sh --bootstrap-server "${kafka_client_hostname}:9093" --consumer.config 'client.properties' --topic 'my-topic' --from-beginning