Skip to content

Instantly share code, notes, and snippets.

@kingluo

kingluo/test.md

Last active May 4, 2023 10:06
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save kingluo/f1f2e8ab69be0a319c9082416a0d5232 to your computer and use it in GitHub Desktop.
Save kingluo/f1f2e8ab69be0a319c9082416a0d5232 to your computer and use it in GitHub Desktop.
Download ZIP
APISIX TLS/MTLS demo
Raw
test.md

Useful bash functions

# create self certificated cert and key
# new_ca <ca name>
# <domain> default value: "<ca name>.apisix.dev"
new_ca() {
    ca=${1:-ca}
    openssl req -new -out ${ca}.crt \
    -days 3650 -x509 -subj "/CN=${2:-${ca}.apisix.dev}" \
    -noenc -newkey rsa:2048 -keyout ${ca}.key \
    -addext "subjectAltName=DNS:${2:-${ca}.apisix.dev}"
}

# create cert and key signed by specific ca cert
# new_cert <cert name> <ca name> [<domain>]
# <ca name> default value: "ca"
# <domain> default value: "<cert name>.apisix.dev"
new_cert() {
    openssl req -new -out $1.csr \
    -noenc -newkey rsa:2048 -keyout $1.key \
    -subj "/CN=${3:-$1.apisix.dev}" \
    -addext "subjectAltName=DNS:${3:-$1.apisix.dev}"

    openssl x509 -days 3650 -req \
    -in $1.csr -out $1.crt \
    -CAkey ${2:-ca}.key -CA ${2:-ca}.crt -CAcreateserial
}

# create ECDSA cert and key signed by specific ca cert, with "-ecc" suffix
# new_cert_ecc <cert name> <ca name> [<domain>]
# <ca name> default value: "ca"
# <domain> default value: "<ca name>.apisix.dev"
new_cert_ecc() {
    openssl ecparam -genkey -name prime256v1 -out $1-ecc.key

    openssl req -key $1-ecc.key -new -out $1-ecc.csr \
    -subj "/CN=${3:-$1.apisix.dev}"

    openssl x509 -days 3650 -req \
    -in $1-ecc.csr -out $1-ecc.crt \
    -CAkey ${2:-ca}.key -CA ${2:-ca}.crt -CAcreateserial
}

# Admin API curl wrapper
# c [get|put|post|delete|...] <resource path> <any curl args> ...
c() {
    method=${1^^}
    resource=$2
    shift 2
    curl ${ADMIN_SCHEME:-http}://${ADMIN_IP:-127.0.0.1}:${ADMIN_PORT:-9180}/apisix/admin${resource} \
    -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X $method "$@"
}

# APISIX curl wrapper
# d [get|put|post|delete|...] <url> <any curl args> ...
d() {
    method=${1^^}
    path=$2
    shift 2
    curl ${NODE_SCHEME:-http}://${NODE_IP:-127.0.0.1}:${NODE_PORT:-9080}${path} -X $method "$@"
}

Downstream TLS/MTLS

new_ca
new_cert server
new_cert client

c put /routes/1 -i -d '
{
    "uri": "/*",
    "hosts": ["*.apisix.dev"],
    "methods": ["GET"],
    "upstream": {
        "type": "roundrobin",
        "nodes": {
            "httpbin.org": 1
        }
    }
}'

# tls
c put /ssls/1 -d '
{
    "cert": "'"$(<server.crt)"'",
    "key": "'"$(<server.key)"'",
    "snis": [
        "*.apisix.dev"
    ]
}'

NODE_SCHEME=https NODE_IP=server.apisix.dev NODE_PORT=9443 \
d get /uuid --resolve 'server.apisix.dev:9443:127.0.0.1' \
-v -k

# mtls
c put /ssls/1 -d '
{
    "cert": "'"$(<server.crt)"'",
    "key": "'"$(<server.key)"'",
    "snis": [
        "*.apisix.dev"
    ],
    "client": {
        "ca": "'"$(<ca.crt)"'",
        "depth": 10
    }
}'

NODE_SCHEME=https NODE_IP=server.apisix.dev NODE_PORT=9443 \
d get /uuid --resolve 'server.apisix.dev:9443:127.0.0.1' \
-v --cacert ca.crt --cert client.crt --key client.key

#
# RSA & ECDSA dual certs
#
c put /ssls/1 -d '
{
    "cert": "'"$(<server.crt)"'",
    "key": "'"$(<server.key)"'",
    "certs": ["'"$(<server-ecc.crt)"'"],
    "keys": ["'"$(<server-ecc.key)"'"],
    "snis": [
        "*.apisix.dev"
    ],
    "client": {
        "ca": "'"$(<ca.crt)"'",
        "depth": 10
    }
}'

openssl s_client \
-connect 127.0.0.1:9443 \
-servername server.apisix.dev \
-cipher ECDHE-RSA-AES256-GCM-SHA384 -tls1_2

openssl s_client \
-connect 127.0.0.1:9443 \
-servername server.apisix.dev \
-cipher ECDHE-ECDSA-AES256-GCM-SHA384 -tls1_2

curl --resolve 'server.apisix.dev:9443:127.0.0.1' \
https://server.apisix.dev:9443/uuid -v -k \
--cacert ca.crt --cert client.crt --key client.key \
--ciphers ECDHE-RSA-AES256-GCM-SHA384 --tlsv1.2 --tls-max 1.2

curl --resolve 'server.apisix.dev:9443:127.0.0.1' \
https://server.apisix.dev:9443/uuid -v -k \
--cacert ca.crt --cert client.crt --key client.key \
--ciphers ECDHE-ECDSA-AES256-GCM-SHA384 --tlsv1.2 --tls-max 1.2

Admin API MTLS

config.yaml

deployment:
  admin:
    https_admin: true
    admin_api_mtls:
      admin_ssl_ca_cert: "/opt/mtls_test/ca.crt"
      admin_ssl_cert: "/opt/mtls_test/server.crt"
      admin_ssl_cert_key: "/opt/mtls_test/server.key"

Test:

curl -v --resolve server.apisix.dev:9180:127.0.0.1 \
--cacert /opt/mtls_test/ca.crt \
https://server.apisix.dev:9180/apisix/admin/routes/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
    "uri": "/uuid",
    "upstream": {
        "type": "roundrobin",
        "nodes": {
            "httpbin.org": 1
        }
    }
}'
<html>
<head><title>400 No required SSL certificate was sent</title></head>
<body>
<center><h1>400 Bad Request</h1></center>
<center>No required SSL certificate was sent</center>
<hr><center>openresty</center>
<p><em>Powered by <a href="https://apisix.apache.org/">APISIX</a>.</em></p></body>
</html>

curl -v --resolve server.apisix.dev:9180:127.0.0.1 \
--cacert /opt/mtls_test/ca.crt \
--cert /opt/mtls_test/client.crt \
--key /opt/mtls_test/client.key \
https://server.apisix.dev:9180/apisix/admin/routes/1 \
-H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
{
    "uri": "/uuid",
    "upstream": {
        "type": "roundrobin",
        "nodes": {
            "httpbin.org": 1
        }
    }
}'

etcd MTLS

config.yaml

deployment:
  etcd:
    host:
      - "https://127.0.0.1:12379"
    tls:
      cert: /opt/mtls_test/etcd_client.crt
      key: /opt/mtls_test/etcd_client.key
      verify: true
      sni: etcd_server.apisix.dev
apisix:
  ssl:
    ssl_trusted_certificate: /opt/mtls_test/ca.crt

Test:

new_cert etcd_server
new_cert etcd_client

/tmp/etcd-download-test/etcd --name singleton \
--data-dir /tmp/etcd-download-test/singleton \
--client-cert-auth \
--trusted-ca-file=/opt/mtls_test/ca.crt \
--cert-file=/opt/mtls_test/etcd_server.crt \
--key-file=/opt/mtls_test/etcd_server.key \
--advertise-client-urls=https://127.0.0.1:12379 \
--listen-client-urls=https://127.0.0.1:12379

c put /routes/1 -i -d '
{
    "uri": "/get",
    "methods": ["GET"],
    "upstream": {
        "type": "roundrobin",
        "nodes": {
            "httpbin.org": 1
        }
    }
}'

c get /routes/1

d get /get

etcdctl get --prefix /apisix \
--endpoints=https://etcd_server.apisix.dev:12379 \
--cacert="/opt/mtls_test/ca.crt" \
--cert="/opt/mtls_test/etcd_client.crt" \
--key="/opt/mtls_test/etcd_client.key"

curl -v --resolve etcd_server.apisix.dev:12379:127.0.0.1 \
--cacert /opt/mtls_test/ca.crt \
--cert /opt/mtls_test/etcd_client.crt \
--key /opt/mtls_test/etcd_client.key \
https://etcd_server.apisix.dev:12379/v3/kv/put \
-X POST -d '{"key": "Zm9v", "value": "YmFy"}'

Upstream MTLS

config.yaml

apisix:
  ssl:
    ssl_trusted_certificate: /opt/mtls_test/mtls_ca.crt
nginx_config:
  http_server_configuration_snippet: |
    proxy_ssl_verify on;

go_mtls_http_server.go

package main

import (
    "crypto/tls"
    "crypto/x509"
    "fmt"
    "io/ioutil"
    "log"
    "net/http"
)

func main() {
    // set up handler to listen to root path
    handler := http.NewServeMux()
    handler.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) {
        log.Println("new request")
        fmt.Fprintf(writer, "hello world \n")
    })

    // load CA certificate file and add it to list of client CAs
    caCertFile, err := ioutil.ReadFile("/opt/mtls_test/mtls_ca.crt")
    if err != nil {
        log.Fatalf("error reading CA certificate: %v", err)
    }
    certPool := x509.NewCertPool()
    certPool.AppendCertsFromPEM(caCertFile)

    // serve on port 9090 of local host
    server := http.Server{
        Addr:    ":19090",
        Handler: handler,
        TLSConfig: &tls.Config{
            ClientAuth: tls.RequireAndVerifyClientCert,
            ClientCAs:  certPool,
            MinVersion: tls.VersionTLS12,
        },
    }

    // serve the endpoint with tls encryption
    if err := server.ListenAndServeTLS("/opt/mtls_test/mtls_server.crt", "/opt/mtls_test/mtls_server.key"); err != nil {
        log.Fatalf("error listening to port: %v", err)
    }
}

test:

new_ca mtls_ca
new_cert mtls_server mtls_ca
new_cert mtls_client mtls_ca

c put /routes/1 -d '{
    "methods": ["GET"],
    "uri": "/hello",
    "upstream": {
        "pass_host": "node",
        "scheme": "https",
        "type": "roundrobin",
        "tls": {
            "client_cert": "'"$(</opt/mtls_test/mtls_client.crt)"'",
            "client_key": "'"$(</opt/mtls_test/mtls_client.key)"'"
        },
        "nodes": {
            "mtls_server.apisix.dev:19090": 1
        }
    }
}'

d get /hello -i
HTTP/1.1 200 OK
Content-Type: text/plain; charset=utf-8
Content-Length: 13
Connection: keep-alive
Date: Tue, 03 Jan 2023 14:47:15 GMT
Server: APISIX/2.99.0

hello world

SNI

1679927643650

http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_name

`proxy_ssl_name $upstream_host;`

https://github.com/apache/apisix/blob/81149cd31567f6a86e33100fa7d09d7550073157/apisix/init.lua#L242

  • By default, pass_host is pass, then $upstream_host is $http_host.
  • if pass_host is node, then $upstream_host is the upstream.nodes[i].host.
  • if pass_host is rewrite, then $upstream_host is the upstream.upstream_host.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment