https://www.haproxy.com/documentation/kubernetes/latest/installation/community/kubernetes/
helm template haproxy haproxytech/kubernetes-ingress --set controller.logging.level=debug --version 1.12.3 --set controller.ingressClass=haproxy --create-namespace --namespace=haproxy
kubectl -n haproxy edit cm haproxy-kubernetes-ingress
apiVersion: v1
data:
maxconn: "15"
ssl-certificate: default/api-test-cert
ssl-passthrough: "false"
kind: ConfigMap
metadata:
..
Verify that the deployment has the ingressClass and others updated
kubectl -n haproxy get deployment
spec:
containers:
- args:
- --default-ssl-certificate=haproxy/haproxy-kubernetes-ingress-default-cert
- --configmap=haproxy/haproxy-kubernetes-ingress
- --default-backend-service=haproxy/haproxy-kubernetes-ingress-default-backend
- --ingress.class=haproxy
- --log=debug
Install MetalLB https://metallb.universe.tf/installation/ and Configure
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- 192.168.0.6-192.168.0.6
EOF
Change Service types of HaProxy Ingress Controller from NodePort to LoadBalancer so that MetalLB can give the IP
kubectl -n haproxy patch svc haproxy-kubernetes-ingress -p '{"spec": {"type": "LoadBalancer"}}'
# kubectl -n haproxy get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
haproxy-kubernetes-ingress LoadBalancer 10.103.252.25 192.168.0.6 80:30776/TCP,443:32622/TCP,1024:30531/TCP 3h11m
haproxy-kubernetes-ingress-default-backend ClusterIP None <none> 8080/TCP
Testing
kubectl create deployment sample-grpc --image=alexcpn/aa_sample_service_go:1.0
kubectl -n test expose deployment sample-grpc --port=50051
kubectl port-forward service/sample-grpc --address 0.0.0.0 50051:50051 --namespace test
Client
//the client part
package main
import (
"context"
"log"
"time"
pb "interfaces/test_server"
"google.golang.org/grpc"
_ "google.golang.org/grpc/credentials"
_ "crypto/tls"
)
const (
address = "samplegrpc.10.XX.XX.XX.io:50051"
)
func main() {
// Set up a connection to the server.
//config := &tls.Config{
// InsecureSkipVerify: true}
//conn, err := grpc.Dial(address, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")))
//conn, err := grpc.Dial(address, grpc.WithTransportCredentials(credentials.NewTLS(config))) //working on 443
conn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
log.Fatalf("did not connect: %v", err)
}
defer conn.Close()
c := pb.NewSearchServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
r, err := c.Search(ctx, &pb.SearchRequest{Query: "Protocol Buffer",EmailId: "alex.test@gmail.com"})
if err != nil {
log.Fatalf("could not execute search: %v", err)
}
//Lets validate the respose
res := r.Validate()
if res != nil {
log.Fatalf("Response validation failed: %v", err)
}
log.Printf("Greeting: %s", r.SearchResponse)
}
Test with Client -OK
test_client> go run .\client.go
2021/04/12 19:37:40 Greeting: Some Valid response from server
First with a GRPC service that talks HTTP2; Not that in frontend Configmap of Haproxy we have configured a certificate; and so TLS is enabled; There is no SSL Pass through by default. So TLS is terminated there
In backend we add the annotation haproxy.org/server-proto: "h2" to the ingress for HTTP2 support
HOST=samplegrpc.10.XX.XX.XX.nip.io
kubectl create -f - <<EOF
apiVersion: "networking.k8s.io/v1beta1"
kind: Ingress
metadata:
name: sample-grpc
namespace: test
annotations:
haproxy.org/ingress.class: "haproxy"
haproxy.org/server-proto: "h2"
spec:
rules:
- host: $HOST
http:
paths:
- backend:
serviceName: sample-grpc
servicePort: 50051
EOF
Slight Modification In Client
//the client part
package main
import (
"context"
"log"
"time"
pb "interfaces/test_server"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"crypto/tls"
)
const (
address = "samplegrpc.10xx.nip.io:443" // PORT is changed to 443, on 80 there is no TLS an no HTTP support;
)
func main() {
// Set up a connection to the server.
config := &tls.Config{
InsecureSkipVerify: true}
conn, err := grpc.Dial(address, grpc.WithTransportCredentials(credentials.NewTLS(config))) //working on 443
//conn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
log.Fatalf("did not connect: %v", err)
}
defer conn.Close()
c := pb.NewSearchServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
r, err := c.Search(ctx, &pb.SearchRequest{Query: "Protocol Buffer",EmailId: "alex.test@gmail.com"})
if err != nil {
log.Fatalf("could not execute search: %v", err)
}
//Lets validate the respose
res := r.Validate()
if res != nil {
log.Fatalf("Response validation failed: %v", err)
}
log.Printf("Greeting: %s", r.SearchResponse)
}
Full Code here https://github.com/alexcpn/grpc_templates/tree/master/grpc-go1.15
Test with client
golang_grpc_example\microservice_x\test_client> go run .\client.go
2021/04/08 15:25:29 Greeting: Some Valid response from server --> Works
kubectl create deployment nginx2 --image nginx:alpine -n test
kubectl -n test expose deployment nginx2 --port=80
service/nginx2 exposed
curl -kv --http3 https://nginx.10.XX.XX.XX.nip.io/
Edit NGINX to seprate HTTP1 and HTTP2 out
kubectl -n test-ingress exec -it nginx-745b4df97d-cklhv /bin/ash
# vi /etc/nginx/conf.d/default.conf
server {
listen 444 http2;
server_name localhost2;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
}
server {
listen 80;
listen [::]:80;
server_name localhost;
#charset koi8-r;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
Test the configuration and reload NGINX in POd
nginx -t
nginx -s reload
Exit out of Pod and test in the cluster using Service ClusteIP
Port 80 Only HTTP/1
curl --http2-prior-knowledge -v 10.103.151.173:80 | NOT OK |Nginx at port 80 does not work with HTTP2 only |Perhaps, peer does not support HTTP/2 properly.
Port 443 Only HTTP/2
curl --http2-prior-knowledge -v 10.103.151.173:443 | OK | HTTP2 at 443 works (cluster IP) |* Using HTTP2, server supports multi-use
Edit the NGINX Service to seprate HTTP1.1 and HTTP2 Out. Nginx cannot support both at same port
- name: http
port: 80
protocol: TCP
targetPort: 80
- name: http2
port: 443
protocol: TCP
targetPort: 444
selector:
Create an Ingress for HTTP1.1 and another for HTTP 2
HOST=nginxhttp1.xx.nip.io
kubectl create -f - <<EOF
apiVersion: "networking.k8s.io/v1beta1"
kind: Ingress
metadata:
name: nginxhttp1
namespace: test-ingress
annotations:
haproxy.org/ingress.class: "haproxy"
spec:
rules:
- host: $HOST
http:
paths:
- backend:
serviceName: nginx
servicePort: 80
path: /
EOF
For HTTP2 add haproxy.org/server-proto: "h2" to ingress
HOST=nginxhttp2.xxx.nip.io
kubectl create -f - <<EOF
apiVersion: "networking.k8s.io/v1beta1"
kind: Ingress
metadata:
name: nginxhttp2
namespace: test-ingress
annotations:
haproxy.org/ingress.class: "haproxy"
haproxy.org/server-proto: "h2"
spec:
rules:
- host: $HOST
http:
paths:
- backend:
serviceName: nginx
servicePort: 443
path: /
EOF
Test with Browser of with Curl -
HTTP1 Works
curl -kv --http2 https://nginxhttp1.xxx.nip.io:443/
HTTP2 - Works
curl -kv --http2 https://nginxhttp2.xx.nip.io:443/
root@k8s-storage-1:~# curl -kv --http2 https://nginxhttp1.xxxx.nip.io:443/
* Trying 10.131.228.112...
* TCP_NODELAY set
* Connected to nginxhttp1.xxx.nip.io xxx8.112) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
* CAfile: /etc/ssl/certs/ca-certificates.crt
CApath: /etc/ssl/certs
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Unknown (8):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Certificate (11):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Finished (20):
* TLSv1.3 (OUT), TLS change cipher, Client hello (1):
* TLSv1.3 (OUT), TLS Unknown, Certificate Status (22):
* TLSv1.3 (OUT), TLS handshake, Finished (20):
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384
* ALPN, server accepted to use h2
* Server certificate:
* subject: C=US; ST=Ohio; L=Columbus; O=MyCompany; CN=api.test.com
* start date: Apr 6 10:30:35 2021 GMT
* expire date: Apr 6 10:30:35 2022 GMT
* issuer: C=US; ST=Ohio; L=Columbus; O=MyCompany; CN=api.test.com
* SSL certificate verify result: self signed certificate (18), continuing anyway.
* Using HTTP2, server supports multi-use
* Connection state changed (HTTP/2 confirmed)
* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* Using Stream ID: 1 (easy handle 0x5616778ab580)
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
> GET / HTTP/2
> Host: nginxhttp1.10.131.228.112.nip.io
> User-Agent: curl/7.58.0
> Accept: */*
>
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
* TLSv1.3 (IN), TLS Unknown, Unknown (23):
* Connection state changed (MAX_CONCURRENT_STREAMS updated)!
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* TLSv1.3 (IN), TLS Unknown, Unknown (23):
< HTTP/2 200
< server: nginx/1.19.9
< date: Mon, 12 Apr 2021 14:49:29 GMT
< content-type: text/html
< content-length: 612
< last-modified: Tue, 30 Mar 2021 15:21:56 GMT
< etag: "60634214-264"
< accept-ranges: bytes
<
* TLSv1.3 (IN), TLS Unknown, Unknown (23):
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
* Connection #0 to host nginxhttp1.xxx.nip.io left intact
root@k8s-storage-1:~# curl -kv --http2 https://nginxhttp2.xxx2.nip.io:443/
* Trying 10.131.228.112...
* TCP_NODELAY set
* Connected to nginxhttp2.xxx.nip.io xxx) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
* CAfile: /etc/ssl/certs/ca-certificates.crt
CApath: /etc/ssl/certs
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
* TLSv1.3 (IN), TLS handshake, Server hello (2):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Unknown (8):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Certificate (11):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Finished (20):
* TLSv1.3 (OUT), TLS change cipher, Client hello (1):
* TLSv1.3 (OUT), TLS Unknown, Certificate Status (22):
* TLSv1.3 (OUT), TLS handshake, Finished (20):
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384
* ALPN, server accepted to use h2
* Server certificate:
* subject: C=US; ST=Ohio; L=Columbus; O=MyCompany; CN=api.test.com
* start date: Apr 6 10:30:35 2021 GMT
* expire date: Apr 6 10:30:35 2022 GMT
* issuer: C=US; ST=Ohio; L=Columbus; O=MyCompany; CN=api.test.com
* SSL certificate verify result: self signed certificate (18), continuing anyway.
* Using HTTP2, server supports multi-use
* Connection state changed (HTTP/2 confirmed)
* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* Using Stream ID: 1 (easy handle 0x55a8eb157580)
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
> GET / HTTP/2
> Host: nginxhttp2.10.131.228.112.nip.io
> User-Agent: curl/7.58.0
> Accept: */*
>
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
* TLSv1.3 (IN), TLS Unknown, Certificate Status (22):
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
* TLSv1.3 (IN), TLS Unknown, Unknown (23):
* Connection state changed (MAX_CONCURRENT_STREAMS updated)!
* TLSv1.3 (OUT), TLS Unknown, Unknown (23):
* TLSv1.3 (IN), TLS Unknown, Unknown (23):
< HTTP/2 200
< server: nginx/1.19.9
< date: Mon, 12 Apr 2021 14:49:39 GMT
< content-type: text/html
< content-length: 612
< last-modified: Tue, 30 Mar 2021 15:21:56 GMT
< etag: "60634214-264"
< accept-ranges: bytes
<
* TLSv1.3 (IN), TLS Unknown, Unknown (23):
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
* Connection #0 to host nginxhttp2.xxx.nip.io left intact