use nginx to proxy otel col,es,vmtrics,fluentbit

nginx-proxy.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-conf
  namespace: insight-system
data:
  proxy.conf: |
    proxy_redirect          off;
    proxy_set_header        Host            $host;
    proxy_set_header        X-Real-IP       $remote_addr;
    proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
    client_max_body_size    10m;
    client_body_buffer_size 128k;
    proxy_connect_timeout   90;
    proxy_send_timeout      90;
    proxy_read_timeout      90;
    proxy_buffers           32 4k;
  elasticsearch.conf: |
    server {
      listen 9200;
      server_name elasticsearch;

      # error logging
      error_log /var/log/nginx/elasticsearch_error.log;

      # authentication: elasticsearch
      #auth_basic "Elasticsearch Auth";
      #auth_basic_user_file /etc/nginx/.secrets_elasticsearch;

      location /es {

        proxy_http_version 1.1;
        proxy_set_header Connection "Keep-Alive";
        proxy_set_header Proxy-Connection "Keep-Alive";
        #proxy_set_header Authorization "";

        proxy_pass https://mcamel-common-es-cluster-masters-es-http.mcamel-system.svc:9200/;
        proxy_redirect off;
        proxy_buffering off;
        #proxy_redirect https://mcamel-common-es-cluster-masters-es-http.mcamel-system.svc:9200/ https://mcamel-common-es-cluster-masters-es-http.mcamel-system.svc:9200/;
        rewrite /es\/?(.*)$ /$1 break;
      }

      # ELB Health Checks
      location /status {
        root /usr/share/nginx/html/;
      }

    }

  vminsert.conf: |
    server {
      listen 8480;
      server_name vminsert;
      client_header_buffer_size 128k;
      large_client_header_buffers 4 128k;

      # error logging
      error_log /var/log/nginx/vminsert_error.log;

      location /metrics {

        proxy_http_version 1.1;
        proxy_set_header Connection "Keep-Alive";
        proxy_set_header Proxy-Connection "Keep-Alive";
        #proxy_set_header Authorization "";
        proxy_redirect off;
        proxy_buffering off;
        proxy_pass http://vminsert-insight-victoria-metrics-k8s-stack:8480/;
        #proxy_redirect http://vminsert-insight-victoria-metrics-k8s-stack:8480/ http://vminsert-insight-victoria-metrics-k8s-stack:8480/;
        rewrite /vm\/?(.*)$ /$1 break;
      }

      # ELB Health Checks
      location /status {
        root /usr/share/nginx/html/;
      }
    }

  otel_col.conf: |
    server {
      # http2 for Grpc proxy
      listen 4317 http2;
      server_name opentelemetry.collector;

      # error logging
      error_log /var/log/nginx/otel_col_error.log;

      # location / {
      #   grpc_pass grpc://insight-opentelemetry-collector:4317;
      # }

      # otlp exporter for traces
      location /opentelemetry.proto.collector.trace.v1.TraceService/Export {
        grpc_pass grpc://insight-opentelemetry-collector:4317;
      }

      # otlp exporter for metrics
      location /opentelemetry.proto.collector.metrics.v1.MetricsService/Export {
        grpc_pass grpc://insight-opentelemetry-collector:4317;
      }

      # # otlp exporter for logs
      # location /opentelemetry.proto.collector.logs.v1.LogsService/Export {
      #   grpc_pass grpc://insight-opentelemetry-collector:4317;
      # }

      # ELB Health Checks
      location /status {
        root /usr/share/nginx/html/;
      }
    }

  fluentbit_forward_tcp.conf: |
    # stream block allows proxying TCP and UDP traffic (should be placed on the same level as http block
    stream {
      server {
        listen 8006;
        proxy_pass insight-opentelemetry-collector:8006;
      }
    }
  nginx.conf: |
    user www-data;
    worker_processes auto;
    pid /run/nginx.pid;
    error_log /var/log/nginx/error.log;

    # events {
    #     worker_connections 1024;
    # }
    events {
      use epoll;
      worker_connections 16384;
    }

    http {
        # Basic Settings
        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        types_hash_max_size 2048;
        server_names_hash_bucket_size 128;

        include /etc/nginx/mime.types;
        include /etc/nginx/proxy.conf;
        default_type application/octet-stream;

      # Logging Settings
        log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                        '$status $body_bytes_sent "$http_referer" '
                        '"$http_user_agent" "$http_x_forwarded_for"';

        access_log /var/log/nginx/access.log main;

        # Gzip Settings
        gzip on;
        gzip_disable "msie6";

        server {
          listen 8080;
          location /metrics {
              stub_status;
          }
        }
        # insight Elasticsearch , vminsert and otel collector Configs
        include /etc/nginx/conf.d/elasticsearch.conf;
        include /etc/nginx/conf.d/vminsert.conf;
        include /etc/nginx/conf.d/otel_col.conf;
    }

    # stream block allows proxying TCP and UDP traffic (should be placed on the same level as http block
    # fluentbit forward to otel col 8006
    include /etc/nginx/conf.d/fluentbit_forward_tcp.conf;
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-demo-nginx
  namespace: insight-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: my-demo-nginx
  template:
    metadata:
      labels:
        app: my-demo-nginx
    spec:
      containers:
      - name: my-demo-nginx
        imagePullPolicy: IfNotPresent
        image: docker.m.daocloud.io/library/nginx:1.23.3
        ports:
        - containerPort: 9200
          name: es
        - containerPort: 8480
          name: vminsert
        - containerPort: 4317
          name: otelcol
        - containerPort: 8006
          name: fbfoward
        volumeMounts:
        - mountPath: /etc/nginx/nginx.conf # mount nginx-conf volumn to /etc/nginx
          name: nginx
          subPath: nginx.conf
        - mountPath: /etc/nginx/proxy.conf
          name: nginx
          subPath: proxy.conf
        - mountPath: /etc/nginx/conf.d/elasticsearch.conf
          name: nginx
          subPath: elasticsearch.conf
        - mountPath: /etc/nginx/conf.d/vminsert.conf
          name: nginx
          subPath: vminsert.conf
        - mountPath: /etc/nginx/conf.d/otel_col.conf
          name: nginx
          subPath: otel_col.conf
        - mountPath: /etc/nginx/conf.d/fluentbit_forward_tcp.conf
          name: nginx
          subPath: fluentbit_forward_tcp.conf
        - mountPath: /var/log/nginx
          name: log
      - name: nginx-exporter
        image: 'nginx/nginx-prometheus-exporter:0.10.0'
        #https://github.com/nginxinc/nginx-prometheus-exporter/blob/main/grafana/dashboard.json
        args:
          - '-nginx.scrape-uri=http://localhost:8080/metrics'
        resources:
          limits:
            memory: 128Mi
            cpu: 500m
        ports:
          - containerPort: 9113
            name: metric
      volumes:
      - name: nginx
        configMap:
          name: nginx-conf # place ConfigMap `nginx-conf` on /etc/nginx
          items:
            - key: nginx.conf
              path: nginx.conf
            - key: proxy.conf
              path: proxy.conf
            - key: elasticsearch.conf
              path: elasticsearch.conf
            - key: vminsert.conf
              path: vminsert.conf
            - key: otel_col.conf
              path: otel_col.conf
            - key: fluentbit_forward_tcp.conf
              path: fluentbit_forward_tcp.conf
            # - key: virtualhost.conf
            #   path: conf.d/virtualhost.conf # dig directory
      - name: log
        emptyDir: {}

---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service
  namespace: insight-system
  labels:
    app: my-demo-nginx
spec:
  type: NodePort
  ports:
  - port: 9200
    name: es
    targetPort: 9200
  - port: 8480
    name: vminsert
    targetPort: 8480
  - port: 4317
    name: otelcol
    targetPort: 4317
  - port: 8006
    name: fbforward
    targetPort: 8006
  - port: 9113
    name: metric
    targetPort: 9113
  selector:
    app: my-demo-nginx
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    operator.insight.io/managed-by: insight
    app: my-demo-nginx
  name: insight-nginx-monitor
spec:
  endpoints:
    - honorLabels: true
      port: metric
      #ptah: /metrics
  namespaceSelector:
    matchNames:
      - insight-system
  selector:
    matchLabels:
      app: my-demo-nginx