Skip to content

Instantly share code, notes, and snippets.

@mdrakiburrahman
Last active June 15, 2022 21:21
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mdrakiburrahman/9fcf398173e73693d31b71e8a945e680 to your computer and use it in GitHub Desktop.
Save mdrakiburrahman/9fcf398173e73693d31b71e8a945e680 to your computer and use it in GitHub Desktop.
OTEL daemonset and fluentforward not working
apiVersion: v1
kind: ConfigMap
metadata:
name: otel-collector-conf
namespace: azure-arc-data
labels:
app: opentelemetry
component: otel-collector-conf
data:
otel-collector-config: |
exporters:
kafka/laas-json:
protocol_version: 2.0.0
brokers: kafka-service.laas.svc:9092
topic: otel.fluentbit.json
encoding: otlp_json
kafka/laas-proto:
protocol_version: 2.0.0
brokers: kafka-service.laas.svc:9092
topic: otel.fluentbit.proto
encoding: otlp_proto
elasticsearch/laas:
endpoints:
- "http://elk-service.laas.svc:9200"
index: otel
logging:
logLevel: DEBUG
extensions:
zpages: null
k8s_observer:
auth_type: serviceAccount
node: ${K8S_NODE_NAME}
processors:
batch: null
k8sattributes:
filter:
node_from_env_var: ${K8S_NODE_NAME}
extract:
annotations:
- from: pod
key: AppCode-annotation
- from: pod
key: foo-annotation
labels:
- key: AppCode-label
- key: foo-label
metadata:
- k8s.namespace.name
- k8s.node.name
- k8s.pod.name
- k8s.pod.uid
- container.id
- container.image.name
- container.image.tag
pod_association:
- from: connection
name: ip
- from: resource_attribute
name: k8s.pod.name
- from: resource_attribute
name: ip
- from: resource_attribute
name: k8s.pod.ip
- from: resource_attribute
name: host.name
receivers:
fluentforward:
endpoint: 0.0.0.0:8006
service:
extensions:
- k8s_observer
- zpages
pipelines:
logs:
receivers:
- fluentforward
processors:
- attributes/k8sify
- k8sattributes
- batch
exporters:
- kafka/laas-json
- kafka/laas-proto
- elasticsearch/laas
#- logging
---
# Service account that allows Collector to interrogate K8s API
apiVersion: v1
kind: ServiceAccount
metadata:
name: opentelemetry-collector-agent
namespace: azure-arc-data
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: otel-collector-agent-role
rules:
- apiGroups: ["*"]
resources: ["*"]
verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: otel-collector-agent-role-binding
subjects:
- kind: ServiceAccount
name: opentelemetry-collector-agent
namespace: azure-arc-data
roleRef:
kind: ClusterRole
name: otel-collector-agent-role
apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: otel-agent-fluentbit
namespace: azure-arc-data
labels:
app: opentelemetry
component: otel-collector
spec:
selector:
matchLabels:
app: opentelemetry
component: otel-collector
template:
metadata:
labels:
app: opentelemetry
component: otel-collector
spec:
hostNetwork: false
nodeSelector:
kubernetes.io/os: linux
# Mount the service account that allows interrogating K8s API
serviceAccountName: opentelemetry-collector-agent
containers:
- image: otel/opentelemetry-collector-contrib:0.53.0
imagePullPolicy: Always
name: otel-collector
env:
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: K8S_NODE_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
- name: K8S_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: K8S_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: K8S_POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: K8S_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
resources:
limits:
cpu: 1
memory: 2Gi
requests:
cpu: 0.5
memory: 1Gi
ports:
- containerPort: 55679 # Default endpoint for ZPages.
- containerPort: 9411 # Default endpoint for Zipkin receiver.
- containerPort: 8888 # Default endpoint for querying metrics.
- name: fluentforward
hostPort: 18006
containerPort: 8006 # fluent.
volumeMounts:
- name: otel-collector-config-vol
mountPath: /etc/otelcol-contrib # For my demo image only - /etc/otel
- name: otel-cluster-config-vol
mountPath: /certs
volumes:
- configMap:
name: otel-collector-conf
items:
- key: otel-collector-config
path: config.yaml
name: otel-collector-config-vol
- configMap:
name: otel-cluster-config
items:
- key: ca.crt
path: ca.crt
- key: fluentbit-cert.pem
path: fluentbit-cert.pem
- key: fluentbit-key.pem
path: fluentbit-key.pem
name: otel-cluster-config-vol
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment