Skip to content

Instantly share code, notes, and snippets.

@mandrean
Created October 28, 2016 13:09
Show Gist options
  • Save mandrean/91935a2783a45541b3fada7d753bd46a to your computer and use it in GitHub Desktop.
Save mandrean/91935a2783a45541b3fada7d753bd46a to your computer and use it in GitHub Desktop.
kube-aws fluentd/elastic search/kibana logging for kubernetes
# ...
write_files:
- path: /opt/bin/install-kube-system
permissions: 0700
owner: root:root
content: |
#!/bin/bash -e
# ...
# Custom replication controllers
for manifest in {kibana-logging,elasticsearch-logging}-rc.json;do
/usr/bin/curl -H "Content-Type: application/json" -XPOST \
-d @"/srv/kubernetes/manifests/$manifest" \
"http://127.0.0.1:8080/api/v1/namespaces/kube-system/replicationcontrollers"
done
# Custom services
for manifest in {elasticsearch-logging,kibana-logging}-svc.json;do
/usr/bin/curl -H "Content-Type: application/json" -XPOST \
-d @"/srv/kubernetes/manifests/$manifest" \
"http://127.0.0.1:8080/api/v1/namespaces/kube-system/services"
done
# Custom daemon sets
/usr/bin/curl -H "Content-Type: application/json" -XPOST \
-d @"/srv/kubernetes/manifests/fluentd-cloud-logging-ds.json" \
"http://127.0.0.1:8080/apis/extensions/v1beta1/namespaces/kube-system/daemonsets"
# ...
- path: /srv/kubernetes/manifests/elasticsearch-logging-rc.json
content: |
{
"apiVersion": "v1",
"kind": "ReplicationController",
"metadata": {
"name": "elasticsearch-logging-v1",
"namespace": "kube-system",
"labels": {
"k8s-app": "elasticsearch-logging",
"version": "v1",
"kubernetes.io/cluster-service": "true"
}
},
"spec": {
"replicas": 2,
"selector": {
"k8s-app": "elasticsearch-logging",
"version": "v1"
},
"template": {
"metadata": {
"labels": {
"k8s-app": "elasticsearch-logging",
"version": "v1",
"kubernetes.io/cluster-service": "true"
}
},
"spec": {
"containers": [
{
"image": "gcr.io/google_containers/elasticsearch:1.8",
"name": "elasticsearch-logging",
"resources": {
"limits": {
"cpu": "100m"
},
"requests": {
"cpu": "100m"
}
},
"ports": [
{
"containerPort": 9200,
"name": "db",
"protocol": "TCP"
},
{
"containerPort": 9300,
"name": "transport",
"protocol": "TCP"
}
],
"volumeMounts": [
{
"name": "es-persistent-storage",
"mountPath": "/data"
}
]
}
],
"volumes": [
{
"name": "es-persistent-storage",
"emptyDir": {
}
}
]
}
}
}
}
- path: /srv/kubernetes/manifests/kibana-logging-rc.json
content: |
{
"apiVersion": "v1",
"kind": "ReplicationController",
"metadata": {
"name": "kibana-logging-v1",
"namespace": "kube-system",
"labels": {
"k8s-app": "kibana-logging",
"version": "v1",
"kubernetes.io/cluster-service": "true"
}
},
"spec": {
"replicas": 1,
"selector": {
"k8s-app": "kibana-logging",
"version": "v1"
},
"template": {
"metadata": {
"labels": {
"k8s-app": "kibana-logging",
"version": "v1",
"kubernetes.io/cluster-service": "true"
}
},
"spec": {
"containers": [
{
"name": "kibana-logging",
"image": "gcr.io/google_containers/kibana:1.3",
"resources": {
"limits": {
"cpu": "100m"
},
"requests": {
"cpu": "100m"
}
},
"env": [
{
"name": "ELASTICSEARCH_URL",
"value": "http://elasticsearch-logging:9200"
}
],
"ports": [
{
"containerPort": 5601,
"name": "ui",
"protocol": "TCP"
}
]
}
]
}
}
}
}
- path: /srv/kubernetes/manifests/elasticsearch-logging-svc.json
content: |
{
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": "elasticsearch-logging",
"namespace": "kube-system",
"labels": {
"k8s-app": "elasticsearch-logging",
"kubernetes.io/cluster-service": "true",
"kubernetes.io/name": "Elasticsearch"
}
},
"spec": {
"ports": [
{
"port": 9200,
"protocol": "TCP",
"targetPort": "db"
}
],
"selector": {
"k8s-app": "elasticsearch-logging"
}
}
}
- path: /srv/kubernetes/manifests/kibana-logging-svc.json
content: |
{
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": "kibana-logging",
"namespace": "kube-system",
"labels": {
"k8s-app": "kibana-logging",
"kubernetes.io/cluster-service": "true",
"kubernetes.io/name": "Kibana"
}
},
"spec": {
"ports": [
{
"port": 5601,
"protocol": "TCP",
"targetPort": "ui"
}
],
"selector": {
"k8s-app": "kibana-logging"
}
}
}
- path: /srv/kubernetes/manifests/fluentd-cloud-logging-ds.json
content: |
{
"apiVersion": "extensions/v1beta1",
"kind": "DaemonSet",
"metadata": {
"name": "fluent-elasticsearch",
"namespace": "kube-system",
"labels": {
"k8s-app": "fluentd-logging"
}
},
"spec": {
"template": {
"metadata": {
"name": "fluentd-elasticsearch",
"namespace": "kube-system",
"labels": {
"k8s-app": "fluentd-logging"
}
},
"spec": {
"containers": [
{
"name": "fluentd-elasticsearch",
"image": "gcr.io/google_containers/fluentd-elasticsearch:1.15",
"resources": {
"limits": {
"memory": "200Mi"
},
"requests": {
"cpu": "100m",
"memory": "200Mi"
}
},
"volumeMounts": [
{
"name": "varlog",
"mountPath": "/var/log"
},
{
"name": "varlibdockercontainers",
"mountPath": "/var/lib/docker/containers",
"readOnly": true
}
]
}
],
"terminationGracePeriodSeconds": 30,
"volumes": [
{
"name": "varlog",
"hostPath": {
"path": "/var/log"
}
},
{
"name": "varlibdockercontainers",
"hostPath": {
"path": "/var/lib/docker/containers"
}
}
]
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment