Skip to content

Instantly share code, notes, and snippets.

@holms

holms/debug.md

Last active Nov 21, 2019
Embed
What would you like to do?
ElasticSearch helm chart probereadiness problem debug

Using DOKS with droplet size of s-2vcpu-4gb, with own namespace and own node pool of 3 nodes.

helm get elasticsearch

REVISION: 1
RELEASED: Thu Nov 21 15:10:47 2019
CHART: elasticsearch-7.4.1
USER-SUPPLIED VALUES:
esJavaOpts: -Xmx1g -Xms1g
extraInitContainers: |
  - name: create
    image: busybox:1.28
    command: ['mkdir', '-p', '/usr/share/elasticsearch/data/nodes/']
    securityContext:
      runAsUser: 0
    volumeMounts:
     - mountPath: /usr/share/elasticsearch/data
       name: elasticsearch-master
  - name: file-permissions
    image: busybox:1.28
    command: ['chown', '-R', '1000:1000', '/usr/share/elasticsearch/']
    securityContext:
       runAsUser: 0
    volumeMounts:
     - mountPath: /usr/share/elasticsearch/data
       name: elasticsearch-master
nodeSelector:
  doks.digitalocean.com/node-pool: elasticsearch
resources:
  limits:
    cpu: 1000m
    memory: 2G
  requests:
    cpu: 100m
    memory: 2G
volumeClaimTemplate:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 20G
  storageClassName: do-block-storage

COMPUTED VALUES:
antiAffinity: hard
antiAffinityTopologyKey: kubernetes.io/hostname
clusterHealthCheckParams: wait_for_status=green&timeout=1s
clusterName: elasticsearch
esConfig: {}
esJavaOpts: -Xmx1g -Xms1g
esMajorVersion: ""
extraEnvs: []
extraInitContainers: |
  - name: create
    image: busybox:1.28
    command: ['mkdir', '-p', '/usr/share/elasticsearch/data/nodes/']
    securityContext:
      runAsUser: 0
    volumeMounts:
     - mountPath: /usr/share/elasticsearch/data
       name: elasticsearch-master
  - name: file-permissions
    image: busybox:1.28
    command: ['chown', '-R', '1000:1000', '/usr/share/elasticsearch/']
    securityContext:
       runAsUser: 0
    volumeMounts:
     - mountPath: /usr/share/elasticsearch/data
       name: elasticsearch-master
extraVolumeMounts: ""
extraVolumes: ""
fsGroup: ""
fullnameOverride: ""
httpPort: 9200
image: docker.elastic.co/elasticsearch/elasticsearch
imagePullPolicy: IfNotPresent
imagePullSecrets: []
imageTag: 7.4.1
ingress:
  annotations: {}
  enabled: false
  hosts:
  - chart-example.local
  path: /
  tls: []
initResources: {}
keystore: []
labels: {}
lifecycle: {}
masterService: ""
masterTerminationFix: false
maxUnavailable: 1
minimumMasterNodes: 2
nameOverride: ""
networkHost: 0.0.0.0
nodeAffinity: {}
nodeGroup: master
nodeSelector:
  doks.digitalocean.com/node-pool: elasticsearch
persistence:
  annotations: {}
  enabled: true
podAnnotations: {}
podManagementPolicy: Parallel
podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000
podSecurityPolicy:
  create: false
  name: ""
  spec:
    fsGroup:
      rule: RunAsAny
    privileged: true
    runAsUser:
      rule: RunAsAny
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
    - secret
    - configMap
    - persistentVolumeClaim
priorityClassName: ""
protocol: http
rbac:
  create: false
  serviceAccountName: ""
readinessProbe:
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 3
  timeoutSeconds: 5
replicas: 3
resources:
  limits:
    cpu: 1000m
    memory: 2G
  requests:
    cpu: 100m
    memory: 2G
roles:
  data: "true"
  ingest: "true"
  master: "true"
schedulerName: ""
secretMounts: []
securityContext:
  capabilities:
    drop:
    - ALL
  runAsNonRoot: true
  runAsUser: 1000
service:
  annotations: {}
  httpPortName: http
  nodePort: ""
  transportPortName: transport
  type: ClusterIP
sidecarResources: {}
sysctlInitContainer:
  enabled: true
sysctlVmMaxMapCount: 262144
terminationGracePeriod: 120
tolerations: []
transportPort: 9300
updateStrategy: RollingUpdate
volumeClaimTemplate:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 20G
  storageClassName: do-block-storage

HOOKS:
---
# elasticsearch-sxuwh-test
apiVersion: v1
kind: Pod
metadata:
  name: "elasticsearch-sxuwh-test"
  annotations:
    "helm.sh/hook": test-success
spec:
  containers:
  - name: "elasticsearch-elexk-test"
    image: "docker.elastic.co/elasticsearch/elasticsearch:7.4.1"
    command:
      - "sh"
      - "-c"
      - |
        #!/usr/bin/env bash -e
        curl -XGET --fail 'elasticsearch-master:9200/_cluster/health?wait_for_status=green&timeout=1s'
  restartPolicy: Never
MANIFEST:

---
# Source: elasticsearch/templates/poddisruptionbudget.yaml
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: "elasticsearch-master-pdb"
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: "elasticsearch-master"
---
# Source: elasticsearch/templates/service.yaml
kind: Service
apiVersion: v1
metadata:
  name: elasticsearch-master
  labels:
    heritage: "Tiller"
    release: "elasticsearch"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  annotations:
    {}
    
spec:
  type: ClusterIP
  selector:
    heritage: "Tiller"
    release: "elasticsearch"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  ports:
  - name: http
    protocol: TCP
    port: 9200
  - name: transport
    protocol: TCP
    port: 9300
---
# Source: elasticsearch/templates/service.yaml
kind: Service
apiVersion: v1
metadata:
  name: elasticsearch-master-headless
  labels:
    heritage: "Tiller"
    release: "elasticsearch"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve
  # Create endpoints also if the related pod isn't ready
  publishNotReadyAddresses: true
  selector:
    app: "elasticsearch-master"
  ports:
  - name: http
    port: 9200
  - name: transport
    port: 9300
---
# Source: elasticsearch/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch-master
  labels:
    heritage: "Tiller"
    release: "elasticsearch"
    chart: "elasticsearch"
    app: "elasticsearch-master"
  annotations:
    esMajorVersion: "7"
spec:
  serviceName: elasticsearch-master-headless
  selector:
    matchLabels:
      app: "elasticsearch-master"
  replicas: 3
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates:
  - metadata:
      name: elasticsearch-master
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 20G
      storageClassName: do-block-storage
      
  template:
    metadata:
      name: "elasticsearch-master"
      labels:
        heritage: "Tiller"
        release: "elasticsearch"
        chart: "elasticsearch"
        app: "elasticsearch-master"
      annotations:
        
    spec:
      securityContext:
        fsGroup: 1000
        runAsUser: 1000
        
      nodeSelector:
        doks.digitalocean.com/node-pool: elasticsearch
        
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - "elasticsearch-master"
            topologyKey: kubernetes.io/hostname
      terminationGracePeriodSeconds: 120
      volumes:
      initContainers:
      - name: configure-sysctl
        securityContext:
          runAsUser: 0
          privileged: true
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.4.1"
        command: ["sysctl", "-w", "vm.max_map_count=262144"]
        resources:
          {}
          

      - name: create
        image: busybox:1.28
        command: ['mkdir', '-p', '/usr/share/elasticsearch/data/nodes/']
        securityContext:
          runAsUser: 0
        volumeMounts:
         - mountPath: /usr/share/elasticsearch/data
           name: elasticsearch-master
      - name: file-permissions
        image: busybox:1.28
        command: ['chown', '-R', '1000:1000', '/usr/share/elasticsearch/']
        securityContext:
           runAsUser: 0
        volumeMounts:
         - mountPath: /usr/share/elasticsearch/data
           name: elasticsearch-master
      
      containers:
      - name: "elasticsearch"
        securityContext:
          capabilities:
            drop:
            - ALL
          runAsNonRoot: true
          runAsUser: 1000
          
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.4.1"
        imagePullPolicy: "IfNotPresent"
        readinessProbe:
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 3
          timeoutSeconds: 5
          
          exec:
            command:
              - sh
              - -c
              - |
                #!/usr/bin/env bash -e
                # If the node is starting up wait for the cluster to be ready (request params: 'wait_for_status=green&timeout=1s' )
                # Once it has started only check that the node itself is responding
                START_FILE=/tmp/.es_start_file

                http () {
                    local path="${1}"
                    if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then
                      BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}"
                    else
                      BASIC_AUTH=''
                    fi
                    curl -XGET -s -k --fail ${BASIC_AUTH} http://127.0.0.1:9200${path}
                }

                if [ -f "${START_FILE}" ]; then
                    echo 'Elasticsearch is already running, lets check the node is healthy'
                    http "/"
                else
                    echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )'
                    if http "/_cluster/health?wait_for_status=green&timeout=1s" ; then
                        touch ${START_FILE}
                        exit 0
                    else
                        echo 'Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )'
                        exit 1
                    fi
                fi
        ports:
        - name: http
          containerPort: 9200
        - name: transport
          containerPort: 9300
        resources:
          limits:
            cpu: 1000m
            memory: 2G
          requests:
            cpu: 100m
            memory: 2G
          
        env:
          - name: node.name
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: cluster.initial_master_nodes
            value: "elasticsearch-master-0,elasticsearch-master-1,elasticsearch-master-2,"
          - name: discovery.seed_hosts
            value: "elasticsearch-master-headless"
          - name: cluster.name
            value: "elasticsearch"
          - name: network.host
            value: "0.0.0.0"
          - name: ES_JAVA_OPTS
            value: "-Xmx1g -Xms1g"
          - name: node.data
            value: "true"
          - name: node.ingest
            value: "true"
          - name: node.master
            value: "true"
        volumeMounts:
          - name: "elasticsearch-master"
            mountPath: /usr/share/elasticsearch/data

Here's kubectl describe:

Name:           elasticsearch-master-0
Namespace:      logging
Priority:       0
Node:           elasticsearch-eqmy/10.135.120.68
Start Time:     Thu, 21 Nov 2019 15:10:52 +0200
Labels:         app=elasticsearch-master
                chart=elasticsearch
                controller-revision-hash=elasticsearch-master-75b5858989
                heritage=Tiller
                release=elasticsearch
                statefulset.kubernetes.io/pod-name=elasticsearch-master-0
Annotations:    <none>
Status:         Running
IP:             10.244.3.26
IPs:            <none>
Controlled By:  StatefulSet/elasticsearch-master
Init Containers:
  configure-sysctl:
    Container ID:  docker://866bd8660cc29c54f3c24e5fca1d2bf9a761431a40af1c5e71343cae57b2a66e
    Image:         docker.elastic.co/elasticsearch/elasticsearch:7.4.1
    Image ID:      docker-pullable://docker.elastic.co/elasticsearch/elasticsearch@sha256:88c2ee30115f378b8f7e66662ec26bca0c8778c69096bee6b161128ce833585f
    Port:          <none>
    Host Port:     <none>
    Command:
      sysctl
      -w
      vm.max_map_count=262144
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Thu, 21 Nov 2019 15:11:02 +0200
      Finished:     Thu, 21 Nov 2019 15:11:02 +0200
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-tkg5b (ro)
  create:
    Container ID:  docker://06580d0f93e4a96e3f1cd0b22d43fb93b314bccc75c52a61c6265c7c6fcd4e97
    Image:         busybox:1.28
    Image ID:      docker-pullable://busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47
    Port:          <none>
    Host Port:     <none>
    Command:
      mkdir
      -p
      /usr/share/elasticsearch/data/nodes/
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Thu, 21 Nov 2019 15:11:03 +0200
      Finished:     Thu, 21 Nov 2019 15:11:03 +0200
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /usr/share/elasticsearch/data from elasticsearch-master (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-tkg5b (ro)
  file-permissions:
    Container ID:  docker://45c5472f995bbb975773767a6f91368e213a756e2fbd6023b5b1522b15372cb0
    Image:         busybox:1.28
    Image ID:      docker-pullable://busybox@sha256:141c253bc4c3fd0a201d32dc1f493bcf3fff003b6df416dea4f41046e0f37d47
    Port:          <none>
    Host Port:     <none>
    Command:
      chown
      -R
      1000:1000
      /usr/share/elasticsearch/
    State:          Terminated
      Reason:       Completed
      Exit Code:    0
      Started:      Thu, 21 Nov 2019 15:11:04 +0200
      Finished:     Thu, 21 Nov 2019 15:11:04 +0200
    Ready:          True
    Restart Count:  0
    Environment:    <none>
    Mounts:
      /usr/share/elasticsearch/data from elasticsearch-master (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-tkg5b (ro)
Containers:
  elasticsearch:
    Container ID:   docker://e5d62ae6a4f8e886745da430ddb7f9c9e0fa6673400ba86af4a3cd377edf88d2
    Image:          docker.elastic.co/elasticsearch/elasticsearch:7.4.1
    Image ID:       docker-pullable://docker.elastic.co/elasticsearch/elasticsearch@sha256:88c2ee30115f378b8f7e66662ec26bca0c8778c69096bee6b161128ce833585f
    Ports:          9200/TCP, 9300/TCP
    Host Ports:     0/TCP, 0/TCP
    State:          Running
      Started:      Thu, 21 Nov 2019 15:11:05 +0200
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     1
      memory:  2G
    Requests:
      cpu:      100m
      memory:   2G
    Readiness:  exec [sh -c #!/usr/bin/env bash -e
# If the node is starting up wait for the cluster to be ready (request params: 'wait_for_status=green&timeout=1s' )
# Once it has started only check that the node itself is responding
START_FILE=/tmp/.es_start_file

http () {
    local path="${1}"
    if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then
      BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}"
    else
      BASIC_AUTH=''
    fi
    curl -XGET -s -k --fail ${BASIC_AUTH} http://127.0.0.1:9200${path}
}

if [ -f "${START_FILE}" ]; then
    echo 'Elasticsearch is already running, lets check the node is healthy'
    http "/"
else
    echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )'
    if http "/_cluster/health?wait_for_status=green&timeout=1s" ; then
        touch ${START_FILE}
        exit 0
    else
        echo 'Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )'
        exit 1
    fi
fi
] delay=10s timeout=5s period=10s #success=3 #failure=3
    Environment:
      node.name:                     elasticsearch-master-0 (v1:metadata.name)
      cluster.initial_master_nodes:  elasticsearch-master-0,elasticsearch-master-1,elasticsearch-master-2,
      discovery.seed_hosts:          elasticsearch-master-headless
      cluster.name:                  elasticsearch
      network.host:                  0.0.0.0
      ES_JAVA_OPTS:                  -Xmx1g -Xms1g
      node.data:                     true
      node.ingest:                   true
      node.master:                   true
    Mounts:
      /usr/share/elasticsearch/data from elasticsearch-master (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-tkg5b (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             True 
  ContainersReady   True 
  PodScheduled      True 
Volumes:
  elasticsearch-master:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  elasticsearch-master-elasticsearch-master-0
    ReadOnly:   false
  default-token-tkg5b:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-tkg5b
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  doks.digitalocean.com/node-pool=elasticsearch
Tolerations:     node.kubernetes.io/not-ready:NoExecute for 300s
                 node.kubernetes.io/unreachable:NoExecute for 300s
Events:
  Type     Reason                  Age                    From                         Message
  ----     ------                  ----                   ----                         -------
  Warning  FailedScheduling        3m52s (x3 over 3m53s)  default-scheduler            pod has unbound immediate PersistentVolumeClaims (repeated 3 times)
  Normal   Scheduled               3m49s                  default-scheduler            Successfully assigned logging/elasticsearch-master-0 to elasticsearch-eqmy
  Normal   SuccessfulAttachVolume  3m42s                  attachdetach-controller      AttachVolume.Attach succeeded for volume "pvc-43d3e4a1-0b71-4dff-923e-121d85459fe7"
  Normal   Pulled                  3m39s                  kubelet, elasticsearch-eqmy  Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
  Normal   Created                 3m39s                  kubelet, elasticsearch-eqmy  Created container configure-sysctl
  Normal   Started                 3m39s                  kubelet, elasticsearch-eqmy  Started container configure-sysctl
  Normal   Pulled                  3m38s                  kubelet, elasticsearch-eqmy  Container image "busybox:1.28" already present on machine
  Normal   Created                 3m38s                  kubelet, elasticsearch-eqmy  Created container create
  Normal   Started                 3m38s                  kubelet, elasticsearch-eqmy  Started container create
  Normal   Pulled                  3m37s                  kubelet, elasticsearch-eqmy  Container image "busybox:1.28" already present on machine
  Normal   Created                 3m37s                  kubelet, elasticsearch-eqmy  Created container file-permissions
  Normal   Started                 3m37s                  kubelet, elasticsearch-eqmy  Started container file-permissions
  Normal   Pulled                  3m36s                  kubelet, elasticsearch-eqmy  Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
  Normal   Created                 3m36s                  kubelet, elasticsearch-eqmy  Created container elasticsearch
  Normal   Started                 3m36s                  kubelet, elasticsearch-eqmy  Started container elasticsearch
  Warning  Unhealthy               3m3s (x3 over 3m23s)   kubelet, elasticsearch-eqmy  Readiness probe failed: Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )

And kubectl get events

LAST SEEN   TYPE      REASON                   OBJECT                       MESSAGE
4m19s       Warning   FailedScheduling         pod/elasticsearch-master-0   pod has unbound immediate PersistentVolumeClaims (repeated 3 times)
4m16s       Normal    Scheduled                pod/elasticsearch-master-0   Successfully assigned logging/elasticsearch-master-0 to elasticsearch-eqmy
4m9s        Normal    SuccessfulAttachVolume   pod/elasticsearch-master-0   AttachVolume.Attach succeeded for volume "pvc-43d3e4a1-0b71-4dff-923e-121d85459fe7"
4m6s        Normal    Pulled                   pod/elasticsearch-master-0   Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
4m6s        Normal    Created                  pod/elasticsearch-master-0   Created container configure-sysctl
4m6s        Normal    Started                  pod/elasticsearch-master-0   Started container configure-sysctl
4m5s        Normal    Pulled                   pod/elasticsearch-master-0   Container image "busybox:1.28" already present on machine
4m5s        Normal    Created                  pod/elasticsearch-master-0   Created container create
4m5s        Normal    Started                  pod/elasticsearch-master-0   Started container create
4m4s        Normal    Pulled                   pod/elasticsearch-master-0   Container image "busybox:1.28" already present on machine
4m4s        Normal    Created                  pod/elasticsearch-master-0   Created container file-permissions
4m4s        Normal    Started                  pod/elasticsearch-master-0   Started container file-permissions
4m3s        Normal    Pulled                   pod/elasticsearch-master-0   Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
4m3s        Normal    Created                  pod/elasticsearch-master-0   Created container elasticsearch
4m3s        Normal    Started                  pod/elasticsearch-master-0   Started container elasticsearch
3m30s       Warning   Unhealthy                pod/elasticsearch-master-0   Readiness probe failed: Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
4m19s       Warning   FailedScheduling         pod/elasticsearch-master-1   pod has unbound immediate PersistentVolumeClaims (repeated 3 times)
4m16s       Normal    Scheduled                pod/elasticsearch-master-1   Successfully assigned logging/elasticsearch-master-1 to elasticsearch-eqmo
4m9s        Normal    SuccessfulAttachVolume   pod/elasticsearch-master-1   AttachVolume.Attach succeeded for volume "pvc-e9273b4b-5a24-4935-a556-d82252969323"
3m57s       Normal    Pulled                   pod/elasticsearch-master-1   Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
3m57s       Normal    Created                  pod/elasticsearch-master-1   Created container configure-sysctl
3m57s       Normal    Started                  pod/elasticsearch-master-1   Started container configure-sysctl
3m56s       Normal    Pulled                   pod/elasticsearch-master-1   Container image "busybox:1.28" already present on machine
3m56s       Normal    Created                  pod/elasticsearch-master-1   Created container create
3m55s       Normal    Started                  pod/elasticsearch-master-1   Started container create
3m55s       Normal    Pulled                   pod/elasticsearch-master-1   Container image "busybox:1.28" already present on machine
3m55s       Normal    Created                  pod/elasticsearch-master-1   Created container file-permissions
3m54s       Normal    Started                  pod/elasticsearch-master-1   Started container file-permissions
3m54s       Normal    Pulled                   pod/elasticsearch-master-1   Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
3m54s       Normal    Created                  pod/elasticsearch-master-1   Created container elasticsearch
3m53s       Normal    Started                  pod/elasticsearch-master-1   Started container elasticsearch
3m18s       Warning   Unhealthy                pod/elasticsearch-master-1   Readiness probe failed: Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
4m18s       Warning   FailedScheduling         pod/elasticsearch-master-2   pod has unbound immediate PersistentVolumeClaims (repeated 3 times)
4m16s       Normal    Scheduled                pod/elasticsearch-master-2   Successfully assigned logging/elasticsearch-master-2 to elasticsearch-eqmx
4m10s       Normal    SuccessfulAttachVolume   pod/elasticsearch-master-2   AttachVolume.Attach succeeded for volume "pvc-77aca0f2-e0f4-4380-b15a-fa78a41eee38"
4m5s        Normal    Pulled                   pod/elasticsearch-master-2   Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
4m4s        Normal    Created                  pod/elasticsearch-master-2   Created container configure-sysctl
4m4s        Normal    Started                  pod/elasticsearch-master-2   Started container configure-sysctl
4m4s        Normal    Pulled                   pod/elasticsearch-master-2   Container image "busybox:1.28" already present on machine
4m4s        Normal    Created                  pod/elasticsearch-master-2   Created container create
4m4s        Normal    Started                  pod/elasticsearch-master-2   Started container create
4m3s        Normal    Pulled                   pod/elasticsearch-master-2   Container image "busybox:1.28" already present on machine
4m3s        Normal    Created                  pod/elasticsearch-master-2   Created container file-permissions
4m3s        Normal    Started                  pod/elasticsearch-master-2   Started container file-permissions
4m2s        Normal    Pulled                   pod/elasticsearch-master-2   Container image "docker.elastic.co/elasticsearch/elasticsearch:7.4.1" already present on machine
4m2s        Normal    Created                  pod/elasticsearch-master-2   Created container elasticsearch
4m1s        Normal    Started                  pod/elasticsearch-master-2   Started container elasticsearch
3m32s       Warning   Unhealthy                pod/elasticsearch-master-2   Readiness probe failed: Waiting for elasticsearch cluster to become cluster to be ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
4m20s       Normal    ExternalProvisioning     persistentvolumeclaim/elasticsearch-master-elasticsearch-master-0   waiting for a volume to be created, either by external provisioner "dobs.csi.digitalocean.com" or manually created by system administrator
4m20s       Normal    Provisioning             persistentvolumeclaim/elasticsearch-master-elasticsearch-master-0   External provisioner is provisioning volume for claim "logging/elasticsearch-master-elasticsearch-master-0"
4m20s       Normal    ExternalProvisioning     persistentvolumeclaim/elasticsearch-master-elasticsearch-master-1   waiting for a volume to be created, either by external provisioner "dobs.csi.digitalocean.com" or manually created by system administrator
4m20s       Normal    Provisioning             persistentvolumeclaim/elasticsearch-master-elasticsearch-master-1   External provisioner is provisioning volume for claim "logging/elasticsearch-master-elasticsearch-master-1"
4m20s       Normal    ExternalProvisioning     persistentvolumeclaim/elasticsearch-master-elasticsearch-master-2   waiting for a volume to be created, either by external provisioner "dobs.csi.digitalocean.com" or manually created by system administrator
4m20s       Normal    Provisioning             persistentvolumeclaim/elasticsearch-master-elasticsearch-master-2   External provisioner is provisioning volume for claim "logging/elasticsearch-master-elasticsearch-master-2"
4m21s       Normal    NoPods                   poddisruptionbudget/elasticsearch-master-pdb                        No matching pods found
4m20s       Normal    SuccessfulCreate         statefulset/elasticsearch-master                                    create Claim elasticsearch-master-elasticsearch-master-0 Pod elasticsearch-master-0 in StatefulSet elasticsearch-master success
4m20s       Normal    SuccessfulCreate         statefulset/elasticsearch-master                                    create Pod elasticsearch-master-0 in StatefulSet elasticsearch-master successful
4m20s       Normal    SuccessfulCreate         statefulset/elasticsearch-master                                    create Claim elasticsearch-master-elasticsearch-master-1 Pod elasticsearch-master-1 in StatefulSet elasticsearch-master success
4m20s       Normal    SuccessfulCreate         statefulset/elasticsearch-master                                    create Pod elasticsearch-master-1 in StatefulSet elasticsearch-master successful
4m20s       Normal    SuccessfulCreate         statefulset/elasticsearch-master                                    create Claim elasticsearch-master-elasticsearch-master-2 Pod elasticsearch-master-2 in StatefulSet elasticsearch-master success
4m20s       Normal    SuccessfulCreate         statefulset/elasticsearch-master                                    create Pod elasticsearch-master-2 in StatefulSet elasticsearch-master successful

Logs:

 kubectl logs -l elasticsearch --namespace=logging

This returns nothing so showing specific pod:

> kubectl logs  elasticsearch-master-0 --namespace logging
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
{"type": "server", "timestamp": "2019-11-21T13:11:11,697Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/disk/by-id/scsi-0DO_Volume_pvc-43d3e4a1-0b71-4dff-923e-121d85459fe7)]], net usable_space [16.6gb], net total_space [17.5gb], types [ext4]" }
{"type": "server", "timestamp": "2019-11-21T13:11:11,703Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "heap size [1015.6mb], compressed ordinary object pointers [true]" }
{"type": "server", "timestamp": "2019-11-21T13:11:11,708Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "node name [elasticsearch-master-0], node ID [C_nHx1VvTJKaPcAzOWY7AQ], cluster name [elasticsearch]" }
{"type": "server", "timestamp": "2019-11-21T13:11:11,709Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "version[7.4.1], pid[1], build[default/docker/fc0eeb6e2c25915d63d871d344e3d0b45ea0ea1e/2019-10-22T17:16:35.176724Z], OS[Linux/4.19.0-0.bpo.6-amd64/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/13/13+33]" }
{"type": "server", "timestamp": "2019-11-21T13:11:11,714Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "JVM home [/usr/share/elasticsearch/jdk]" }
{"type": "server", "timestamp": "2019-11-21T13:11:11,714Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch-2210672167454109916, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -Des.cgroups.hierarchy.override=/, -Xmx1g, -Xms1g, -Dio.netty.allocator.type=unpooled, -XX:MaxDirectMemorySize=536870912, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [data-frame]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [flattened]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,506Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,507Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,507Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,508Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,508Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,508Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,509Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,510Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,510Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,511Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,512Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,512Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,513Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,514Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,516Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,517Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,517Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,517Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,518Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,519Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,519Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,519Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,519Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,520Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,520Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,520Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,521Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,521Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,521Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2019-11-21T13:11:17,522Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2019-11-21T13:11:28,072Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2019-11-21T13:11:29,521Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "[controller/86] [Main.cc@110] controller (64 bit): Version 7.4.1 (Build 973380bdacc5e8) Copyright (c) 2019 Elasticsearch BV" }
{"type": "server", "timestamp": "2019-11-21T13:11:31,117Z", "level": "DEBUG", "component": "o.e.a.ActionModule", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "Using REST wrapper from plugin org.elasticsearch.xpack.security.Security" }
{"type": "server", "timestamp": "2019-11-21T13:11:31,898Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2019-11-21T13:11:34,405Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "initialized" }
{"type": "server", "timestamp": "2019-11-21T13:11:34,405Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "starting ..." }
{"type": "server", "timestamp": "2019-11-21T13:11:34,733Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "publish_address {10.244.3.26:9300}, bound_addresses {[::]:9300}" }
{"type": "server", "timestamp": "2019-11-21T13:11:34,799Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2019-11-21T13:11:43,043Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "setting initial configuration to VotingConfiguration{{bootstrap-placeholder}-elasticsearch-master-1,C_nHx1VvTJKaPcAzOWY7AQ,ZJSi-XhYS6q4yl847ER_sQ}" }
{"type": "server", "timestamp": "2019-11-21T13:11:43,302Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-0}{C_nHx1VvTJKaPcAzOWY7AQ}{h0vJDEjxT3y0U_hHGpcD-A}{10.244.3.26}{10.244.3.26:9300}{dilm}{ml.machine_memory=1999998976, xpack.installed=true, ml.max_open_jobs=20} elect leader, {elasticsearch-master-2}{ZJSi-XhYS6q4yl847ER_sQ}{lfakg92cTGqF0ZY62V1nZA}{10.244.4.209}{10.244.4.209:9300}{dilm}{ml.machine_memory=1999998976, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 1, version: 1, reason: master node changed {previous [], current [{elasticsearch-master-0}{C_nHx1VvTJKaPcAzOWY7AQ}{h0vJDEjxT3y0U_hHGpcD-A}{10.244.3.26}{10.244.3.26:9300}{dilm}{ml.machine_memory=1999998976, xpack.installed=true, ml.max_open_jobs=20}]}, added {{elasticsearch-master-2}{ZJSi-XhYS6q4yl847ER_sQ}{lfakg92cTGqF0ZY62V1nZA}{10.244.4.209}{10.244.4.209:9300}{dilm}{ml.machine_memory=1999998976, ml.max_open_jobs=20, xpack.installed=true},}" }
{"type": "server", "timestamp": "2019-11-21T13:11:43,498Z", "level": "INFO", "component": "o.e.c.c.CoordinationState", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "cluster UUID set to [k4zx5tmdRdik8ovYRkiEtA]" }
{"type": "server", "timestamp": "2019-11-21T13:11:43,727Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "master node changed {previous [], current [{elasticsearch-master-0}{C_nHx1VvTJKaPcAzOWY7AQ}{h0vJDEjxT3y0U_hHGpcD-A}{10.244.3.26}{10.244.3.26:9300}{dilm}{ml.machine_memory=1999998976, xpack.installed=true, ml.max_open_jobs=20}]}, added {{elasticsearch-master-2}{ZJSi-XhYS6q4yl847ER_sQ}{lfakg92cTGqF0ZY62V1nZA}{10.244.4.209}{10.244.4.209:9300}{dilm}{ml.machine_memory=1999998976, ml.max_open_jobs=20, xpack.installed=true},}, term: 1, version: 1, reason: Publication{term=1, version=1}" }
{"type": "server", "timestamp": "2019-11-21T13:11:43,899Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "publish_address {10.244.3.26:9200}, bound_addresses {[::]:9200}", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:43,900Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "started", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:44,240Z", "level": "INFO", "component": "o.e.g.GatewayService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "recovered [0] indices into cluster_state", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:45,228Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.triggered_watches] for index patterns [.triggered_watches*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:45,431Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.watch-history-10] for index patterns [.watcher-history-10*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:45,633Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.watches] for index patterns [.watches*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:45,847Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.slm-history] for index patterns [.slm-history-1*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:45,950Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:46,159Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.monitoring-es] for index patterns [.monitoring-es-7-*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:46,313Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.monitoring-beats] for index patterns [.monitoring-beats-7-*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:46,433Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:46,644Z", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:46,896Z", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding index lifecycle policy [watch-history-ilm-policy]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:47,101Z", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "adding index lifecycle policy [slm-history-ilm-policy]", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:47,795Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "license [352fec6c-081d-4309-b260-236597a84d73] mode [basic] - valid", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:47,797Z", "level": "INFO", "component": "o.e.x.s.s.SecurityStatusChangeListener", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "Active license is now [BASIC]; Security is disabled", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:50,952Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "node-join[{elasticsearch-master-1}{RqtlG7ozSiO_RC18-cEVEg}{IM5CBtNmSgaN910OBsS5fQ}{10.244.5.194}{10.244.5.194:9300}{dilm}{ml.machine_memory=1999998976, ml.max_open_jobs=20, xpack.installed=true} join existing leader], term: 1, version: 18, reason: added {{elasticsearch-master-1}{RqtlG7ozSiO_RC18-cEVEg}{IM5CBtNmSgaN910OBsS5fQ}{10.244.5.194}{10.244.5.194:9300}{dilm}{ml.machine_memory=1999998976, ml.max_open_jobs=20, xpack.installed=true},}", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }
{"type": "server", "timestamp": "2019-11-21T13:11:52,445Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "elasticsearch", "node.name": "elasticsearch-master-0", "message": "added {{elasticsearch-master-1}{RqtlG7ozSiO_RC18-cEVEg}{IM5CBtNmSgaN910OBsS5fQ}{10.244.5.194}{10.244.5.194:9300}{dilm}{ml.machine_memory=1999998976, ml.max_open_jobs=20, xpack.installed=true},}, term: 1, version: 18, reason: Publication{term=1, version=18}", "cluster.uuid": "k4zx5tmdRdik8ovYRkiEtA", "node.id": "C_nHx1VvTJKaPcAzOWY7AQ"  }

As you can see probe failed :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment