Demo use case: integrating a COVID-19 tracker API with the Elastic Stack plus story telling with Kibana

Demo use case: integrating a COVID-19 tracker API with the Elastic Stack plus story telling with Kibana.txt

Demo use case: integrating a COVID-19 tracker API with the Elastic Stack plus story telling with Kibana

https://medium.com/@quoeamaster/use-case-integrating-a-covid-19-tracker-api-with-the-elastic-stack-plus-story-telling-with-kibana-8805cb67678

Create the pipeline in the Elasticsearch before start the filebeat.

=== Update for filebeat

filebeat.inputs:
- type: log
  paths:
    - covid19-02042020.csv
  exclude_lines: ['^Lat']

output.elasticsearch:
  hosts: ["http://localhost:9200"]
  index: coronavirus
  pipeline: "coronavirus_parser"

setup.ilm.enabled: false
setup.template.name: coronavirus
setup.template.pattern: coronavirus


=== update for ingest pipeline


PUT _ingest/pipeline/coronavirus_parser
{
  "version": 5,
  "processors": [
    {
      "gsub": {
        "if": "ctx.message != null", 
        "field": "message",
        "pattern": "\"",
        "replacement": "'"
      }
    },
    {
      "grok": {
        "if": "ctx.message != null", 
        "field": "message",
        "patterns": ["%{NUMBER:lat:float},%{NUMBER:lon:float},'%{DATA:city}',%{DATA:country},%{NUMBER:infected:float},%{NUMBER:death:float}", "%{NUMBER:lat:float},%{NUMBER:lon:float},%{DATA:city},%{DATA:country},%{NUMBER:infected:float},%{NUMBER:death:float}"]
      }
    },
    {
      "script": {
        "source": """ 
ctx.infected = (int)Math.floor(ctx.infected);
ctx.death = (int)Math.floor(ctx.death);
        """
      }
    },
    {
      "set": {
        "field": "location.lat",
        "value": "{{lat}}"
      }
    },
    {
      "set": {
        "field": "location.lon",
        "value": "{{lon}}"
      }
    },
    {
      "set": {
        "if": "ctx.country == 'City NA'",
        "field": "country",
        "value": "{{city}}"
      }
    },
    {
      "remove": {
        "field": ["log", "input", "ecs", "host", "agent", "message"],
        "if": "ctx.log != null && ctx.input != null && ctx.ecs != null && ctx.host != null && ctx.agent != null && ctx.message != null"
      }
    }
  ]
}