Ingest pipeline definition for measuring ingest delay based on @timestamp field

ingest_pipeline_delay

# Ingest pipeline that records the timestamp the event was processed (`@received`)
# by the ingest pipeline and calculates the difference in milliseconds compared to 
# the event timestamp (`@timestamp`).

POST _scripts/calculate_ingest_delay
{
  "script": {
    "lang": "painless",
    "source": "SimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"); ctx.ingest_delay = (sdf.parse(ctx['received']).getTime() - sdf.parse(ctx['@timestamp']).getTime()) / 1000.0"
  }
}

PUT _ingest/pipeline/ingest_delay
{
  "description" : "Pipeline measuring ingest delay for records based on the @timestamp and @received fields",
  "processors" : [
    {
      "set": {
        "field": "received",
        "value": "{{_ingest.timestamp}}"
      }
    },
    {
      "script": {
        "id": "calculate_ingest_delay"
      }
    }
  ]
}