filebeat.template.json

{
  "mappings": {
    "_default_": {
      "_meta": {
        "version": "5.6.2"
      },
      "date_detection": false,
      "dynamic_templates": [
        {
          "strings_as_keyword": {
            "mapping": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "match_mapping_type": "string"
          }
        }
      ],
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "beat": {
          "properties": {
            "hostname": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "version": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        },
        "error": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "fileset": {
          "properties": {
            "module": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "name": {
              "ignore_above": 1024,
              "type": "keyword"
            }
          }
        },
        "input_type": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "message": {
          "norms": false,
          "type": "text"
        },
        "meta": {
          "properties": {
            "cloud": {
              "properties": {
                "availability_zone": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "instance_id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "machine_type": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "project_id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "provider": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "region": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              }
            }
          }
        },
        "nginx": {
          "properties": {
            "access": {
              "properties": {
                "remote_addr": {
                  "type": "ip"
                },
                "remote_user": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "time_local": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "request": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "status": {
                  "type": "long"
                },
                "bytes_sent": {
                  "type": "long"
                },
                "http_referrer": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "http_user_agent": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              }
            },
            "error": {
              "properties": {
                "connection_id": {
                  "type": "long"
                },
                "level": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "message": {
                  "norms": false,
                  "type": "text"
                },
                "pid": {
                  "type": "long"
                },
                "tid": {
                  "type": "long"
                }
              }
            }
          }
        },
        "offset": {
          "type": "long"
        },
        "read_timestamp": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "source": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "system": {
          "properties": {
            "auth": {
              "properties": {
                "groupadd": {
                  "properties": {
                    "gid": {
                      "type": "long"
                    },
                    "name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "hostname": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "message": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "pid": {
                  "type": "long"
                },
                "program": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "ssh": {
                  "properties": {
                    "dropped_ip": {
                      "type": "ip"
                    },
                    "event": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "ip": {
                      "type": "ip"
                    },
                    "method": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "port": {
                      "type": "long"
                    },
                    "signature": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "sudo": {
                  "properties": {
                    "command": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "error": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "pwd": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "tty": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "user": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "timestamp": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "user": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "useradd": {
                  "properties": {
                    "gid": {
                      "type": "long"
                    },
                    "home": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "shell": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "uid": {
                      "type": "long"
                    }
                  }
                }
              }
            },
            "syslog": {
              "properties": {
                "hostname": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "message": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "pid": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "program": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "timestamp": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              }
            }
          }
        },
        "tags": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "type": {
          "ignore_above": 1024,
          "type": "keyword"
        }
      }
    }
  },
  "order": 0,
  "settings": {
    "index.mapping.total_fields.limit": 10000,
    "index.refresh_interval": "5s",
    "number_of_shards": 1,
    "number_of_replicas": 0
  },
  "template": "filebeat-*"
}

log_format.conf

# vim: ft=nginx:

# Default "combined"
#log_format combined '$remote_addr - $remote_user [$time_local] '
#                    '"$request" $status $body_bytes_sent '
#                    '"$http_referer" "$http_user_agent"';

# Extending "combined" to include more fields
log_format main
  '$remote_addr - $remote_user [$time_local] '
  '"$request" $status $body_bytes_sent '
  '"$http_referer" "$http_user_agent" '
  '"'
  'host=$host|'
  'request_time=$request_time|'
  'http_x_forwarded_for=$http_x_forwarded_for|'
  'http_cloudfront_viewer_country=$http_cloudfront_viewer_country|'
  'upstream_addr=$upstream_addr|'
  'upstream_status=$upstream_status|'
  'upstream_response_time=$upstream_response_time'
  '"'
;

nginx.access.pipeline.json

{
  "description": "Pipeline for parsing Nginx access logs.",
  "processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{IP:nginx.access.remote_addr} - %{DATA:nginx.access.remote_user} \\[%{HTTPDATE:nginx.access.time_local}\\] \"%{DATA:nginx.access.request}\" %{NUMBER:nginx.access.status} %{NUMBER:nginx.access.bytes_sent} \"%{DATA:nginx.access.http_referrer}\" \"%{DATA:nginx.access.http_user_agent}\"( \"%{DATA:nginx.access._kvs}\")?"
        ],
        "ignore_missing": true
      }
    },
    {
      "remove": {
        "field": "message"
      }
    },
    {
      "rename": {
        "field": "@timestamp",
        "target_field": "read_timestamp"
      }
    },
    {
      "date": {
        "field": "nginx.access.time_local",
        "target_field": "@timestamp",
        "formats": [
          "dd/MMM/YYYY:H:m:s Z"
        ]
      }
    },
    {
      "kv": {
        "field": "nginx.access._kvs",
        "field_split": "\\|",
        "value_split": "=",
        "target_field": "nginx.access",
        "ignore_missing": true,
        "ignore_failure": true
      }
    },
    {
      "remove": {
        "field": "nginx.access._kvs",
        "ignore_failure": true
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error.message",
        "value": "{{ _ingest.on_failure_message }}"
      }
    }
  ]
}

nginx.error.pipeline.json

{
  "description": "Pipeline for parsing the Nginx error logs",
  "processors": [
    {
      "grok": {
        "field": "message",
        "patterns": [
          "%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}"
        ],
        "ignore_missing": true
      }
    },
    {
      "remove": {
        "field": "message"
      }
    },
    {
      "rename": {
        "field": "@timestamp",
        "target_field": "read_timestamp"
      }
    },
    {
      "date": {
        "field": "nginx.error.time",
        "target_field": "@timestamp",
        "formats": [
          "YYYY/MM/dd H:m:s"
        ],
        "timezone": "America/New_York"
      }
    },
    {
      "remove": {
        "field": "nginx.error.time"
      }
    }
  ],
  "on_failure": [
    {
      "set": {
        "field": "error.message",
        "value": "{{ _ingest.on_failure_message }}"
      }
    }
  ]
}