iamhowardtheduck

input {
# Use file input to collect all CSVs from "git clone https://github.com/CSSEGISandData/COVID-19.git"
# Simply run the command while in /var/log:   git clone https://github.com/CSSEGISandData/COVID-19.git
file {
path => "/var/log/COVID-19/csse_covid_19_data/csse_covid_19_daily_reports/*.csv"
start_position => "beginning"
} }


filter {

iamhowardtheduck

clear
echo -e "\n\n\n\n\n\n\n"
if [[ $EUID -eq 0 ]]; then
  echo "This script must NOT be run as \"root\" OR as \"sudo $USER\"; please try again." 1>&2
  exit 1
fi
#
# BEGIN WELCOME SCREEN & INITIAL UPDATING
#
clear

iamhowardtheduck

PUT _ingest/pipeline/pihole
{
    "description" : "Pi-Hole",
    "processors" : [
      {
        "grok" : {
          "field" : "message",
          "patterns" : [
		  "%{SYSLOGTIMESTAMP:timestamp} %{GREEDYDATA:message}"]},
		  

iamhowardtheduck

PUT _template/filebeat-majestic
{
  "index_patterns": ["filebeat-majestic"],
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1
  },
{
  "filebeat-majestic" : {
    "order" : 0,

iamhowardtheduck

PUT /_ingest/pipeline/majestic
{
    "description" : "Majestic Parser",
    "version" : 1,
    "processors" : [
      { "set":{"field":"@timestamp","value": "{{_ingest.timestamp}}"}},
      {
        "dissect" : {
          "field" : "message",
          "pattern" :"%{GlobalRank},%{TldRank},%{Domain},%{TLD},%{RefSubNets},%{RefIPs},%{IDN_Domain},%{IDN_TLD},%{PrevGlobalRank},%{PrevTldRank},%{PrevRefSubNets},%{PrevRefIPs}"

iamhowardtheduck

{"attributes":{"fieldFormatMap":"{\"client.bytes\":{\"id\":\"bytes\"},\"client.nat.port\":{\"id\":\"string\"},\"client.port\":{\"id\":\"string\"},\"destination.bytes\":{\"id\":\"bytes\"},\"destination.nat.port\":{\"id\":\"string\"},\"destination.port\":{\"id\":\"string\"},\"event.duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\",\"outputFormat\":\"asMilliseconds\",\"outputPrecision\":1}},\"event.sequence\":{\"id\":\"string\"},\"event.severity\":{\"id\":\"string\"},\"http.request.body.bytes\":{\"id\":\"bytes\"},\"http.request.bytes\":{\"id\":\"bytes\"},\"http.response.body.bytes\":{\"id\":\"bytes\"},\"http.response.bytes\":{\"id\":\"bytes\"},\"http.response.status_code\":{\"id\":\"string\"},\"network.bytes\":{\"id\":\"bytes\"},\"process.pgid\":{\"id\":\"string\"},\"process.pid\":{\"id\":\"string\"},\"process.ppid\":{\"id\":\"string\"},\"process.thread.id\":{\"id\":\"string\"},\"server.bytes\":{\"id\":\"bytes\"},\"server.nat.port\":{\"id\":\"string\"},\"server.port\":{\"id\":\"string\"

iamhowardtheduck

PUT /_ingest/pipeline/pi-hole
{
    "description" : "Pi-Hole Parser",
    "version" : 1,
    "processors" : [
      {
        "grok" : {
          "field" : "message",
          "patterns" : [
"%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\\[(?<dns.question.type>A)\\] %{NOTSPACE:dns.question.name} (?<dns.question.class>from) %{IP:client.ip}", 

iamhowardtheduck

{"attributes":{"fieldFormatMap":"{\"member.photo\":{\"id\":\"url\",\"params\":{\"type\":\"img\",\"urlTemplate\":\"\"}},\"member.other_services.facebook.identifier\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://www.facebook.com/{{value}}\"}}}","fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\

iamhowardtheduck

{"attributes":{"fieldFormatMap":"{\"member.photo\":{\"id\":\"url\",\"params\":{\"type\":\"img\",\"urlTemplate\":\"\"}},\"member.other_services.facebook.identifier\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://www.facebook.com/{{value}}\"}}}","fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\

iamhowardtheduck

input {
  stdin {
    codec => json_lines
  }
}

filter {
    mutate  {
        uppercase => [ "[group][group_country]" ]
        replace => { "location" => "%{[group][group_lat]},%{[group][group_lon]}" }