Peter Titov iamhowardtheduck
iamhowardtheduck
/ PEOPLE.sh
Last active
March 2, 2021 21:32
Placing Elastic On Premise Lovingly & Expeditiously
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| clear | |
| echo -e "\n\n\n\n\n\n\n" | |
| if [[ $EUID -eq 0 ]]; then | |
| echo "This script must NOT be run as \"root\" OR as \"sudo $USER\"; please try again." 1>&2 | |
| exit 1 | |
| fi | |
| # | |
| # BEGIN WELCOME SCREEN & INITIAL UPDATING | |
| # | |
| clear |
iamhowardtheduck
/ PEOPLE-RC-DNU.sh
Last active
January 14, 2020 02:38
Rolling build of PEOPLE, Release Candidate- DO NOT USE
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| clear | |
| echo -e "\n\n\n\n\n\n\n" | |
| if [[ $EUID -eq 0 ]]; then | |
| echo "This script must NOT be run as \"root\" OR as \"sudo $USER\"; please try again." 1>&2 | |
| exit 1 | |
| fi | |
| # | |
| # BEGIN WELCOME SCREEN & INITIAL UPDATING | |
| # | |
| clear |
iamhowardtheduck
/ Pi-Hole_ECS_Logstash_Pipeline
Last active
October 26, 2019 22:20
Pi-Hole ECS Compliant Pipeline
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| filter { | |
| if "/var/log/pihole.log" == [log][file][path] { | |
| grok { match => [ "message", "%{SYSLOGTIMESTAMP:Timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\[(?<dns.question.type>SRV)\] %{NOTSPACE:dns.question.name} %{WORD:dns.answers.class} %{IP:client.ip}", | |
| "message", "%{SYSLOGTIMESTAMP:Timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\[(?<dns.question.type>PTR)\] %{HOSTNAME:dns.question.name} %{WORD:dns.answers.class} %{IP:client.ip}", | |
| #"message", "%{SYSLOGTIMESTAMP:Timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\[(?<dns.question.type>A)\] %{HOSTNAME:dns.question.name} %{WORD:dns.answers.class} %{IP:client.ip}", | |
| "message", "%{SYSLOGTIMESTAMP:Timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\[(?<dns.question.type>A)\] %{NOTSPACE:dns.question.name} %{WORD:dns.answers.class} %{IP:client.ip}", | |
| #"message", "%{SYSLOGTIMESTAMP:Timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\[(?<dns.question.type>AAAA)\] %{HOSTNAME:dns.question.name} %{WORD:dns.answers.class} (?<client.ip>\d+.\d+.\d+.\d+)", | |
| "message", "%{SYSL |
iamhowardtheduck
/ Pi-Hole_ECS_Pipeline_Dashboard.ndjson
Last active
September 8, 2019 03:51
Pi-Hole ECS Compliant Dashboard
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"attributes":{"fieldFormatMap":"{\"client.bytes\":{\"id\":\"bytes\"},\"client.port\":{\"id\":\"string\"},\"coredns.query.size\":{\"id\":\"bytes\"},\"coredns.response.size\":{\"id\":\"bytes\"},\"destination.bytes\":{\"id\":\"bytes\"},\"destination.port\":{\"id\":\"string\"},\"envoyproxy.upstream_service_time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\"}},\"event.duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\",\"outputFormat\":\"asMilliseconds\",\"outputPrecision\":1}},\"event.severity\":{\"id\":\"string\"},\"http.request.body.bytes\":{\"id\":\"bytes\"},\"http.request.bytes\":{\"id\":\"bytes\"},\"http.response.body.bytes\":{\"id\":\"bytes\"},\"http.response.bytes\":{\"id\":\"bytes\"},\"http.response.status_code\":{\"id\":\"string\"},\"mysql.slowlog.bytes_received\":{\"id\":\"bytes\"},\"mysql.slowlog.bytes_sent\":{\"id\":\"bytes\"},\"mysql.slowlog.innodb.io_r_bytes\":{\"id\":\"bytes\"},\"mysql.slowlog.tmp_table_sizes\":{\"id\":\"bytes\"},\"nats.log.msg.bytes\":{ |
iamhowardtheduck
/ meetups.conf
Last active
September 19, 2019 02:27
Simply add your desired output section and save it as a *.conf file. Then run this command: curl http://stream.meetup.com/2/rsvps | /usr/share/logstash/bin/logstash -f wherever_you_saved_it_as.conf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| stdin { | |
| codec => json_lines | |
| } | |
| } | |
| filter { | |
| mutate { | |
| uppercase => [ "[group][group_country]" ] | |
| replace => { "location" => "%{[group][group_lat]},%{[group][group_lon]}" } |
iamhowardtheduck
/ meetups.ndjson
Last active
January 28, 2020 01:54
MeetUp.com Index Pattern & Template
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"attributes":{"fieldFormatMap":"{\"member.photo\":{\"id\":\"url\",\"params\":{\"type\":\"img\",\"urlTemplate\":\"\"}},\"member.other_services.facebook.identifier\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://www.facebook.com/{{value}}\"}}}","fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\ |
iamhowardtheduck
/ MeetUpDashboards.ndjson
Last active
September 18, 2019 11:39
MeetUp Event Viewer & Event Finder dashboards for Kibana.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"attributes":{"fieldFormatMap":"{\"member.photo\":{\"id\":\"url\",\"params\":{\"type\":\"img\",\"urlTemplate\":\"\"}},\"member.other_services.facebook.identifier\":{\"id\":\"url\",\"params\":{\"urlTemplate\":\"https://www.facebook.com/{{value}}\"}}}","fields":"[{\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\ |
iamhowardtheduck
/ Pi-Hole_ECS_Pipeline
Last active
December 2, 2019 18:29
Pi-Hole ECS Elastic Ingest Node Pipeline
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| PUT /_ingest/pipeline/pi-hole | |
| { | |
| "description" : "Pi-Hole Parser", | |
| "version" : 1, | |
| "processors" : [ | |
| { | |
| "grok" : { | |
| "field" : "message", | |
| "patterns" : [ | |
| "%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGPROG:Prog}: (?<dns.type>query)\\[(?<dns.question.type>A)\\] %{NOTSPACE:dns.question.name} (?<dns.question.class>from) %{IP:client.ip}", |
iamhowardtheduck
/ pihole-ecs-dashboard.ndjson
Last active
December 2, 2019 18:32
Pi-Hole ECS Dashboard V2: Now with Forbidden Trigram Detection
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"attributes":{"fieldFormatMap":"{\"client.bytes\":{\"id\":\"bytes\"},\"client.nat.port\":{\"id\":\"string\"},\"client.port\":{\"id\":\"string\"},\"destination.bytes\":{\"id\":\"bytes\"},\"destination.nat.port\":{\"id\":\"string\"},\"destination.port\":{\"id\":\"string\"},\"event.duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"nanoseconds\",\"outputFormat\":\"asMilliseconds\",\"outputPrecision\":1}},\"event.sequence\":{\"id\":\"string\"},\"event.severity\":{\"id\":\"string\"},\"http.request.body.bytes\":{\"id\":\"bytes\"},\"http.request.bytes\":{\"id\":\"bytes\"},\"http.response.body.bytes\":{\"id\":\"bytes\"},\"http.response.bytes\":{\"id\":\"bytes\"},\"http.response.status_code\":{\"id\":\"string\"},\"network.bytes\":{\"id\":\"bytes\"},\"process.pgid\":{\"id\":\"string\"},\"process.pid\":{\"id\":\"string\"},\"process.ppid\":{\"id\":\"string\"},\"process.thread.id\":{\"id\":\"string\"},\"server.bytes\":{\"id\":\"bytes\"},\"server.nat.port\":{\"id\":\"string\"},\"server.port\":{\"id\":\"string\" |
iamhowardtheduck
/ Majestic_ECS_Pipeline
Created
October 27, 2019 20:57
Majestic Top 1 Million CSV Ingest Node Pipeline
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| PUT /_ingest/pipeline/majestic | |
| { | |
| "description" : "Majestic Parser", | |
| "version" : 1, | |
| "processors" : [ | |
| { "set":{"field":"@timestamp","value": "{{_ingest.timestamp}}"}}, | |
| { | |
| "dissect" : { | |
| "field" : "message", | |
| "pattern" :"%{GlobalRank},%{TldRank},%{Domain},%{TLD},%{RefSubNets},%{RefIPs},%{IDN_Domain},%{IDN_TLD},%{PrevGlobalRank},%{PrevTldRank},%{PrevRefSubNets},%{PrevRefIPs}" |