Mark Walkom markwalkom

## logstash.conf
input {
  elasticsearch {
    hosts => [ "HOSTNAME_HERE" ]
    port => "9200"
    index => "INDEXNAME_HERE"
    size => 1000
    scroll => "5m"
    docinfo => true
    scan => true
  }

## gist:3766250c5f6b6206bafcc6c23562b3fc
WINDNS %{NUMBER:log_date} %{TIME:log_time} %{WORD:dns_thread_id} %{WORD:dns_context}%{SPACE}%{WORD:dns_packet_id} %{WORD:dns_ip_protocol} %{WORD:dns_direction} %{IP:dns_client_address}%{SPACE}%{WORD:dns_xid}%{SPACE}(?:Q|R|U) ?(Q|R|U)?%{SPACE}[%{GREEDYDATA:dns_hex_flags}%{SPACE}%{WORD:dns_response}]%{SPACE}%{WORD:dns_recordtype}%{SPACE}([1-9][0-9]?)%{GREEDYDATA:dns_query_name}

## README.md

      
              3 files
            
          
              1 fork
            
          
                0 comments
              
            
              2 stars
            
          
                markwalkom
                / README.md
            
            
              Last active
              April 21, 2020 20:50
            
              
                PiHole + Elasticsearch Ingest processing
              
          
    NOTE - this was specifically built for a docker instance, with the Filebeat docker module collecting the logs.

  
## Beats.md

      
              5 files
            
          
              0 forks
            
          
                0 comments
              
            
              5 stars
            
          
                markwalkom
                / Beats.md
            
            
              Last active
              March 9, 2020 19:05
            
              
                LCA2019 - Awesome Monitoring Infrastructure Using the Elastic Stack
              
          
    Metricbeat

https://www.elastic.co/guide/en/beats/metricbeat/current/index.html
./metricbeat
./metricbeat setup --help
./metricbeat modules --help

./metricbeat setup --template -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["localhost:9200"]'

  
## gist:a96e8b40220872610d1098d891e7cff0
@elastic OR @logstash OR @elasticsearch OR Elasticsearch OR Logstash OR Kibana OR packetbeat OR "elastic stack" OR "elastic search" OR elasticbeats OR filebeat OR elasticon OR "elk stack” OR swiftype OR auditbeat OR “elastic apm” OR “open source apm” OR elkstack OR belkstack OR opbeat OR “Elastic APM” OR elastic.co OR “elk stack” OR “elastic cloud” OR elastalert OR Swiftype OR Swifttype OR skedlr OR lifeatelastic OR searchguard OR “elastic endpoint” -@Kibana_DRAGON_ -@kibana_love

## Elasticsearch-5.6.2.xml
<?xml version="1.0"?>
<Container version="2">
  <Name>Elasticsearch-5.6.2</Name>
  <Repository>59b11c02b218</Repository>
  <Registry>https://docker.elastic.co/</Registry>
  <Network>bridge</Network>
  <Privileged>false</Privileged>
  <Support>https://discuss.elastic.co/c/elasticsearch</Support>
  <Overview>Elasticsearch is a open source, distributed, RESTful search and analytics engine.</Overview>
  <Category>Tools:</Category>

## crlf.csv

          
            field0
            field1
            fieldcrlf
            field3
            field4

            
              hello
              1234
              this is a 
line feed
              3.0
              AYX

## BOM-json.py
from ftplib import FTP
import os
import xml.etree.ElementTree as ET
import xmltodict
import json

url = 'ftp.bom.gov.au'
filename = 'IDN65068.xml'

def writeline(data):

## 1 - kibana.yml
# Custom Region Maps
regionmap:
  layers:
     - name: "Australian States"
       url: "http://localhost:8000/aus_state.geojson"
       attribution: "exploratory.io"
       fields:
          - name: "STATE_NAME"
            description: "State Name"

## paypal-transactions.conf
input {
  stdin {}
}
filter {
  csv {
    columns => ["Date","Time","Time Zone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Counterparty Status","Shipping address","Address Status","Item Title","Item ID","Shipping and Handling Amount","Compensation Amount","GST","Option 1 Name","Option 1 Value","Option 2 Name","Option 2 Value","Auction Site","Buyer ID","Item URL","Closing Date","Escrow ID","Invoice ID","Reference Txn ID","Invoice Number","Custom Number","Quantity","Receipt ID","Balance","Contact Phone Number"]
    add_field => [ "timestamp", "%{Date} %{Time}" ]
    remove_field => [ "Date", "Time", "Time Zone" ]
  }
  date {
	input {
	elasticsearch {
	hosts => [ "HOSTNAME_HERE" ]
	port => "9200"
	index => "INDEXNAME_HERE"
	size => 1000
	scroll => "5m"
	docinfo => true
	scan => true
	}
	<?xml version="1.0"?>
	<Container version="2">
	<Name>Elasticsearch-5.6.2</Name>
	<Repository>59b11c02b218</Repository>
	<Registry>https://docker.elastic.co/</Registry>
	<Network>bridge</Network>
	<Privileged>false</Privileged>
	<Support>https://discuss.elastic.co/c/elasticsearch</Support>
	<Overview>Elasticsearch is a open source, distributed, RESTful search and analytics engine.</Overview>
	<Category>Tools:</Category>
	field0	field1	fieldcrlf	field3	field4
	hello	1234	this is a line feed	3.0	AYX
	from ftplib import FTP
	import os
	import xml.etree.ElementTree as ET
	import xmltodict
	import json

	url = 'ftp.bom.gov.au'
	filename = 'IDN65068.xml'

	def writeline(data):
	# Custom Region Maps
	regionmap:
	layers:
	- name: "Australian States"
	url: "http://localhost:8000/aus_state.geojson"
	attribution: "exploratory.io"
	fields:
	- name: "STATE_NAME"
	description: "State Name"
	input {
	stdin {}
	}
	filter {
	csv {
	columns => ["Date","Time","Time Zone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Counterparty Status","Shipping address","Address Status","Item Title","Item ID","Shipping and Handling Amount","Compensation Amount","GST","Option 1 Name","Option 1 Value","Option 2 Name","Option 2 Value","Auction Site","Buyer ID","Item URL","Closing Date","Escrow ID","Invoice ID","Reference Txn ID","Invoice Number","Custom Number","Quantity","Receipt ID","Balance","Contact Phone Number"]
	add_field => [ "timestamp", "%{Date} %{Time}" ]
	remove_field => [ "Date", "Time", "Time Zone" ]
	}
	date {