use the force

FinalConfigrations.txt

## filebeat.yml

- type: log

  # Change to true to enable this prospector configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /home/tez/anaweb3/access.log.*
 #  - /home/tez/log.log
 #- c:\programdata\elasticsearch\logs\*
 
 #-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
  # Array of hosts to connect to.
 #hosts: ["localhost:9200"]

  # Optional protocol and basic auth credentials.
  #protocol: "https"
  #username: "elastic"
  #password: "changeme"

#----------------------------- Logstash output --------------------------------
output.logstash:
  # The Logstash hosts
  hosts: ["localhost:5044"]
  
  
  ##########
  
  # nginx-filebeat.conf                                                                                    

input {
        beats{
                port => "5044"
        }
#    file {
#        path =>"/home/enojen/log.log" start_position => "beginning"
#    }
}
filter {
    grok {
        match => { "message" => "%{NGINX}" }
    }
    date {
        match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    geoip {
        source => "clientip"
        target => "geoip"
  }
    useragent {
        source => "agent"
        target => "user_agent"
  }
}
output {
    elasticsearch {
              hosts => ["localhost:9200"]
        index => "baumfilebeat11"
        document_type => "logs"
    }
}

##################

# nginx.pattern

NGINX %{DATA:ipadress} - %{DATA:user_name} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:response_code} %{NUMBER:bytes:int} %{NUMBER:request_time:float} %{NUMBER:upstream_response_time:float} %{NOTSPACE:referer} %{QS:agent} %{DATA:zipcode}

someip - - [02/Apr/2018:06:50:08 +0300] "POST /response-code-byte HTTP/1.1" 200 35080 0.110 0.110 ."-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7_1 rv:2.0; it-IT) AppleWebKit/535.43.5 (KHTML, like Gecko) Version/4.0.4 Safari/535.43.5" "-"

someip - anadoluwebui [02/Apr/2018:06:50:08 +0300] "GET /response-code-byte HTTP/1.1" 200 35080 0.110 0.110 ."/enes/some/site" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7_1 rv:2.0; it-IT) AppleWebKit/535.43.5 (KHTML, like Gecko) Version/4.0.4 Safari/535.43.5" "26170"

ServerCompleteInstallation.txt

######### ZSH INSTALLATION #########

sudo apt-get update && sudo apt-get install zsh
zsh --version
chsh -s $(which zsh)
sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"
cd $ZSH_CUSTOM/plugins/

wget https://github.com/robbyrussell/oh-my-zsh/tree/master/plugins/colorize
wget https://github.com/robbyrussell/oh-my-zsh/tree/master/plugins/copydir
git clone https://github.com/zsh-users/zsh-autosuggestions
git clone https://github.com/zsh-users/zsh-syntax-highlighting
git clone https://github.com/trapd00r/zsh-syntax-highlighting-filetypes

vi ~/.zshrc
################## ADD PLUGIN ##################
 plugins=(
  git copydir colorize zsh-autosuggestions zsh-syntax-highlighting zsh-syntax-highlighting-filetypes
)


alias c="clear"
alias elasticsearch="/usr/share/elasticsearch/bin/elasticsearch"
alias elasticsearch-plugin="/usr/share/elasticsearch/bin/elasticsearch-plugin"
alias logstash="/usr/share/logstash/bin/logstash"
alias kibana="/usr/share/kibana/bin/kibana"
alias filebeat="/usr/share/filebeat/bin/filebeat"
alias estatus="sudo systemctl status elasticsearch"
alias kstatus="sudo systemctl status kibana"
alias nstatus="sudo systemctl status nginx"

source ~/.zshrc

######### END #########

######### ALL ELK STACK AND NGINX INSTALLATION #########

sudo apt-get update
sudo apt update && sudo apt -y upgrade
sudo apt install apt-transport-https software-properties-common wget nano
sudo add-apt-repository ppa:webupd8team/java
sudo apt update
sudo apt install oracle-java8-installer
java -version
## This is version checker. It should be java version "1.8.0_131"
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
sudo apt-get update && sudo apt-get install elasticsearch
sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-geoip
sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install ingest-user-agent
sudo nano /etc/elasticsearch/elasticsearch.yml
# Uncomment >> network.host: localhost
sudo apt install kibana
sudo nano /etc/kibana/kibana.yml
# Uncomment >> server.host: "localhost"
sudo apt install logstash
sudo apt-get update && sudo apt install filebeat
sudo apt-get install nginx
echo "admin:$(openssl passwd -apr1 password)" | sudo tee -a /etc/nginx/htpasswd.kibana
sudo rm /etc/nginx/sites-enabled/default

######### Creating SSL keys #########

cd && mkdir keys && cd keys
openssl genrsa -des3 -passout pass:x -out server.pass.key 2048
openssl rsa -passin pass:x -in server.pass.key -out server.key
rm server.pass.key
openssl req -new -key server.key -out server.csr
openssl x509 -req -sha256 -days 365 -in server.csr -signkey server.key -out server.crt

######### END #########

## Change IP adress and certificates
sudo nano /etc/nginx/sites-available/kibana

# /home/NAMEOFCOMPUTER/keys/server.crt
# /home/NAMEOFCOMPUTER/keys/server.key

######### FILE CONFIGRATION #########

server {
    listen 80 default_server;
    server_name IP_ADRESS;
    return 301 https://$server_name$request_uri;
}

server {
    listen 443 default_server ssl http2;
 
    server_name IP_ADRESS;
 
    ssl_certificate /home/NAMEOFCOMPUTER/keys/server.crt;
    ssl_certificate_key /home/NAMEOFCOMPUTER/keys/server.key;
    ssl_session_cache shared:SSL:10m;
 
    auth_basic "Restricted Access";
    auth_basic_user_file /etc/nginx/htpasswd.kibana;
 
    location / {
        proxy_pass http://localhost:5601;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }
}

######### END #########

sudo ln -s /etc/nginx/sites-available/kibana /etc/nginx/sites-enabled/kibana
sudo nginx -t
## Result should be successfull.

sudo systemctl enable elasticsearch && sudo systemctl enable kibana
sudo systemctl start elasticsearch && sudo systemctl start kibana && sudo systemctl start nginx
sudo systemctl restart elasticsearch && sudo systemctl restart kibana && sudo systemctl restart nginx
sudo systemctl stop elasticsearch && sudo systemctl stop kibana && sudo systemctl stop nginx

#################################### LOGSTASH CONFIGRATION ####################################

sudo mkdir /usr/share/logstash/patterns
cd /usr/share/logstash/patterns && sudo nano nginx.pattern
######### Nginx Pattern #########

NGINX %{IPORHOST:clientip} - %{DATA:user_name} \[%{HTTPDATE:timestamp}] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:status_code} (?:%{NUMBER:bytes}|-) %{NUMBER:request_time} %{NUMBER:upstream_response_time} %{NOTSPACE:referer} %{QS:agent} %{QS:zipcode}

######### END #########

cd && sudo nano nginx-logstash.conf

########################### The Configration File ###########################

input {
    file {
        path =>"/some/path/log.log"
        start_position => "beginning"
    }
}

filter {
    grok {
        match => { "message" => "%{NGINX}" }
    }
    date {
        match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    geoip {
        source => "clientip"
        target => "geoip"
  }

    useragent {
        source => "agent"
        target => "user_agent"
  }
}

output {
    elasticsearch { 
	      hosts => ["localhost:9200"]
        index => "baum"
        document_type => "logs"
    }
}

#################################### END ####################################

# Logstash should run with root auth.
sudo -s
cd && logstash -f nginx-logstash.conf -r

############################################################################################################################

#################################### Filebeat Configurations ###########################################################

cd && sudo nano nginx-filebeat.conf

## nginx-filebeat.conf file

input {

        beats{
                port => "5044"

        }

#    file {
#        path =>"/home/enojen/log.log"
#        start_position => "beginning"
#    }

}
filter {
    grok {
        match => { "message" => "%{NGINX}" }
    }
    date {
        match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    geoip {
        source => "clientip"
        target => "geoip"
  }
    useragent {
        source => "agent"
        target => "user_agent"
  }
}
output {
    elasticsearch {
              hosts => ["localhost:9200"]
        index => "tezfilebeat"
        document_type => "logs"
    }
}



## filebeat.yml

- type: log

  # Change to true to enable this prospector configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /home/enojen/log2.log
    #- c:\programdata\elasticsearch\logs\*
    
#output.elasticsearch:
  # Array of hosts to connect to.
 # hosts: ["localhost:9200"]
 
output.logstash:
  # The Logstash hosts
  hosts: ["localhost:5044"]
  
  
############################# TO RUN #############################
  
sudo logstash -f nginx-logstash.conf -r
sudo filebeat -e -c /etc/filebeat/filebeat.yml  -d "publish"