logstash.conf

input {
  file {
            type => "app"
            path => ["/home/u/archives/urlteam_2021-02-27-21-17-02/goo-gl/______.txt"]
            start_position => "beginning"
            sincedb_path => "/dev/null"
      } 
}


filter {

if [message] =~ /^\s*$/ { drop { }}


  csv {
      separator => "|"
      skip_header => "true"
      columns => ["linkid","url"]
      }
  
mutate { remove_field => [ "message" ]}

if "www.amazon" in [url] { drop { } }
if "shopclues" in [url] { drop { } }
if "http://localhost" in [url] { drop { } }
if "meet/devices" in [url] { drop { } }
if "www.facebook.com" in [url] { drop { } }
if "googlevideo.com" in [url] { drop { } }
if "maps.google.com" in [url] { drop { } }
if "plus.google.com" in [url] { drop { } }
if "imgres?imgurl" in [url] { drop { } }
if "pay.airtel.in" in [url] { drop { } }
if "snaptubeapp.com" in [url] { drop { } }
if "mail.yahoo.com" in [url] { drop { }  }
if "mail.google.com" in [url] { drop { } }
if "news.google.com" in [url] { drop { } }
if "news.yahoo.co" in [url] { drop { } }
if "play.google.com" in [url] { drop { } }
if "google.com/cardboard" in [url] { drop { }
if "google.org" in [url] { drop { } 
if "blogspot" in [url] { drop { } }
if "zuosa.com" in [url] { drop { } }
if "feedproxy.google.com" in [url] { drop { } }
if "surveymonkey.com/r/" in [url] { drop { } }
if "AdminHome?hf=DeviceDetails" in [url] { drop { } }
if "uspehstudio.ru" in [url] { drop { } }
if "//////////////" in [url] { drop { } }
if ".askmepay.com" in [url] { drop { } }
if "www.pdffiller.com" in [url] { drop { } }
if "www.playworld.id" in [url] { drop { } }
if "api.kredivo.com" in [url] { drop { } }
if "-tv.jp" in [url] { drop { } }
if "bcn.cat" in [url] { drop { } }
if ".xyz" in [url] { drop { } }
if "www.youtube.com" in [url] { drop { } }
if "www.booking.com" in [url] { drop { } }
if ".edu" in [url] { drop { } }
if "k12." in [url] { drop { } }
if "juttlog.com" in [url] { drop { } }
if "itunes.apple.com" in [url] { drop { } }
if "moviedown1.net" in [url] { drop { } }
if "rssfeed" in [url] { drop { } }
if "wikipedia" in [url] { drop { } }










}
   

output {

stdout { codec => rubydebug }
elasticsearch { hosts => ["192.168.1.210:9200"] index => "urlhunter" }

 }