logstash-debug

gistfile1.txt

root@local-dev-box:/usr/share/kibana# bin/logstash --modules netflow --setup --log.level=debug
-bash: bin/logstash: No such file or directory
root@local-dev-box:/usr/share/kibana# cd ..
root@local-dev-box:/usr/share# cd logstash/
root@local-dev-box:/usr/share/logstash# bin/logstash --modules netflow --setup --log.level=debug
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[DEBUG] 2018-10-09 10:28:34.440 [main] scaffold - Found module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[DEBUG] 2018-10-09 10:28:34.446 [main] registry - Adding plugin to the registry {:name=>"fb_apache", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x95d80b1 @directory="/usr/share/logstash/modules/fb_apache/configuration", @module_name="fb_apache", @kibana_version_parts=["6", "0", "0"]>}
[DEBUG] 2018-10-09 10:28:34.450 [main] scaffold - Found module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[DEBUG] 2018-10-09 10:28:34.450 [main] registry - Adding plugin to the registry {:name=>"netflow", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x13494cca @directory="/usr/share/logstash/modules/netflow/configuration", @module_name="netflow", @kibana_version_parts=["6", "0", "0"]>}
[DEBUG] 2018-10-09 10:28:35.089 [LogStash::Runner] runner - -------- Logstash Settings (* means modified) ---------
[DEBUG] 2018-10-09 10:28:35.090 [LogStash::Runner] runner - node.name: "local-dev-box"
[DEBUG] 2018-10-09 10:28:35.090 [LogStash::Runner] runner - path.data: "/usr/share/logstash/data"
[DEBUG] 2018-10-09 10:28:35.090 [LogStash::Runner] runner - *modules.cli: [{"name"=>"netflow"}] (default: [])
[DEBUG] 2018-10-09 10:28:35.090 [LogStash::Runner] runner - modules: []
[DEBUG] 2018-10-09 10:28:35.090 [LogStash::Runner] runner - modules_list: ["netflow"]
[DEBUG] 2018-10-09 10:28:35.090 [LogStash::Runner] runner - modules_variable_list: []
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - *modules_setup: true (default: false)
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - config.test_and_exit: false
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - config.reload.automatic: false
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - config.reload.interval: 3000000000
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - config.support_escapes: false
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - config.field_reference.parser: "COMPAT"
[DEBUG] 2018-10-09 10:28:35.091 [LogStash::Runner] runner - metric.collect: true
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.id: "main"
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.system: false
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.workers: 4
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.output.workers: 1
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.batch.size: 125
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.batch.delay: 50
[DEBUG] 2018-10-09 10:28:35.092 [LogStash::Runner] runner - pipeline.unsafe_shutdown: false
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - pipeline.java_execution: false
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - pipeline.reloadable: true
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - path.plugins: []
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - config.debug: false
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - *log.level: "debug" (default: "info")
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - version: false
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - help: false
[DEBUG] 2018-10-09 10:28:35.093 [LogStash::Runner] runner - log.format: "plain"
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - http.host: "127.0.0.1"
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - http.port: 9600..9700
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - http.environment: "production"
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - queue.type: "memory"
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - queue.drain: false
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - queue.page_capacity: 67108864
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - queue.max_bytes: 1073741824
[DEBUG] 2018-10-09 10:28:35.094 [LogStash::Runner] runner - queue.max_events: 0
[DEBUG] 2018-10-09 10:28:35.095 [LogStash::Runner] runner - queue.checkpoint.acks: 1024
[DEBUG] 2018-10-09 10:28:35.095 [LogStash::Runner] runner - queue.checkpoint.writes: 1024
[DEBUG] 2018-10-09 10:28:35.095 [LogStash::Runner] runner - queue.checkpoint.interval: 1000
[DEBUG] 2018-10-09 10:28:35.095 [LogStash::Runner] runner - dead_letter_queue.enable: false
[DEBUG] 2018-10-09 10:28:35.095 [LogStash::Runner] runner - dead_letter_queue.max_bytes: 1073741824
[DEBUG] 2018-10-09 10:28:35.095 [LogStash::Runner] runner - slowlog.threshold.warn: -1
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - slowlog.threshold.info: -1
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - slowlog.threshold.debug: -1
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - slowlog.threshold.trace: -1
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - keystore.classname: "org.logstash.secret.store.backend.JavaKeyStore"
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - keystore.file: "/usr/share/logstash/config/logstash.keystore"
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - path.queue: "/usr/share/logstash/data/queue"
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - path.dead_letter_queue: "/usr/share/logstash/data/dead_letter_queue"
[DEBUG] 2018-10-09 10:28:35.096 [LogStash::Runner] runner - path.settings: "/usr/share/logstash/config"
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - path.logs: "/usr/share/logstash/logs"
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.enabled: false
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.logstash.poll_interval: 5000000000
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.pipeline.id: ["main"]
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.elasticsearch.username: "logstash_system"
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.elasticsearch.url: ["https://localhost:9200"]
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.elasticsearch.ssl.verification_mode: "certificate"
[DEBUG] 2018-10-09 10:28:35.097 [LogStash::Runner] runner - xpack.management.elasticsearch.sniffing: false
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.enabled: false
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.url: ["http://localhost:9200"]
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.collection.interval: 10000000000
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.collection.timeout_interval: 600000000000
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.username: "logstash_system"
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.ssl.verification_mode: "certificate"
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.sniffing: false
[DEBUG] 2018-10-09 10:28:35.098 [LogStash::Runner] runner - xpack.monitoring.collection.pipeline.details.enabled: true
[DEBUG] 2018-10-09 10:28:35.099 [LogStash::Runner] runner - xpack.monitoring.collection.config.enabled: true
[DEBUG] 2018-10-09 10:28:35.099 [LogStash::Runner] runner - node.uuid: ""
[DEBUG] 2018-10-09 10:28:35.099 [LogStash::Runner] runner - --------------- Logstash Settings -------------------
[WARN ] 2018-10-09 10:28:35.166 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[DEBUG] 2018-10-09 10:28:35.260 [LogStash::Runner] agent - Setting global FieldReference parsing mode: COMPAT
[DEBUG] 2018-10-09 10:28:35.287 [LogStash::Runner] agent - Setting up metric collection
[DEBUG] 2018-10-09 10:28:35.418 [LogStash::Runner] os - Starting {:polling_interval=>5, :polling_timeout=>120}
[DEBUG] 2018-10-09 10:28:35.757 [LogStash::Runner] jvm - Starting {:polling_interval=>5, :polling_timeout=>120}
[DEBUG] 2018-10-09 10:28:35.930 [LogStash::Runner] jvm - collector name {:name=>"ParNew"}
[DEBUG] 2018-10-09 10:28:35.941 [LogStash::Runner] jvm - collector name {:name=>"ConcurrentMarkSweep"}
[DEBUG] 2018-10-09 10:28:35.976 [LogStash::Runner] persistentqueue - Starting {:polling_interval=>5, :polling_timeout=>120}
[DEBUG] 2018-10-09 10:28:35.987 [LogStash::Runner] deadletterqueue - Starting {:polling_interval=>5, :polling_timeout=>120}
[INFO ] 2018-10-09 10:28:36.057 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.4.2"}
[DEBUG] 2018-10-09 10:28:36.091 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - Starting agent
[DEBUG] 2018-10-09 10:28:36.203 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] modulescommon - Specified modules {:modules_array=>"[{\"name\"=>\"netflow\"}]"}
[INFO ] 2018-10-09 10:28:36.289 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] modulescommon - Setting up the netflow module
[DEBUG] 2018-10-09 10:28:41.047 [pool-2-thread-2] jvm - collector name {:name=>"ParNew"}
[DEBUG] 2018-10-09 10:28:41.052 [pool-2-thread-2] jvm - collector name {:name=>"ConcurrentMarkSweep"}
[ERROR] 2018-10-09 10:28:42.258 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] kibanaclient - Error when executing Kibana client request {:error=>#<Manticore::ConnectTimeout: Read timed out>}
[DEBUG] 2018-10-09 10:28:46.062 [pool-2-thread-2] jvm - collector name {:name=>"ParNew"}
[DEBUG] 2018-10-09 10:28:46.062 [pool-2-thread-2] jvm - collector name {:name=>"ConcurrentMarkSweep"}
[ERROR] 2018-10-09 10:28:47.449 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] kibanaclient - Error when executing Kibana client request {:error=>#<Manticore::ConnectTimeout: Read timed out>}
[ERROR] 2018-10-09 10:28:47.642 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] sourceloader - Could not fetch all the sources {:exception=>LogStash::ConfigLoadingError, :message=>"Failed to import module configurations to Elasticsearch and/or Kibana. Module: netflow has Elasticsearch hosts: [\"localhost:9200\"] and Kibana hosts: [\"localhost:5601\"]", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/config/modules_common.rb:108:in `block in pipeline_configs'", "org/jruby/RubyArray.java:1734:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/config/modules_common.rb:54:in `pipeline_configs'", "/usr/share/logstash/logstash-core/lib/logstash/config/source/modules.rb:14:in `pipeline_configs'", "/usr/share/logstash/logstash-core/lib/logstash/config/source_loader.rb:61:in `block in fetch'", "org/jruby/RubyArray.java:2481:in `collect'", "/usr/share/logstash/logstash-core/lib/logstash/config/source_loader.rb:60:in `fetch'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:142:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:93:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:362:in `block in execute'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'"]}
[ERROR] 2018-10-09 10:28:47.653 [Ruby-0-Thread-1: /usr/share/logstash/lib/bootstrap/environment.rb:6] agent - An exception happened when converging configuration {:exception=>RuntimeError, :message=>"Could not fetch the configuration, message: Failed to import module configurations to Elasticsearch and/or Kibana. Module: netflow has Elasticsearch hosts: [\"localhost:9200\"] and Kibana hosts: [\"localhost:5601\"]", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/agent.rb:149:in `converge_state_and_update'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:93:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:362:in `block in execute'", "/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:24:in `block in initialize'"]}
[DEBUG] 2018-10-09 10:28:47.707 [Api Webserver] agent - Starting puma
[DEBUG] 2018-10-09 10:28:47.741 [Api Webserver] agent - Trying to start WebServer {:port=>9600}
[DEBUG] 2018-10-09 10:28:47.751 [LogStash::Runner] os - Stopping
[DEBUG] 2018-10-09 10:28:47.872 [LogStash::Runner] jvm - Stopping
[DEBUG] 2018-10-09 10:28:47.876 [LogStash::Runner] persistentqueue - Stopping
[DEBUG] 2018-10-09 10:28:47.877 [LogStash::Runner] deadletterqueue - Stopping
[DEBUG] 2018-10-09 10:28:47.912 [Api Webserver] service - [api-service] start
[INFO ] 2018-10-09 10:28:48.110 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9600}
[DEBUG] 2018-10-09 10:28:53.082 [LogStash::Runner] agent - Shutting down all pipelines {:pipelines_count=>0}
[DEBUG] 2018-10-09 10:28:53.111 [LogStash::Runner] agent - Converging pipelines state {:actions_count=>0}