Skip to content

Instantly share code, notes, and snippets.

/gist:b892fdb52e250e9430785f52ad6b508a Secret

Created June 19, 2017 18:07
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anonymous/b892fdb52e250e9430785f52ad6b508a to your computer and use it in GitHub Desktop.
Save anonymous/b892fdb52e250e9430785f52ad6b508a to your computer and use it in GitHub Desktop.
Download ZIP
Logstash & Oracle log
Raw
gistfile1.txt
[2017-06-19T10:35:23,923][DEBUG][logstash.runner ] -------- Logstash Settings (* means modified) ---------
[2017-06-19T10:35:23,925][DEBUG][logstash.runner ] *node.name: "ls5-poc" (default: "l5-poc-hostname.com")
[2017-06-19T10:35:23,925][DEBUG][logstash.runner ] *path.config: "/etc/logstash/conf.d"
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] *path.data: "/home/worker/data" (default: "/usr/share/logstash/data")
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] config.test_and_exit: false
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] config.reload.automatic: false
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] config.reload.interval: 3
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] metric.collect: true
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] pipeline.id: "main"
[2017-06-19T10:35:23,926][DEBUG][logstash.runner ] pipeline.system: false
[2017-06-19T10:35:23,927][DEBUG][logstash.runner ] pipeline.workers: 2
[2017-06-19T10:35:23,927][DEBUG][logstash.runner ] pipeline.output.workers: 1
[2017-06-19T10:35:23,927][DEBUG][logstash.runner ] *pipeline.batch.size: 1200 (default: 125)
[2017-06-19T10:35:23,927][DEBUG][logstash.runner ] pipeline.batch.delay: 5
[2017-06-19T10:35:23,927][DEBUG][logstash.runner ] pipeline.unsafe_shutdown: false
[2017-06-19T10:35:23,928][DEBUG][logstash.runner ] path.plugins: []
[2017-06-19T10:35:23,928][DEBUG][logstash.runner ] *config.debug: true (default: false)
[2017-06-19T10:35:23,928][DEBUG][logstash.runner ] *log.level: "debug" (default: "info")
[2017-06-19T10:35:23,928][DEBUG][logstash.runner ] version: false
[2017-06-19T10:35:23,928][DEBUG][logstash.runner ] help: false
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] log.format: "plain"
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] *http.host: "1.2.3.4" (default: "127.0.0.1")
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] *http.port: 9600..9600 (default: 9600..9700)
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] http.environment: "production"
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] queue.type: "memory"
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] queue.drain: false
[2017-06-19T10:35:23,929][DEBUG][logstash.runner ] queue.page_capacity: 262144000
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] queue.max_bytes: 1073741824
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] queue.max_events: 0
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] queue.checkpoint.acks: 1024
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] queue.checkpoint.writes: 1024
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] queue.checkpoint.interval: 1000
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] slowlog.threshold.warn: -1
[2017-06-19T10:35:23,930][DEBUG][logstash.runner ] slowlog.threshold.info: -1
[2017-06-19T10:35:23,931][DEBUG][logstash.runner ] slowlog.threshold.debug: -1
[2017-06-19T10:35:23,931][DEBUG][logstash.runner ] slowlog.threshold.trace: -1
[2017-06-19T10:35:23,931][DEBUG][logstash.runner ] *path.queue: "/home/worker/data/queue" (default: "/usr/share/logstash/data/queue")
[2017-06-19T10:35:23,931][DEBUG][logstash.runner ] *path.settings: "/etc/logstash" (default: "/usr/share/logstash/config")
[2017-06-19T10:35:23,931][DEBUG][logstash.runner ] *path.logs: "/home/worker/logs" (default: "/usr/share/logstash/logs")
[2017-06-19T10:35:23,931][DEBUG][logstash.runner ] --------------- Logstash Settings -------------------
[2017-06-19T10:35:23,946][DEBUG][logstash.agent ] Agent: Configuring metric collection
[2017-06-19T10:35:23,948][DEBUG][logstash.instrument.periodicpoller.os] PeriodicPoller: Starting {:polling_interval=>5, :polling_timeout=>120}
[2017-06-19T10:35:23,966][DEBUG][logstash.instrument.periodicpoller.jvm] PeriodicPoller: Starting {:polling_interval=>5, :polling_timeout=>120}
[2017-06-19T10:35:23,996][DEBUG][logstash.instrument.periodicpoller.persistentqueue] PeriodicPoller: Starting {:polling_interval=>5, :polling_timeout=>120}
[2017-06-19T10:35:24,004][DEBUG][logstash.agent ] Reading config file {:config_file=>"/etc/logstash/conf.d/oracledb-input-01.conf"}
[2017-06-19T10:35:24,004][DEBUG][logstash.agent ]
The following is the content of a file {:config_file=>"/etc/logstash/conf.d/oracledb-input-01.conf"}
[2017-06-19T10:35:24,004][DEBUG][logstash.agent ]
input {
jdbc {
id => "es-poc-input-oracledb"
connection_retry_attempts => "10"
connection_retry_attempts_wait_time => "5"
jdbc_driver_library => "/usr/share/logstash/ojdbc7.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
jdbc_connection_string => "jdbc:oracle:thin:@database.hostname.com:1521/oracledb"
jdbc_user => "db_user"
jdbc_password => "db_user"
schedule => "* * * * *"
statement => "SELECT eventid, id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, id11, id12, id13, id14, id15, id16, id17, id18, id19, id20, id21, id22, id23, id24, id25, id26, id27, id28, id29, id30, id31, id32, id33, id34, id35, id36, id37, id38, id39, id40 FROM TABLENAME WHERE eventid > :sql_last_value"
use_column_value => true
tracking_column => eventid
last_run_metadata_path => "/home/worker/.logstash_jdbc_last_run"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
clean_run => "false"
record_last_run => "true"
sql_log_level => "debug"
}
}
output {
elasticsearch {
id => "es5-poc-output-01"
hosts => ["es5-endpoint.hostname.com:80"]
index => "details%{+YYYY.MM.dd}"
document_type => "errordetail"
}
stdout {
}
}
[2017-06-19T10:35:24,004][DEBUG][logstash.agent ]
The following is the merged configuration
[2017-06-19T10:35:24,005][DEBUG][logstash.agent ]
input {
jdbc {
id => "es-poc-input-oracledb"
connection_retry_attempts => "10"
connection_retry_attempts_wait_time => "5"
jdbc_driver_library => "/usr/share/logstash/ojdbc7.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
jdbc_connection_string => "jdbc:oracle:thin:@database.hostname.com:1521/oracledb"
jdbc_user => "db_user"
jdbc_password => "db_user"
schedule => "* * * * *"
statement => "SELECT eventid, id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, id11, id12, id13, id14, id15, id16, id17, id18, id19, id20, id21, id22, id23, id24, id25, id26, id27, id28, id29, id30, id31, id32, id33, id34, id35, id36, id37, id38, id39, id40 FROM TABLENAME WHERE eventid > :sql_last_value"
use_column_value => true
tracking_column => eventid
last_run_metadata_path => "/home/worker/.logstash_jdbc_last_run"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
clean_run => "false"
record_last_run => "true"
sql_log_level => "debug"
}
}
output {
elasticsearch {
id => "es5-poc-output-01"
hosts => ["es5-endpoint.hostname.com:80"]
index => "details%{+YYYY.MM.dd}"
document_type => "errordetail"
}
stdout {
}
}
[2017-06-19T10:35:24,096][DEBUG][logstash.pipeline ] Compiled pipeline code {:code=>" @inputs = []\n @filters = []\n @outputs = []\n @periodic_flushers = []\n @shutdown_flushers = []\n @generated_objects = {}\n\n @generated_objects[:input_jdbc_1] = plugin(\"input\", \"jdbc\", LogStash::Util.hash_merge_many({ \"id\" => (\"es-poc-input-oracledb\") }, { \"connection_retry_attempts\" => (\"10\") }, { \"connection_retry_attempts_wait_time\" => (\"5\") }, { \"jdbc_driver_library\" => (\"/usr/share/logstash/ojdbc7.jar\") }, { \"jdbc_driver_class\" => (\"Java::oracle.jdbc.driver.OracleDriver\") }, { \"jdbc_connection_string\" => (\"jdbc:oracle:thin:@database.hostname.com:1521/oracledb\") }, { \"jdbc_user\" => (\"db_user\") }, { \"jdbc_password\" => (\"db_user\") }, { \"schedule\" => (\"* * * * *\") }, { \"statement\" => (\"SELECT eventid, id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, id11, id12, id13, id14, id15, id16, id17, id18, id19, id20, id21, id22, id23, id24, id25, id26, id27, id28, id29, id30, id31, id32, id33, id34, id35, id36, id37, id38, id39, id40 FROM TABLENAME WHERE eventid > :sql_last_value\") }, { \"use_column_value\" => (\"true\") }, { \"tracking_column\" => (\"eventid\") }, { \"last_run_metadata_path\" => (\"/home/worker/.logstash_jdbc_last_run\") }, { \"jdbc_paging_enabled\" => (\"true\") }, { \"jdbc_page_size\" => (\"50000\") }, { \"clean_run\" => (\"false\") }, { \"record_last_run\" => (\"true\") }, { \"sql_log_level\" => (\"debug\") }))\n\n @inputs << @generated_objects[:input_jdbc_1]\n\n @generated_objects[:output_elasticsearch_2] = plugin(\"output\", \"elasticsearch\", LogStash::Util.hash_merge_many({ \"id\" => (\"es5-poc-output-01\") }, { \"hosts\" => [(\"es5-endpoint.hostname.com:80\")] }, { \"index\" => (\"details%{+YYYY.MM.dd}\") }, { \"document_type\" => (\"errordetail\") }))\n\n @outputs << @generated_objects[:output_elasticsearch_2]\n\n @generated_objects[:output_stdout_3] = plugin(\"output\", \"stdout\")\n\n @outputs << @generated_objects[:output_stdout_3]\n\n define_singleton_method :filter_func do |event|\n events = [event]\n @logger.debug? && @logger.debug(\"filter received\", \"event\" => event.to_hash)\n events\n end\n define_singleton_method :output_func do |event|\n targeted_outputs = []\n @logger.debug? && @logger.debug(\"output received\", \"event\" => event.to_hash)\n targeted_outputs << @generated_objects[:output_elasticsearch_2]\n targeted_outputs << @generated_objects[:output_stdout_3]\n \n targeted_outputs\n end"}
[2017-06-19T10:35:24,112][DEBUG][logstash.plugins.registry] On demand adding plugin to the registry {:name=>"jdbc", :type=>"input", :class=>LogStash::Inputs::Jdbc}
[2017-06-19T10:35:24,122][DEBUG][logstash.plugins.registry] On demand adding plugin to the registry {:name=>"plain", :type=>"codec", :class=>LogStash::Codecs::Plain}
[2017-06-19T10:35:24,124][DEBUG][logstash.codecs.plain ] config LogStash::Codecs::Plain/@id = "plain_5556ba41-02d4-45b7-be3a-6786239f3df3"
[2017-06-19T10:35:24,124][DEBUG][logstash.codecs.plain ] config LogStash::Codecs::Plain/@enable_metric = true
[2017-06-19T10:35:24,124][DEBUG][logstash.codecs.plain ] config LogStash::Codecs::Plain/@charset = "UTF-8"
[2017-06-19T10:35:24,126][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@id = "es-poc-input-oracledb"
[2017-06-19T10:35:24,126][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@connection_retry_attempts = 10
[2017-06-19T10:35:24,126][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@connection_retry_attempts_wait_time = 5
[2017-06-19T10:35:24,127][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_driver_library = "/usr/share/logstash/ojdbc7.jar"
[2017-06-19T10:35:24,127][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_driver_class = "Java::oracle.jdbc.driver.OracleDriver"
[2017-06-19T10:35:24,127][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_connection_string = "jdbc:oracle:thin:@database.hostname.com:1521/oracledb"
[2017-06-19T10:35:24,127][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_user = "db_user"
[2017-06-19T10:35:24,127][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_password = <password>
[2017-06-19T10:35:24,127][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@schedule = "* * * * *"
[2017-06-19T10:35:24,128][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@statement = "SELECT eventid, id1, id2, id3, id4, id5, id6, id7, id8, id9, id10, id11, id12, id13, id14, id15, id16, id17, id18, id19, id20, id21, id22, id23, id24, id25, id26, id27, id28, id29, id30, id31, id32, id33, id34, id35, id36, id37, id38, id39, id40 FROM TABLENAME WHERE eventid > :sql_last_value"
[2017-06-19T10:35:24,129][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@use_column_value = true
[2017-06-19T10:35:24,129][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@tracking_column = "eventid"
[2017-06-19T10:35:24,129][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@last_run_metadata_path = "/home/worker/.logstash_jdbc_last_run"
[2017-06-19T10:35:24,129][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_paging_enabled = true
[2017-06-19T10:35:24,129][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_page_size = 50000
[2017-06-19T10:35:24,130][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@clean_run = false
[2017-06-19T10:35:24,131][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@record_last_run = true
[2017-06-19T10:35:24,131][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@sql_log_level = "debug"
[2017-06-19T10:35:24,132][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@enable_metric = true
[2017-06-19T10:35:24,132][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@codec = <LogStash::Codecs::Plain id=>"plain_5556ba41-02d4-45b7-be3a-6786239f3df3", enable_metric=>true, charset=>"UTF-8">
[2017-06-19T10:35:24,132][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@add_field = {}
[2017-06-19T10:35:24,132][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_validate_connection = false
[2017-06-19T10:35:24,132][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_validation_timeout = 3600
[2017-06-19T10:35:24,132][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@jdbc_pool_timeout = 5
[2017-06-19T10:35:24,133][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@sequel_opts = {}
[2017-06-19T10:35:24,133][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@parameters = {}
[2017-06-19T10:35:24,133][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@tracking_column_type = "numeric"
[2017-06-19T10:35:24,133][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@lowercase_column_names = true
[2017-06-19T10:35:24,133][DEBUG][logstash.inputs.jdbc ] config LogStash::Inputs::Jdbc/@columns_charset = {}
[2017-06-19T10:35:24,238][DEBUG][logstash.plugins.registry] On demand adding plugin to the registry {:name=>"elasticsearch", :type=>"output", :class=>LogStash::Outputs::ElasticSearch}
[2017-06-19T10:35:24,260][DEBUG][logstash.codecs.plain ] config LogStash::Codecs::Plain/@id = "plain_76452dfc-c8fd-47db-961a-c694853183a9"
[2017-06-19T10:35:24,260][DEBUG][logstash.codecs.plain ] config LogStash::Codecs::Plain/@enable_metric = true
[2017-06-19T10:35:24,260][DEBUG][logstash.codecs.plain ] config LogStash::Codecs::Plain/@charset = "UTF-8"
[2017-06-19T10:35:24,269][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@id = "es5-poc-output-01"
[2017-06-19T10:35:24,269][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@hosts = [//es5-endpoint.hostname.com:80]
[2017-06-19T10:35:24,269][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@index = "details%{+YYYY.MM.dd}"
[2017-06-19T10:35:24,269][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@document_type = "errordetail"
[2017-06-19T10:35:24,270][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@enable_metric = true
[2017-06-19T10:35:24,270][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@codec = <LogStash::Codecs::Plain id=>"plain_76452dfc-c8fd-47db-961a-c694853183a9", enable_metric=>true, charset=>"UTF-8">
[2017-06-19T10:35:24,270][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@workers = 1
[2017-06-19T10:35:24,270][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@manage_template = true
[2017-06-19T10:35:24,270][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@template_name = "logstash"
[2017-06-19T10:35:24,270][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@template_overwrite = false
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@parent = nil
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@idle_flush_time = 1
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@upsert = ""
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@doc_as_upsert = false
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@script = ""
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@script_type = "inline"
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@script_lang = "painless"
[2017-06-19T10:35:24,271][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@script_var_name = "event"
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@scripted_upsert = false
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@retry_initial_interval = 2
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@retry_max_interval = 64
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@retry_on_conflict = 1
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@pipeline = nil
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@action = "index"
[2017-06-19T10:35:24,272][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@ssl_certificate_verification = true
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@sniffing = false
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@sniffing_delay = 5
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@timeout = 60
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@failure_type_logging_whitelist = []
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@pool_max = 1000
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@pool_max_per_route = 100
[2017-06-19T10:35:24,273][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@resurrect_delay = 5
[2017-06-19T10:35:24,274][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@validate_after_inactivity = 10000
[2017-06-19T10:35:24,274][DEBUG][logstash.outputs.elasticsearch] config LogStash::Outputs::ElasticSearch/@http_compression = false
[2017-06-19T10:35:24,285][DEBUG][logstash.plugins.registry] On demand adding plugin to the registry {:name=>"stdout", :type=>"output", :class=>LogStash::Outputs::Stdout}
[2017-06-19T10:35:24,295][DEBUG][logstash.plugins.registry] On demand adding plugin to the registry {:name=>"line", :type=>"codec", :class=>LogStash::Codecs::Line}
[2017-06-19T10:35:24,298][DEBUG][logstash.codecs.line ] config LogStash::Codecs::Line/@id = "line_ae44f291-6873-4720-b0fb-68bc0794a6a6"
[2017-06-19T10:35:24,298][DEBUG][logstash.codecs.line ] config LogStash::Codecs::Line/@enable_metric = true
[2017-06-19T10:35:24,299][DEBUG][logstash.codecs.line ] config LogStash::Codecs::Line/@charset = "UTF-8"
[2017-06-19T10:35:24,299][DEBUG][logstash.codecs.line ] config LogStash::Codecs::Line/@delimiter = "\n"
[2017-06-19T10:35:24,303][DEBUG][logstash.outputs.stdout ] config LogStash::Outputs::Stdout/@id = "a813bb0ada35db92ed1e970b31f1f36911bd412f-3"
[2017-06-19T10:35:24,303][DEBUG][logstash.outputs.stdout ] config LogStash::Outputs::Stdout/@enable_metric = true
[2017-06-19T10:35:24,303][DEBUG][logstash.outputs.stdout ] config LogStash::Outputs::Stdout/@codec = <LogStash::Codecs::Line id=>"line_ae44f291-6873-4720-b0fb-68bc0794a6a6", enable_metric=>true, charset=>"UTF-8", delimiter=>"\n">
[2017-06-19T10:35:24,303][DEBUG][logstash.outputs.stdout ] config LogStash::Outputs::Stdout/@workers = 1
[2017-06-19T10:35:24,307][DEBUG][logstash.agent ] starting agent
[2017-06-19T10:35:24,309][DEBUG][logstash.agent ] starting pipeline {:id=>"main"}
[2017-06-19T10:35:24,316][DEBUG][logstash.outputs.elasticsearch] Normalizing http path {:path=>nil, :normalized=>nil}
[2017-06-19T10:35:24,551][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://es5-endpoint.hostname.com/]}}
[2017-06-19T10:35:24,551][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://es5-endpoint.hostname.com/, :path=>"/"}
[2017-06-19T10:35:24,635][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>#<URI::HTTP:0xe64eff1 URL:http://es5-endpoint.hostname.com/>}
[2017-06-19T10:35:24,636][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2017-06-19T10:35:24,675][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"_all"=>{"enabled"=>true, "norms"=>false}, "dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword"}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date", "include_in_all"=>false}, "@version"=>{"type"=>"keyword", "include_in_all"=>false}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2017-06-19T10:35:24,688][DEBUG][logstash.outputs.elasticsearch] Found existing Elasticsearch template. Skipping template management {:name=>"logstash"}
[2017-06-19T10:35:24,689][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>[#<URI::Generic:0x79756d6d URL://es5-endpoint.hostname.com:80>]}
[2017-06-19T10:35:24,691][INFO ][logstash.pipeline ] Starting pipeline {"id"=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>1200, "pipeline.batch.delay"=>5, "pipeline.max_inflight"=>2400}
[2017-06-19T10:35:24,816][INFO ][logstash.pipeline ] Pipeline main started
[2017-06-19T10:35:24,841][DEBUG][logstash.agent ] Starting puma
[2017-06-19T10:35:24,847][DEBUG][logstash.agent ] Trying to start WebServer {:port=>9600}
[2017-06-19T10:35:24,848][DEBUG][logstash.api.service ] [api-service] start
[2017-06-19T10:35:24,880][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2017-06-19T10:35:29,820][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:35:34,820][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:35:39,820][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:35:44,819][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:35:49,820][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:35:54,819][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:35:59,819][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:04,818][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:09,819][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:14,818][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:19,818][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:24,817][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:29,816][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:34,816][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:39,815][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:44,815][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:49,814][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:54,813][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:36:59,813][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:37:03,098][WARN ][logstash.runner ] SIGINT received. Shutting down the agent.
[2017-06-19T10:37:03,104][DEBUG][logstash.instrument.periodicpoller.os] PeriodicPoller: Stopping
[2017-06-19T10:37:03,105][DEBUG][logstash.instrument.periodicpoller.jvm] PeriodicPoller: Stopping
[2017-06-19T10:37:03,105][DEBUG][logstash.instrument.periodicpoller.persistentqueue] PeriodicPoller: Stopping
[2017-06-19T10:37:03,108][WARN ][logstash.agent ] stopping pipeline {:id=>"main"}
[2017-06-19T10:37:03,110][DEBUG][logstash.pipeline ] Closing inputs
[2017-06-19T10:37:03,110][DEBUG][logstash.inputs.jdbc ] stopping {:plugin=>"LogStash::Inputs::Jdbc"}
[2017-06-19T10:37:03,111][DEBUG][logstash.pipeline ] Closed inputs
[2017-06-19T10:37:03,366][DEBUG][logstash.inputs.jdbc ] closing {:plugin=>"LogStash::Inputs::Jdbc"}
[2017-06-19T10:37:03,367][DEBUG][logstash.pipeline ] Input plugins stopped! Will shutdown filter/output workers.
[2017-06-19T10:37:03,449][DEBUG][logstash.pipeline ] Pushing flush onto pipeline
[2017-06-19T10:37:03,449][DEBUG][logstash.pipeline ] Pushing shutdown {:thread=>"#<Thread:0x495c527a run>"}
[2017-06-19T10:37:03,450][DEBUG][logstash.pipeline ] Pushing shutdown {:thread=>"#<Thread:0x4c37023e sleep>"}
[2017-06-19T10:37:03,450][DEBUG][logstash.pipeline ] Shutdown waiting for worker thread #<Thread:0x495c527a>
[2017-06-19T10:37:03,473][DEBUG][logstash.pipeline ] Shutdown waiting for worker thread #<Thread:0x4c37023e>
[2017-06-19T10:37:03,474][DEBUG][logstash.outputs.elasticsearch] closing {:plugin=>"LogStash::Outputs::ElasticSearch"}
[2017-06-19T10:37:03,474][DEBUG][logstash.outputs.elasticsearch] Stopping sniffer
[2017-06-19T10:37:03,474][DEBUG][logstash.outputs.elasticsearch] Stopping resurrectionist
[2017-06-19T10:37:03,656][DEBUG][logstash.outputs.elasticsearch] Waiting for in use manticore connections
[2017-06-19T10:37:03,657][DEBUG][logstash.outputs.elasticsearch] Closing adapter #<LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter:0x3e750687>
[2017-06-19T10:37:03,657][DEBUG][logstash.outputs.stdout ] closing {:plugin=>"LogStash::Outputs::Stdout"}
[2017-06-19T10:37:03,658][DEBUG][logstash.pipeline ] Pipeline main has been shutdown
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment