/hekakafka2file.toml
Last active Aug 29, 2015
Heka Kafka to file
| [hekad] | |
| maxprocs = 32 | |
| [Kafka] | |
| type = "KafkaInput" | |
| topic = "mmbb" | |
| addrs = ["kafka0:9092","kafka1:9092","kafka2:9092"] | |
| decoder = "AccesslogDecoder" | |
| offset_method = "Oldest" | |
| [AccesslogDecoder] | |
| type = "PayloadRegexDecoder" | |
| match_regex = '(?P<local_ip>[^\s]+?)\s+(?P<hostname>[^\s]+?)\s+(?P<private_ip>.*?),?\s*(?P<forwarded>[^\s]+?)\s-\s(?P<Timestamp>\d[^\s]+) \"(?P<method>\w+) (?P<url_path>[^\?]+)(?:\?(?P<query>[^\s]*?)\s+)?HTTP\/(?P<http_version>[^\s]+?)\" (?P<code>\d+) (-|(?P<size>\d+)) (-|(?P<response_time>\d+))\s+(?P<page_id>[^\s]+?)\s+(-|(?P<db>\d+))\s+(-|(?P<java>\d+))\s+(-|(?P<memcache>\d+))\s+(-|(?P<xcache>\d+))\s+(-|(?P<ftp>\d+))\s+(-|(?P<solr>\d+))\s+(-|(?P<redis>\d+))\s+\"-?(?:(http|https):\/\/(?P<referer_host>[^\/]+)\/?\??(?P<referer_query>[^\s]*?))?\" \"(?P<unique_ident>[^\s]+?)\"\s+\"(?P<user_agent>.+)\"' | |
| timestamp_layout= '20060102150405' | |
| [AccesslogDecoder.message_fields] | |
| local_ip = "%local_ip%" | |
| hostname = "%hostname%" | |
| private_ip = "%private_ip%" | |
| forwarded = "%forwarded%" | |
| timestamp = "%Timestamp%" | |
| method = "%method%" | |
| url_path = "%url_path%" | |
| query = "%query%" | |
| http_version = "%http_version%" | |
| code = "%code%" | |
| size = "%size%" | |
| response_time = "%response_time%" | |
| page_id = "%page_id%" | |
| db = "%db%" | |
| java = "%java%" | |
| memcache = "%memcache%" | |
| xcache = "%xcache%" | |
| ftp = "%ftp%" | |
| solr = "%solr%" | |
| redis = "%redis%" | |
| referer_host = "%referer_host%" | |
| referer_query = "%referer_query%" | |
| unique_ident = "%unique_ident%" | |
| user_agent = "%user_agent%" | |
| [ESJsonEncoder] | |
| index = "%{Type}-%{2006.01.02}" | |
| es_index_from_timestamp = true | |
| type_name = "%{Type}" | |
| [BenchOutput] | |
| type = "FileOutput" | |
| message_matcher = "TRUE" | |
| path = "/tmp/heka-output.log" | |
| perm = "666" | |
| flush_count = 100 | |
| flush_operator = "OR" | |
| encoder = "ESJsonEncoder" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment