gistfile1.txt

[hekad]
share_dir = "/usr/share/heka"
pid_file = "/var/run/hekad.pid"
maxprocs = 4
poolsize = 500

[DashboardOutput]
ticker_interval = 5
[ApiInput]
type = "UdpInput"
address = "127.0.0.1:5566"

[ApiEncoder]
type = "SandboxEncoder"
filename = "lua_encoders/es_payload.lua"

	[ApiEncoder.config]
	index = "log_%{%Y.%m.%d}"
	type_name = "api-dispatch"
	es_index_from_timestamp = true

[ApiOuput]
type="ElasticSearchOutput"
server = "http://xxx:9200"
message_matcher = "Logger == 'ApiInput'"
encoder = "ApiEncoder"
use_buffering = false
flush_count = 2500
flush_interval = 500
http_timeout = 1500
connect_timeout = 1500

[NginxAccess]
type = "LogstreamerInput"
log_directory = "/var/log/nginx"
file_match = 'access_log'
decoder = "NginxAccessDecoder"
oldest_duration = "15m"

[NginxAccessDecoder]
type = "SandboxDecoder"
filename = "lua_decoders/nginx_access.lua"

	[NginxAccessDecoder.config]
	type = "nginx.access"
	user_agent_transform = true
	log_format = '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$gzip_ratio"'

[NginxError]
type = "LogstreamerInput"
log_directory = "/var/log/nginx"
file_match = 'error_log'
decoder = "NginxErrorDecoder"
oldest_duration = "15m"

[NginxErrorDecoder]
type = "SandboxDecoder"
filename = "lua_decoders/nginx_error.lua"

	[NginxErrorDecoder.config]

[NginxEncoder]
type = "ESJsonEncoder"
index = "%{Type}-%{2006.01.02}"
es_index_from_timestamp = true
type_name = "%{Type}"

[NginxOutput]
type = "ElasticSearchOutput"
server = "http://xxx:9200"
message_matcher = "Type == 'nginx.access' || Type == 'nginx.error'"
encoder = 'NginxEncoder'
use_buffering = false
flush_interval = 500
http_timeout = 1500
connect_timeout = 1500

ls -l /var/cache/hekad/
total 12
drwx------ 7 root root 4096 Apr 27 17:25 dashboard
drwxr--r-- 2 root root 4096 Apr 27 17:25 logstreamer
drwx------ 2 root root 4096 Jan 13 15:05 sandbox_preservation

A second or two of strace:

strace -p 29401
Process 29401 attached
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
futex(0xc2080ce458, FUTEX_WAKE, 1) = 1
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
write(2, "2015/06/22 10:37:18 Plugin 'ApiO"..., 140) = 140
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
futex(0xc2080cf958, FUTEX_WAKE, 1) = 1
epoll_wait(4, {}, 128, 0) = 0
epoll_wait(4, {{EPOLLIN|EPOLLOUT, {u32=2903057632, u64=139778318738656}}}, 128, -1) = 1
futex(0xc2080cf958, FUTEX_WAKE, 1) = 1
read(3, "{"timestamp":"2015-06-22T14:37:1"..., 65536) = 240
futex(0xc2080cf258, FUTEX_WAKE, 1) = 1
read(3, "{"timestamp":"2015-06-22T14:37:1"..., 65536) = 218
futex(0xc2080ce458, FUTEX_WAKE, 1) = 1
read(3, "{"timestamp":"2015-06-22T14:37:1"..., 65536) = 223
futex(0xc2080cf258, FUTEX_WAKE, 1) = 1
read(3, "{"timestamp":"2015-06-22T14:37:1"..., 65536) = 221
futex(0xc2080ce458, FUTEX_WAKE, 1) = 1
read(3, 0xc2853d8000, 65536) = -1 EAGAIN (Resource temporarily unavailable)
epoll_wait(4, {}, 128, 0) = 0
futex(0xf98c18, FUTEX_WAIT, 0, NULL) = 0
futex(0xc2080cf258, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 5360413}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080ce0d8, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 197846330}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080cf958, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 1358126}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080cf258, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 44558271}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080cf958, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 5190666}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080cf958, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 244335935}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080ce458, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 5176833}) = -1 ETIMEDOUT (Connection timed out)
futex(0xf980e0, FUTEX_WAKE, 1) = 1
futex(0xf98060, FUTEX_WAKE, 1) = 1
futex(0xc2080ce458, FUTEX_WAKE, 1) = 1
futex(0xf97858, FUTEX_WAIT, 0, {0, 197901143}^CProcess 29401 detached

2015/06/29 10:59:48 Plugin 'ApiOuput' error: FATAL: process_message() /usr/share/heka/lua_encoders/es_payload.lua:79: output_limit exceeded