https://discuss.elastic.co/t/filebeat-harvester-never-stops-running-and-doesnt-detect-file-changes/77069

filebeat.stack

SIGQUIT: quit
PC=0x45dee7 m=1
goroutine 0 [idle]:
runtime.usleep(0x2710, 0x14a9f1ba34c694ec, 0x44048ad9d9a6, 0x45d964b801, 0x2710004286fc, 0x14a9f725455d6711, 0x45d964b800, 0x6b, 0x44048ad9d9a6, 0x0)
    /usr/local/go/src/runtime/sys_linux_amd64.s:95 +0x37
runtime.sysmon()
    /usr/local/go/src/runtime/proc.go:3606 +0x92
runtime.mstart1()
    /usr/local/go/src/runtime/proc.go:1126 +0x11e
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1096 +0x64
goroutine 1 [chan receive, 269 minutes]:
github.com/elastic/beats/filebeat/beater.(*Filebeat).Run(0xc4201611d0, 0xc42008ec60, 0x0, 0x0)
    /go/src/github.com/elastic/beats/filebeat/beater/filebeat.go:138 +0xb9f
github.com/elastic/beats/libbeat/beat.(*Beat).launch(0xc42008ec60, 0x9da250, 0x0, 0x0)
    /go/src/github.com/elastic/beats/libbeat/beat/beat.go:211 +0x699
github.com/elastic/beats/libbeat/beat.Run(0x992101, 0x8, 0x0, 0x0, 0x9da250, 0xc4200dab40, 0xc4200001a0)
    /go/src/github.com/elastic/beats/libbeat/beat/beat.go:136 +0x65
main.main()
    /go/src/github.com/elastic/beats/filebeat/main.go:22 +0x57
goroutine 10 [syscall, 269 minutes]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:116 +0x157
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
    /usr/local/go/src/os/signal/signal_unix.go:28 +0x41
goroutine 24 [chan receive99 minutes]:
github.com/elastic/beats/libbeat/logp.logExpvars(0xc42008ed48)
    /go/src/github.com/elastic/beats/libbeat/logp/logp.go:224 +0x145
created by github.com/elastic/beats/libbeat/logp.Init
    /go/src/github.com/elastic/beats/libbeat/logp/logp.go:137 +0x294
goroutine 25 [select, 104 minutes]:
github.com/elastic/beats/libbeat/outputs/mode/lb.(*context).receive(0xc42000f510, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /go/src/github.com/elastic/beats/libbeat/outputs/mode/lb/context.go:132 +0x40f
github.com/elastic/beats/libbeat/outputs/mode/lb.(*syncWorker).sendLoop(0xc42000f4f0, 0x0)
    /go/src/github.com/elastic/beats/libbeat/outputs/mode/lb/sync_worker.go:98 +0x46
github.com/elastic/beats/libbeat/outputs/mode/lb.(*syncWorker).run(0xc42000f4f0)
    /go/src/github.com/elastic/beats/libbeat/outputs/mode/lb/sync_worker.go:71 +0x107
github.com/elastic/beats/libbeat/outputs/mode/lb.(*LB).start.func1(0xc07380, 0xc42000f4f0)
    /go/src/github.com/elastic/beats/libbeat/outputs/mode/lb/lb.go:103 +0x79
created by github.com/elastic/beats/libbeat/outputs/mode/lb.(*LB).start
    /go/src/github.com/elastic/beats/libbeat/outputs/mode/lb/lb.go:114 +0x1a9
goroutine 26 [running]:
    goroutine running on other thread; stack unavailable
created by github.com/elastic/beats/libbeat/outputs/mode/lb.(*LB).start
    /go/src/github.com/elastic/beats/libbeat/outputs/mode/lb/lb.go:114 +0x1a9
goroutine 29 [select, 104 minutes]:
github.com/elastic/beats/libbeat/publisher.(*messageWorker).run(0xc42000f770)
    /go/src/github.com/elastic/beats/libbeat/publisher/worker.go:63 +0x2a6
created by github.com/elastic/beats/libbeat/publisher.(*messageWorker).init
    /go/src/github.com/elastic/beats/libbeat/publisher/worker.go57 +0x103
goroutine 30 [select]:
github.com/elastic/beats/libbeat/publisher.(*bulkWorker).run(0xc420154b60)
    /go/src/github.com/elastic/beats/libbeat/publisher/bulk.go:54 +0x34d
created by github.com/elastic/beats/libbeat/publisher.newBulkWorker
    /go/src/github.com/elastic/beats/libbeat/publisher/bulk.go:42 +0x225
goroutine 31 [select, 269 minutes, locked to thread]:
runtime.gopark(0x9dafc0, 0x0, 0x99015f, 0x6, 0x18, 0x2)
    /usr/local/go/src/runtime/proc.go:259 +0x13a
runtime.selectgoImpl(0xc420023730, 0x0, 0x18)
    /usr/local/go/src/runtime/select.go:423 +0x11d9
runtime.selectgo(0xc420023730)
    /usr/local/go/src/runtime/select.go:238 +0x1c
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal1_unix.go:304 +0x2f3
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1
goroutine 32 [chan receive, 269 minutes]:
github.com/elastic/beats/libbeat/service.HandleSignals.func1(0xc420159740, 0xc4201611e0, 0xc42017e5c0)
    /go/src/github.com/elastic/beats/libbeat/service/service.go:31 +0x44
created by github.com/elastic/beats/libbeat/service.HandleSignals
    /go/src/github.com/elastic/beats/libbeat/service/service.go:34 +0x195
goroutine 49 [runnable]:
github.com/elastic/beats/filebeat/prospector.(*Prospector).Run.func1(0xc42009a540)
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:127 +0x16b
created by github.com/elastic/beats/filebeat/prospector.(*Prospector).Run
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go138 +0x10c
goroutine 34 [select, 104 minutes]:
github.com/elastic/beats/filebeat/registrar.(*Registrar).Run(0xc42000fa40)
    /go/src/github.com/elastic/beats/filebeat/registrar/registrar.go:246 +0x436
created by github.com/elastic/beats/filebeat/registrar.(*Registrar).Start
    /go/src/github.com/elastic/beats/filebeat/registrar/registrar.go:230 +0x7d
goroutine 35 [select, 104 minutes]:
github.com/elastic/beats/libbeat/publisher.(*syncPipeline).publish(0xc420024270, 0xc42015d500, 0x101, 0xc0dd00, 0xc420024098, 0x0, 0x0, 0xc4201c64e0, 0xd, 0xd, ...)
    /go/src/github.com/elastic/beats/libbeat/publisher/sync.go:35 +0x2d3
github.com/elastic/beats/libbeat/publisher.(*client).PublishEvents(0xc42015d500, 0xc4201d2070, 0xd, 0xd, 0xc420380a50, 0x2, 0x2, 0x1)
    /go/src/github.com/elastic/beats/libbeat/publisher/client.go:133 +0x408
github.com/elastic/beats/filebeat/publisher.(*syncLogPublisher).Publish(0xc42000fa90, 0x0, 0x0)
    /go/src/github.com/elastic/beats/filebeat/publisher/sync.go:61 +0x1ed
github.com/elastic/beats/filebeat/publisher.(*syncLogPublisher).Start.func1(0xc42000fa90)
    /go/src/github.com/elastic/beats/filebeat/publisher/sync.go:45 +0xf4
created by github.com/elastic/beats/filebeat/publisher.(*syncLogPublisher).Start
    /go/src/github.com/elastic/beats/filebeat/publisher/sync.go:50 +0x90
goroutine 36 [select, 99 minutes]:
github.com/elastic/beats/filebeat/beater.(*publisherChannel).Send(0xc420161290, 0xc4201ce000, 0xb, 0xb, 0xc4201a8000)
    /go/src/github.com/elastic/beats/filebeat/beater/channels.go:78 +0x124
github.com/elastic/beats/filebeat/spooler.(*Spooler).flush(0xc42000fae0, 0x37)
    /go/src/github.com/elastic/beats/filebeat/spooler/spooler.go:142 +0xec
github.com/elastic/beats/filebeat/spooler.(*Spooler).run(0xc42000fae0)
    /go/src/github.com/elastic/beats/filebeat/spooler/spooler.go:90 +0x3ed
created by github.com/elastic/beats/filebeat/spooler.(*Spooler).Start
/go/src/github.com/elastic/beats/filebeat/spooler/spooler.go:56 +0x5c
goroutine 37 [select]:
github.com/elastic/beats/filebeat/prospector.(*Prospector).Run(0xc42009a460, 0x0)
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:151 +0x275
github.com/elastic/beats/filebeat/crawler.(*Crawler).Start.func1(0xc42000fb30, 0x0, 0x0, 0xc42009a460)
    /go/src/github.com/elastic/beats/filebeat/crawler/crawler.go:59 +0x109
created by github.com/elastic/beats/filebeat/crawler.(*Crawler).Start
    /go/src/github.com/elastic/beats/filebeat/crawler/crawler.go:60 +0x374
goroutine 38 [select]:
github.com/elastic/beats/filebeat/prospector.(*Prospector).Run(0xc42009a540, 0x0)
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:151 +0x275
github.com/elastic/beats/filebeat/crawler.(*Crawler).Start.func1(0xc42000fb30, 0x0, 0x1, 0xc42009a540)
    /go/src/github.com/elastic/beats/filebeat/crawler/crawler.go:59 +0x109
created by github.com/elastic/beats/filebeat/crawler.(*Crawler).Start
    /go/src/github.com/elastic/beats/filebeat/crawler/crawler.go:60 +0x374
goroutine 39 [chan receive, 269 minutes]:
github.com/elastic/beats/filebeat/beater.waitChannel.func1()
    /go/src/github.com/elastic/beats/filebeat/beater/signalwait.go:62 +0x3f
github.com/elastic/beats/filebeat/beater.(*signalWait).Add.func1(0xc4201b2850, 0xc420161230)
    /go/src/github.com/elastic/beats/filebeat/beater/signalwait.go:35 +0x27
created by github.com/elastic/beats/filebeat/beater.(*signalWait).Add
    /go/src/github.com/elastic/beats/filebeat/beater/signalwait.go:38 +0x52
goroutine 40 [select]:
github.com/elastic/beats/filebeat/prospector.(*Prospector).Run.func1(0xc42009a460)
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:127 +0x16b
created by github.com/elastic/beats/filebeat/prospector.(*Prospector).Run
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:138 +
goroutine 90 [select]:
github.com/elastic/beats/filebeat/harvester.(*LogFile).wait(0xc4201d0000)
    /go/src/github.com/elastic/beats/filebeat/harvester/log_file.go:152 +0x14d
github.com/elastic/beats/filebeat/harvester.(*LogFile).Read(0xc4201d0000, 0xc42033a000, 0x4000, 0x4000, 0x4000, 0x4000, 0xc4201b6bc0)
    /go/src/github.com/elastic/beats/filebeat/harvester/log_file.go:85 +0x275
github.com/elastic/beats/filebeat/harvester/reader.(*Line).advance(0xc420048070, 0x2, 0xc42023f1e0)
    /go/src/github.com/elastic/beats/filebeat/harvester/reader/line.go:112 +0x111
github.com/elastic/beats/filebeat/harvester/reader.(*Line).Next(0xc420048070, 0xc4201b6c78, 0xc420159da0, 0xc420159da0, 0xc4201b6cac, 0xc42000e2d0, 0xc4201b6cac)
    /go/src/github.com/elastic/beats/filebeat/harvester/reader/line.go:57 +0x3d
github.com/elastic/beats/filebeat/harvester/reader.Encode.Next(0xc420048070, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x43a283, ...)
    /go/src/github.com/elastic/beats/filebeat/harvester/reader/encode.go:30 +0x5e
github.com/elastic/beats/filebeat/harvester/reader.(*StripNewline).Next(0xc4205a2dc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x180001, ...)
    /go/src/github.com/elastic/beats/filebeat/harvester/reader/strip_newline.go:16 +0x66
github.com/elastic/beats/filebeat/harvester/reader.(*Limit).Next(0xc4201e7f00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xed0521b62, ...)
    /go/src/github.com/elastic/beats/filebeat/harvester/reader/limit.go:17 +0x66
github.com/elastic/beats/filebeat/harvester.(*Harvester).Harvest(0xc42013d520, 0xc07180, 0xc4201e7f00)
    /go/src/github.com/elastic/beats/filebeat/harvester/log.go:93 +0x24d
github.com/elastic/beats/filebeat/prospector.(*Prospector).startHarvester.func1(0xc42009a460, 0xc42013d520, 0xc07180, 0xc4201e7f00)
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:248 +0x65
created by github.com/elastic/beats/filebeat/prospector.(*Prospector).startHarvester
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:249 +0x2e4
goroutine 89 [select, 106 minutes]:
github.com/elastic/beats/filebeat/harvester.(*Harvester).Harvest.func1(0xc4201af040, 0xc4201cf380)
    /go/src/github.com/elastic/beats/filebeat/harvester/log.go:72 +0x1fd
created by github.com/elastic/beats/filebeat/harvester.(*Harvester).Harvest
/go/src/github.com/elastic/beats/filebeat/harvester/log.go:82 +0x158
goroutine 88 [running]:
    goroutine running on other thread; stack unavailable
created by github.com/elastic/beats/filebeat/prospector.(*Prospector).startHarvester
    /go/src/github.com/elastic/beats/filebeat/prospector/prospector.go:249 +0x2e4
goroutine 91 [select]:
github.com/elastic/beats/filebeat/harvester.(*Harvester).Harvest.func1(0xc42013d520, 0xc4201cf3e0)
    /go/src/github.com/elastic/beats/filebeat/harvester/log.go:72 +0x1fd
created by github.com/elastic/beats/filebeat/harvester.(*Harvester).Harvest
    /go/src/github.com/elastic/beats/filebeat/harvester/log.go:82 +0x158
rax    0xfffffffffffffffc
rbx    0x6b
rcx    0x45dee7
rdx    0x0
rdi    0x0
rsi    0x0
rbp    0xc420035ef0
rsp    0xc420035ee0
r8     0xc420035ee0
r9     0x0
r10    0x0
r11    0x246
r12    0x42ef50
r13    0xf3
r14    0x30
r15    0x3
rip    0x45dee7
rflags 0x246
cs     0x33
fs     0x0
gs     0x0