-
-
Save StevenOchsCGI/5370629ed85a8ee26f49222f6ce73343 to your computer and use it in GitHub Desktop.
Files for Graylog / Metricbeat Troubleshooting Part 1 of 2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| lin-play-02>root>>/etc/metricbeat>>tail -100 /var/log/metricbeat/metricbeat | |
| 2018-06-12T14:14:32.548-0700 INFO instance/beat.go:468 Home path: [/usr/share/metricbeat] Config path: [/etc/metricbeat] Data path: [/var/lib/metricbeat] Logs path: [/var/log/metricbeat] | |
| 2018-06-12T14:14:32.549-0700 INFO instance/beat.go:475 Beat UUID: c333adf1-4565-43e3-940a-1ae5654b8e8e | |
| 2018-06-12T14:14:32.549-0700 INFO instance/beat.go:213 Setup Beat: metricbeat; Version: 6.2.4 | |
| 2018-06-12T14:14:32.551-0700 INFO pipeline/module.go:76 Beat name: lin-play-02 | |
| 2018-06-12T14:14:32.551-0700 INFO instance/beat.go:301 metricbeat start running. | |
| 2018-06-12T14:14:32.551-0700 INFO [monitoring] log/log.go:97 Starting metrics logging every 30s | |
| 2018-06-12T14:14:32.554-0700 INFO cfgfile/reload.go:127 Config reloader started | |
| 2018-06-12T14:14:32.556-0700 INFO cfgfile/reload.go:258 Starting 3 runners ... | |
| 2018-06-12T14:14:32.556-0700 INFO cfgfile/reload.go:219 Loading of config files completed. | |
| 2018-06-12T14:15:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":310,"time":319},"total":{"ticks":690,"time":701,"value":690},"user":{"ticks":380,"time":382}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":30067}},"memstats":{"gc_next":7226464,"memory_alloc":3620096,"memory_total":48064960,"rss":21970944}},"libbeat":{"config":{"module":{"running":3,"starts":3},"reloads":1},"output":{"events":{"acked":59,"batches":3,"total":59},"read":{"bytes":18},"type":"logstash","write":{"bytes":7577}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":59,"retry":27,"total":85},"queue":{"acked":59}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":26,"success":26},"process_summary":{"events":3,"success":3},"uptime":{"events":1,"success":1}}},"system":{"cpu":{"cores":2},"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:15:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":590,"time":591},"total":{"ticks":1300,"time":1310,"value":1300},"user":{"ticks":710,"time":719}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":60066}},"memstats":{"gc_next":7243136,"memory_alloc":3629360,"memory_total":89656416,"rss":380928}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6864}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:16:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":860,"time":863},"total":{"ticks":1890,"time":1900,"value":1890},"user":{"ticks":1030,"time":1037}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":90066}},"memstats":{"gc_next":7226944,"memory_alloc":3633632,"memory_total":130906016,"rss":708608}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7641}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:16:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":1160,"time":1163},"total":{"ticks":2480,"time":2488,"value":2480},"user":{"ticks":1320,"time":1325}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":120064}},"memstats":{"gc_next":7247936,"memory_alloc":3631168,"memory_total":172248632,"rss":53248}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":47,"batches":3,"total":47},"read":{"bytes":18},"write":{"bytes":6653}},"pipeline":{"clients":6,"events":{"active":0,"published":47,"total":47},"queue":{"acked":47}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:17:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":1450,"time":1456},"total":{"ticks":3120,"time":3126,"value":3120},"user":{"ticks":1670,"time":1670}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":150064}},"memstats":{"gc_next":7231168,"memory_alloc":3625056,"memory_total":213661624,"rss":40960}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":55,"batches":3,"total":55},"read":{"bytes":18},"write":{"bytes":7419}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":55,"total":81},"queue":{"acked":55}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:17:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":1740,"time":1743},"total":{"ticks":3690,"time":3701,"value":3690},"user":{"ticks":1950,"time":1958}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":180064}},"memstats":{"gc_next":7247360,"memory_alloc":3639296,"memory_total":255039720,"rss":208896}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6850}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:18:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":2040,"time":2049},"total":{"ticks":4320,"time":4333,"value":4320},"user":{"ticks":2280,"time":2284}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":210064}},"memstats":{"gc_next":7247648,"memory_alloc":3631184,"memory_total":296059112,"rss":118784}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7527}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.01,"norm":{"1":0,"15":0.025,"5":0.005}}}}}} | |
| 2018-06-12T14:18:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":2340,"time":2342},"total":{"ticks":4960,"time":4965,"value":4960},"user":{"ticks":2620,"time":2623}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":240064}},"memstats":{"gc_next":7232480,"memory_alloc":3631888,"memory_total":337215904,"rss":94208}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6694}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.08,"15":0.05,"5":0.03,"norm":{"1":0.04,"15":0.025,"5":0.015}}}}}} | |
| 2018-06-12T14:19:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":2600,"time":2604},"total":{"ticks":5530,"time":5541,"value":5530},"user":{"ticks":2930,"time":2937}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":270064}},"memstats":{"gc_next":7248352,"memory_alloc":3628464,"memory_total":378452936,"rss":12288}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7520}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.05,"15":0.05,"5":0.03,"norm":{"1":0.025,"15":0.025,"5":0.015}}}}}} | |
| 2018-06-12T14:19:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":2850,"time":2855},"total":{"ticks":6110,"time":6118,"value":6110},"user":{"ticks":3260,"time":3263}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":300065}},"memstats":{"gc_next":7247232,"memory_alloc":3715728,"memory_total":419707648,"rss":24576}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6911}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.03,"15":0.05,"5":0.02,"norm":{"1":0.015,"15":0.025,"5":0.01}}}}}} | |
| 2018-06-12T14:20:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":3100,"time":3104},"total":{"ticks":6660,"time":6666,"value":6660},"user":{"ticks":3560,"time":3562}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":330064}},"memstats":{"gc_next":7238752,"memory_alloc":3627488,"memory_total":460619344,"rss":122880}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":55,"batches":3,"total":55},"read":{"bytes":18},"write":{"bytes":7396}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":55,"total":81},"queue":{"acked":55}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.02,"15":0.05,"5":0.02,"norm":{"1":0.01,"15":0.025,"5":0.01}}}}}} | |
| 2018-06-12T14:20:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":3390,"time":3397},"total":{"ticks":7280,"time":7291,"value":7280},"user":{"ticks":3890,"time":3894}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":360066}},"memstats":{"gc_next":7230688,"memory_alloc":3631872,"memory_total":501979128,"rss":12288}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":7019}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.01,"15":0.05,"5":0.02,"norm":{"1":0.005,"15":0.025,"5":0.01}}}}}} | |
| 2018-06-12T14:21:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":3660,"time":3667},"total":{"ticks":7890,"time":7906,"value":7890},"user":{"ticks":4230,"time":4239}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":390064}},"memstats":{"gc_next":7232832,"memory_alloc":3631712,"memory_total":543016600,"rss":12288}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7437}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.01,"15":0.05,"5":0.02,"norm":{"1":0.005,"15":0.025,"5":0.01}}}}}} | |
| 2018-06-12T14:21:32.555-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":3920,"time":3928},"total":{"ticks":8500,"time":8512,"value":8500},"user":{"ticks":4580,"time":4584}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":420065}},"memstats":{"gc_next":4914400,"memory_alloc":4391464,"memory_total":584130896,"rss":4096}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":47,"batches":3,"total":47},"read":{"bytes":18},"write":{"bytes":6779}},"pipeline":{"clients":6,"events":{"active":0,"published":47,"total":47},"queue":{"acked":47}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.16,"15":0.06,"5":0.06,"norm":{"1":0.08,"15":0.03,"5":0.03}}}}}} | |
| 2018-06-12T14:22:02.553-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4210,"time":4216},"total":{"ticks":9110,"time":9124,"value":9110},"user":{"ticks":4900,"time":4908}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":450064}},"memstats":{"gc_next":7242624,"memory_alloc":3624864,"memory_total":625329088,"rss":65536}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":55,"batches":3,"total":55},"read":{"bytes":18},"write":{"bytes":7386}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":55,"total":81},"queue":{"acked":55}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.18,"15":0.06,"5":0.07,"norm":{"1":0.09,"15":0.03,"5":0.035}}}}}} | |
| 2018-06-12T14:22:32.553-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4480,"time":4488},"total":{"ticks":9710,"time":9724,"value":9710},"user":{"ticks":5230,"time":5236}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":480064}},"memstats":{"gc_next":7249856,"memory_alloc":3632272,"memory_total":666449192,"rss":16384}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":47,"batches":3,"total":47},"read":{"bytes":18},"write":{"bytes":6822}},"pipeline":{"clients":6,"events":{"active":0,"published":47,"total":47},"queue":{"acked":47}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.16,"15":0.06,"5":0.08,"norm":{"1":0.08,"15":0.03,"5":0.04}}}}}} | |
| 2018-06-12T14:23:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":4750,"time":4753},"total":{"ticks":10320,"time":10329,"value":10320},"user":{"ticks":5570,"time":5576}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":510064}},"memstats":{"gc_next":7233024,"memory_alloc":3631904,"memory_total":707484328,"rss":40960}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7407}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.1,"15":0.06,"5":0.07,"norm":{"1":0.05,"15":0.03,"5":0.035}}}}}} | |
| 2018-06-12T14:23:32.553-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":5040,"time":5042},"total":{"ticks":10940,"time":10944,"value":10940},"user":{"ticks":5900,"time":5902}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":540064}},"memstats":{"gc_next":7232640,"memory_alloc":3632080,"memory_total":748648904,"rss":20480}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6886}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.06,"15":0.06,"5":0.07,"norm":{"1":0.03,"15":0.03,"5":0.035}}}}}} | |
| 2018-06-12T14:24:02.553-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":5320,"time":5323},"total":{"ticks":11540,"time":11549,"value":11540},"user":{"ticks":6220,"time":6226}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":570064}},"memstats":{"gc_next":7250848,"memory_alloc":3640304,"memory_total":789901752,"rss":8192}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7636}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.04,"15":0.06,"5":0.06,"norm":{"1":0.02,"15":0.03,"5":0.03}}}}}} | |
| 2018-06-12T14:24:32.555-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":5600,"time":5606},"total":{"ticks":12130,"time":12141,"value":12130},"user":{"ticks":6530,"time":6535}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":600065}},"memstats":{"gc_next":7300384,"memory_alloc":3666048,"memory_total":831051056,"rss":4096}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6936}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.08,"15":0.06,"5":0.07,"norm":{"1":0.04,"15":0.03,"5":0.035}}}}}} | |
| 2018-06-12T14:25:02.553-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":5860,"time":5867},"total":{"ticks":12730,"time":12743,"value":12730},"user":{"ticks":6870,"time":6876}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":630064}},"memstats":{"gc_next":7316512,"memory_alloc":3665200,"memory_total":871977880,"rss":8192}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7475}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.05,"15":0.05,"5":0.06,"norm":{"1":0.025,"15":0.025,"5":0.03}}}}}} | |
| 2018-06-12T14:25:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6120,"time":6121},"total":{"ticks":13300,"time":13302,"value":13300},"user":{"ticks":7180,"time":7181}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":660064}},"memstats":{"gc_next":7249920,"memory_alloc":3632768,"memory_total":913294608,"rss":20480}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":47,"batches":3,"total":47},"read":{"bytes":18},"write":{"bytes":6676}},"pipeline":{"clients":6,"events":{"active":0,"published":47,"total":47},"queue":{"acked":47}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":23,"success":23},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.03,"15":0.05,"5":0.06,"norm":{"1":0.015,"15":0.025,"5":0.03}}}}}} | |
| 2018-06-12T14:26:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6390,"time":6395},"total":{"ticks":13880,"time":13894,"value":13880},"user":{"ticks":7490,"time":7499}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":690064}},"memstats":{"gc_next":7300480,"memory_alloc":3663024,"memory_total":954315728,"rss":12288}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7472}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.02,"15":0.05,"5":0.05,"norm":{"1":0.01,"15":0.025,"5":0.025}}}}}} | |
| 2018-06-12T14:26:32.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6670,"time":6674},"total":{"ticks":14480,"time":14492,"value":14480},"user":{"ticks":7810,"time":7818}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":720064}},"memstats":{"gc_next":7252736,"memory_alloc":3641952,"memory_total":995503960,"rss":4096}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6765}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.01,"15":0.05,"5":0.05,"norm":{"1":0.005,"15":0.025,"5":0.025}}}}}} | |
| 2018-06-12T14:27:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":6980,"time":6986},"total":{"ticks":15130,"time":15143,"value":15130},"user":{"ticks":8150,"time":8157}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":750064}},"memstats":{"gc_next":7302464,"memory_alloc":3666864,"memory_total":1036736944,"rss":12288}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7602}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0.01,"15":0.05,"5":0.04,"norm":{"1":0.005,"15":0.025,"5":0.02}}}}}} | |
| 2018-06-12T14:27:32.553-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":7220,"time":7224},"total":{"ticks":15700,"time":15712,"value":15700},"user":{"ticks":8480,"time":8488}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":780064}},"memstats":{"gc_next":7303264,"memory_alloc":3667696,"memory_total":1077888704,"rss":16384}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":48,"batches":3,"total":48},"read":{"bytes":18},"write":{"bytes":6827}},"pipeline":{"clients":6,"events":{"active":0,"published":48,"total":48},"queue":{"acked":48}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.04,"norm":{"1":0,"15":0.025,"5":0.02}}}}}} | |
| 2018-06-12T14:28:02.554-0700 INFO [monitoring] log/log.go:124 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":7500,"time":7503},"total":{"ticks":16260,"time":16269,"value":16260},"user":{"ticks":8760,"time":8766}},"info":{"ephemeral_id":"4fa0b144-517c-48f8-af61-14d6c480e381","uptime":{"ms":810064}},"memstats":{"gc_next":7254144,"memory_alloc":3631360,"memory_total":1118911552}},"libbeat":{"config":{"module":{"running":3}},"output":{"events":{"acked":56,"batches":3,"total":56},"read":{"bytes":18},"write":{"bytes":7774}},"pipeline":{"clients":6,"events":{"active":0,"filtered":26,"published":56,"total":82},"queue":{"acked":56}}},"metricbeat":{"system":{"cpu":{"events":3,"success":3},"filesystem":{"events":33,"success":33},"fsstat":{"events":1,"success":1},"load":{"events":3,"success":3},"memory":{"events":3,"success":3},"network":{"events":12,"success":12},"process":{"events":24,"success":24},"process_summary":{"events":3,"success":3}}},"system":{"load":{"1":0,"15":0.05,"5":0.04,"norm":{"1":0,"15":0.025,"5":0.02}}}}}} | |
| lin-play-02>root>>/etc/metricbeat>> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| lin-play-02>root>>/etc/metricbeat>>cat metricbeat.yml | |
| ###################### Metricbeat Configuration Example ####################### | |
| # This file is an example configuration file highlighting only the most common | |
| # options. The metricbeat.reference.yml file from the same directory contains all the | |
| # supported options with more comments. You can use it as a reference. | |
| # | |
| # You can find the full configuration reference here: | |
| # https://www.elastic.co/guide/en/beats/metricbeat/index.html | |
| #========================== Modules configuration ============================ | |
| metricbeat.config.modules: | |
| # Glob pattern for configuration loading | |
| path: ${path.config}/modules.d/*.yml | |
| # Set to true to enable config reloading | |
| reload.enabled: false | |
| # Period on which files under path should be checked for changes | |
| #reload.period: 10s | |
| #==================== Elasticsearch template setting ========================== | |
| setup.template.settings: | |
| index.number_of_shards: 1 | |
| index.codec: best_compression | |
| #_source.enabled: false | |
| #================================ General ===================================== | |
| # The name of the shipper that publishes the network data. It can be used to group | |
| # all the transactions sent by a single shipper in the web interface. | |
| #name: | |
| # The tags of the shipper are included in their own field with each | |
| # transaction published. | |
| #tags: ["service-X", "web-tier"] | |
| # Optional fields that you can specify to add additional information to the | |
| # output. | |
| #fields: | |
| # env: staging | |
| #============================== Dashboards ===================================== | |
| # These settings control loading the sample dashboards to the Kibana index. Loading | |
| # the dashboards is disabled by default and can be enabled either by setting the | |
| # options here, or by using the `-setup` CLI flag or the `setup` command. | |
| #setup.dashboards.enabled: false | |
| # The URL from where to download the dashboards archive. By default this URL | |
| # has a value which is computed based on the Beat name and version. For released | |
| # versions, this URL points to the dashboard archive on the artifacts.elastic.co | |
| # website. | |
| #setup.dashboards.url: | |
| #============================== Kibana ===================================== | |
| # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. | |
| # This requires a Kibana endpoint configuration. | |
| setup.kibana: | |
| # Kibana Host | |
| # Scheme and port can be left out and will be set to the default (http and 5601) | |
| # In case you specify and additional path, the scheme is required: http://localhost:5601/path | |
| # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 | |
| #host: "localhost:5601" | |
| #============================= Elastic Cloud ================================== | |
| # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). | |
| # The cloud.id setting overwrites the `output.elasticsearch.hosts` and | |
| # `setup.kibana.host` options. | |
| # You can find the `cloud.id` in the Elastic Cloud web UI. | |
| #cloud.id: | |
| # The cloud.auth setting overwrites the `output.elasticsearch.username` and | |
| # `output.elasticsearch.password` settings. The format is `<user>:<pass>`. | |
| #cloud.auth: | |
| #================================ Outputs ===================================== | |
| # Configure what output to use when sending the data collected by the beat. | |
| #-------------------------- Elasticsearch output ------------------------------ | |
| # output.elasticsearch: | |
| # Array of hosts to connect to. | |
| # hosts: ["162.70.27.206:5043"] | |
| # Optional protocol and basic auth credentials. | |
| # protocol: "https" | |
| # username: "advadmin" | |
| # password: "disadvantage4$" | |
| #----------------------------- Logstash output -------------------------------- | |
| output.logstash: | |
| # The Logstash hosts | |
| #hosts: ["localhost:5044"] | |
| hosts: ["162.70.27.206:15043"] | |
| # Optional SSL. By default is off. | |
| # List of root certificates for HTTPS server verifications | |
| #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] | |
| # Certificate for SSL client authentication | |
| #ssl.certificate: "/etc/pki/client/cert.pem" | |
| # Client Certificate Key | |
| #ssl.key: "/etc/pki/client/cert.key" | |
| #================================ Logging ===================================== | |
| # Sets log level. The default log level is info. | |
| # Available log levels are: error, warning, info, debug | |
| #logging.level: debug | |
| # At debug level, you can selectively enable logging only for some components. | |
| # To enable all selectors use ["*"]. Examples of other selectors are "beat", | |
| # "publish", "service". | |
| #logging.selectors: ["*"] | |
| #============================== Xpack Monitoring =============================== | |
| # metricbeat can export internal metrics to a central Elasticsearch monitoring | |
| # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The | |
| # reporting is disabled by default. | |
| # Set to true to enable the monitoring reporter. | |
| #xpack.monitoring.enabled: false | |
| # Uncomment to send the metrics to Elasticsearch. Most settings from the | |
| # Elasticsearch output are accepted here as well. Any setting that is not set is | |
| # automatically inherited from the Elasticsearch output configuration, so if you | |
| # have the Elasticsearch output configured, you can simply uncomment the | |
| # following line. | |
| #xpack.monitoring.elasticsearch: | |
| lin-play-02>root>>/etc/metricbeat>> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [2018-06-12 08:52:16,376][INFO ][node ] [graylog] version[2.4.6], pid[1970], build[5376dca/2017-07-18T12:17:44Z] | |
| [2018-06-12 08:52:16,377][INFO ][node ] [graylog] initializing ... | |
| [2018-06-12 08:52:16,792][INFO ][plugins ] [graylog] modules [reindex, lang-expression, lang-groovy], plugins [], sites [] | |
| [2018-06-12 08:52:16,834][INFO ][env ] [graylog] using [1] data paths, mounts [[/var (/dev/mapper/rhel-var)]], net usable_space [498mb], net total_space [20.3gb], spins? [no], types [xfs] | |
| [2018-06-12 08:52:16,834][INFO ][env ] [graylog] heap size [989.8mb], compressed ordinary object pointers [true] | |
| [2018-06-12 08:52:18,107][INFO ][node ] [graylog] initialized | |
| [2018-06-12 08:52:18,107][INFO ][node ] [graylog] starting ... | |
| [2018-06-12 08:52:18,169][INFO ][transport ] [graylog] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300} | |
| [2018-06-12 08:52:18,173][INFO ][discovery ] [graylog] elasticsearch/jj7Zg4ErRs2Kbkmn9W_M_Q | |
| [2018-06-12 08:52:21,195][INFO ][cluster.service ] [graylog] new_master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300}, reason: zen-disco-join(elected_as_master, [0] joins received) | |
| [2018-06-12 08:52:21,226][INFO ][http ] [graylog] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200} | |
| [2018-06-12 08:52:21,226][INFO ][node ] [graylog] started | |
| [2018-06-12 08:52:21,253][INFO ][gateway ] [graylog] recovered [2] indices into cluster_state | |
| [2018-06-12 08:52:21,954][INFO ][cluster.routing.allocation] [graylog] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[graylog_0][2], [graylog_0][2], [graylog_0][0], [graylog_0][0]] ...]). | |
| [2018-06-12 08:52:51,225][WARN ][cluster.routing.allocation.decider] [graylog] high disk watermark [90%] exceeded on [jj7Zg4ErRs2Kbkmn9W_M_Q][graylog][/var/lib/elasticsearch/elasticsearch/nodes/0] free: 86.6mb[0.4%], shards will be relocated away from this node | |
| [2018-06-12 08:52:51,225][INFO ][cluster.routing.allocation.decider] [graylog] rerouting shards: [high disk watermark exceeded on one or more nodes] | |
| [2018-06-12 08:52:56,219][ERROR][index.engine ] [graylog] [graylog_1][0] failed to merge | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingS[2018-06-12 08:52:56,228][DEBUG][action.bulk ] [graylog] [graylog_1][2] failed to execute bulk item (index) index {[graylog_deflector][message][aa806a0d-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730266","collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_level":"Information","winlogbeat_tags":["windows","iis"],"source":"advadmin","type":"wineventlog","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"AMS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_process_id":668,"winlogbeat_opcode":"Info","winlogbeat_task":"Special Logon","winlogbeat_event_data_SubjectUserSid":"S-1-5-21-682003330-1383384898-1801674531-428089","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 13:01:19.471","winlogbeat_log_name":"Security","winlogbeat_event_data_SubjectLogonId":"0xb6f4c33","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_PrivilegeList":"SeSecurityPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeImpersonatePrivilege","winlogbeat_event_data_SubjectUserName":"a-anthony.gould","winlogbeat_thread_id":6656,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-682003330-1383384898-1801674531-428089\n\tAccount Name:\t\ta-anthony.gould\n\tAccount Domain:\t\tAMS\n\tLogon ID:\t\t0xB6F4C33\n\nPrivileges:\t\tSeSecurityPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeImpersonatePrivilege","tags":["windows","iis"],"winlogbeat_event_id":4672,"name":"WINADVWORKS","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#aa806a0d-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.store.DataOutput.writeVInt(DataOutput.java:191) | |
| at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.finishTerm(Lucene50PostingsWriter.java:392) | |
| at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:169) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1041) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| [2018-06-12 08:52:56,220][WARN ][index.engine ] [graylog] [graylog_1][0] Failed to close translog | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.sync(BufferingTranslogWriter.java:125) | |
| at org.elasticsearch.index.translog.Translog.close(Translog.java:420) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:97) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:84) | |
| at org.elasticsearch.index.engine.InternalEngine.closeNoLock(InternalEngine.java:987) | |
| at org.elasticsearch.index.engine.Engine.failEngine(Engine.java:540) | |
| at org.elasticsearch.index.engine.InternalEngine.maybeFailEngine(InternalEngine.java:915) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:672) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,229][DEBUG][action.bulk ] [graylog] [graylog_1][0] failed to execute bulk item (index) index {[graylog_deflector][message][aa70d9b2-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9729825","winlogbeat_version":1,"gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_tags":["windows","iis"],"source":"advadmin","type":"wineventlog","winlogbeat_event_data_ProcessId":"0x0","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_event_data_SubjectDomainName":"-","winlogbeat_process_id":668,"winlogbeat_event_data_TargetUserName":"a-anthony.gould","winlogbeat_opcode":"Info","winlogbeat_task":"Logon","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","winlogbeat_event_data_TransmittedServices":"-","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"winlogbeat_event_data_SubjectLogonId":"0x0","winlogbeat_log_name":"Security","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_thread_id":6772,"streams":["000000000000000000000001"],"winlogbeat_event_data_LogonGuid":"{1ACF3F12-B0E9-41A5-E29F-728741CFF507}","winlogbeat_type":"wineventlog","winlogbeat_event_data_IpPort":"59257","tags":["windows","iis"],"winlogbeat_event_data_TargetLogonId":"0xb43d12f","winlogbeat_event_data_TargetDomainName":"AMS","name":"WINADVWORKS","winlogbeat_computer_name":"WINADVWORKS.ams.com","winlogbeat_event_data_IpAddress":"162.70.113.37","collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_level":"Information","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_LogonProcessName":"Kerberos","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_event_data_ImpersonationLevel":"%%1832","winlogbeat_event_data_ProcessName":"-","winlogbeat_event_data_SubjectUserSid":"S-1-0-0","winlogbeat_event_data_KeyLength":"0","timestamp":"2018-06-09 11:31:11.375","winlogbeat_event_data_SubjectUserName":"-","message":"An account was successfully logged on.\n\nSubject:\n\tSecurity ID:\t\tS-1-0-0\n\tAccount Name:\t\t-\n\tAccount Domain:\t\t-\n\tLogon ID:\t\t0x0\n\nLogon Type:\t\t\t3\n\nImpersonation Level:\t\tIdentification\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-21-682003330-1383384898-1801674531-428089\n\tAccount Name:\t\ta-anthony.gould\n\tAccount Domain:\t\tAMS\n\tLogon ID:\t\t0xB43D12F\n\tLogon GUID:\t\t{1ACF3F12-B0E9-41A5-E29F-728741CFF507}\n\nProcess Information:\n\tProcess ID:\t\t0x0\n\tProcess Name:\t\t-\n\nNetwork Information:\n\tWorkstation Name:\t-\n\tSource Network Address:\t162.70.113.37\n\tSource Port:\t\t59257\n\nDetailed Authentication Information:\n\tLogon Process:\t\tKerberos\n\tAuthentication Package:\tKerberos\n\tTransited Services:\t-\n\tPackage Name (NTLM only):\t-\n\tKey Length:\t\t0\n\nThis event is generated when a logon session is created. It is generated on the computer that was accessed.\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThe network fields indicate where a remote logon request originated. Workstation name is not always available and may be left blank in some cases.\n\nThe impersonation level field indicates the extent to which a process in the logon session can impersonate.\n\nThe authentication information fields provide detailed information about this specific logon request.\n\t- Logon GUID is a unique identifier that can be used to correlate this event with a KDC event.\n\t- Transited services indicate which intermediate services have participated in this logon request.\n\t- Package name indicates which sub-protocol was used among the NTLM protocols.\n\t- Key length indicates the length of the generated session key. This will be 0 if no session key was requested.","winlogbeat_event_data_LmPackageName":"-","winlogbeat_event_data_LogonType":"3","winlogbeat_event_id":4624,"winlogbeat_event_data_TargetUserSid":"S-1-5-21-682003330-1383384898-1801674531-428089","facility":"winlogbeat","winlogbeat_event_data_WorkstationName":"-","winlogbeat_event_data_AuthenticationPackageName":"Kerberos"}]} | |
| [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#aa70d9b2-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| [2018-06-12 08:52:56,229][DEBUG][action.bulk ] [graylog] [graylog_1][3] failed to execute bulk item (index) index {[graylog_deflector][message][aa7fa6c1-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730235","collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_level":"Information","winlogbeat_tags":["windows","iis"],"source":"advadmin","type":"wineventlog","winlogbeat_event_data_ProcessId":"0x3c8","winlogbeat_event_data_ObjectServer":"Security","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_event_data_ObjectType":"Token","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"AMS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_process_id":668,"winlogbeat_opcode":"Info","winlogbeat_event_data_ProcessName":"C:\\Windows\\System32\\svchost.exe","winlogbeat_task":"Authorization Policy Change","winlogbeat_event_data_SubjectUserSid":"S-1-5-18","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","winlogbeat_event_data_ObjectName":"-","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_event_data_NewSd":"D:(A;;GA;;;SY)(A;;RC;;;OW)(A;;GA;;;S-1-5-86-615999462-62705297-2911207457-59056572-3668589837)","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 12:51:30.626","winlogbeat_log_name":"Security","winlogbeat_event_data_SubjectLogonId":"0x3e7","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_OldSd":"D:(A;;GA;;;SY)(A;;GA;;;NS)","winlogbeat_event_data_SubjectUserName":"WINADVWORKS$","winlogbeat_thread_id":676,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"Permissions on an object were changed.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tWINADVWORKS$\n\tAccount Domain:\t\tAMS\n\tLogon ID:\t\t0x3E7\n\nObject:\n\tObject Server:\tSecurity\n\tObject Type:\tToken\n\tObject Name:\t-\n\tHandle ID:\t0x14e0\n\nProcess:\n\tProcess ID:\t0x3c8\n\tProcess Name:\tC:\\Windows\\System32\\svchost.exe\n\nPermissions Change:\n\tOriginal Security Descriptor:\tD:(A;;GA;;;SY)(A;;GA;;;NS)\n\tNew Security Descriptor:\tD:(A;;GA;;;SY)(A;;RC;;;OW)(A;;GA;;;S-1-5-86-615999462-62705297-2911207457-59056572-3668589837)","winlogbeat_event_data_HandleId":"0x14e0","tags":["windows","iis"],"winlogbeat_event_id":4670,"name":"WINADVWORKS","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#aa7fa6c1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.store.DataOutput.writeVInt(DataOutput.java:191) | |
| at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.finishTerm(Lucene50PostingsWriter.java:343) | |
| at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:169) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1041) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| [2018-06-12 08:52:56,233][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,237][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [already closed by tragic event on the index writer] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.index.IndexWriter$1.createOutput(IndexWriter.java:4850) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| [2018-06-12 08:52:56,237][WARN ][index.shard ] [graylog] [graylog_1][3] Failed to perform scheduled engine refresh | |
| [graylog_1][[graylog_1][3]] RefreshFailedEngineException[Refresh failed]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5om_Lucene50_0.tip: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:677) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5om_Lucene50_0.tip: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.<init>(BlockTreeTermsWriter.java:343) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.<init>(BlockTreeTermsWriter.java:280) | |
| at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsConsumer(Lucene50PostingsFormat.java:428) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:196) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| ... 5 more | |
| [2018-06-12 08:52:56,237][ERROR][index.engine ] [graylog] [graylog_1][3] failed to merge | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.index.IndexWriter$1.createOutput(IndexWriter.java:4850) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| [2018-06-12 08:52:56,238][DEBUG][action.bulk ] [graylog] [graylog_1][3] failed to execute bulk item (index) index {[graylog_deflector][message][aa7fcdc1-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730245","collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_level":"Information","winlogbeat_tags":["windows","iis"],"source":"advadmin","type":"wineventlog","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"AMS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_process_id":668,"winlogbeat_opcode":"Info","winlogbeat_task":"Special Logon","winlogbeat_event_data_SubjectUserSid":"S-1-5-21-682003330-1383384898-1801674531-428089","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 12:55:12.935","winlogbeat_event_data_SubjectLogonId":"0xb6ddaa9","winlogbeat_log_name":"Security","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_PrivilegeList":"SeSecurityPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeImpersonatePrivilege","winlogbeat_event_data_SubjectUserName":"a-anthony.gould","winlogbeat_thread_id":688,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"Special privileges assigned to new logon.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-682003330-1383384898-1801674531-428089\n\tAccount Name:\t\ta-anthony.gould\n\tAccount Domain:\t\tAMS\n\tLogon ID:\t\t0xB6DDAA9\n\nPrivileges:\t\tSeSecurityPrivilege\n\t\t\tSeBackupPrivilege\n\t\t\tSeRestorePrivilege\n\t\t\tSeTakeOwnershipPrivilege\n\t\t\tSeDebugPrivilege\n\t\t\tSeSystemEnvironmentPrivilege\n\t\t\tSeLoadDriverPrivilege\n\t\t\tSeImpersonatePrivilege","tags":["windows","iis"],"winlogbeat_event_id":4672,"name":"WINADVWORKS","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#aa7fcdc1-6e58-11e8-a5a1-525400f3698f]]; nested: AlreadyClosedException[this IndexWriter is closed]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:720) | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:734) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1473) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.index.IndexWriter$1.createOutput(IndexWriter.java:4850) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| [2018-06-12 08:52:56,239][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [already closed by tragic event on the index writer]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.index.IndexWriter$1.createOutput(IndexWriter.java:4850) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| [2018-06-12 08:52:56,240][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [236] requests] on [[graylog_1][3]] | |
| [graylog_1][[graylog_1][3]] IllegalIndexShardStateException[CurrentState[CLOSED] operation only allowed when started/recovering, origin [PRIMARY]] | |
| at org.elasticsearch.index.shard.IndexShard.ensureWriteAllowed(IndexShard.java:1066) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:597) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,241][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[6], s[STARTED], a[id=ps7PljXhTr-BpM1QU-tfYg]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [already closed by tragic event on the index writer]], failure [FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.index.IndexWriter$1.createOutput(IndexWriter.java:4850) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:139) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| [2018-06-12 08:52:56,244][INFO ][cluster.routing.allocation] [graylog] Cluster health status changed from [GREEN] to [RED] (reason: [shards failed [[graylog_1][3]] ...]). | |
| [2018-06-12 08:52:56,246][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,246][DEBUG][action.bulk ] [graylog] [graylog_1][1] failed to execute bulk item (index) index {[graylog_deflector][message][aa7fcdda-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730264","collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_tags":["windows","iis"],"winlogbeat_level":"Information","source":"advadmin","type":"wineventlog","winlogbeat_event_data_ProcessId":"0x1d48","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_event_data_Status":"0x0","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"WINADVWORKS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_process_id":4,"winlogbeat_opcode":"Info","winlogbeat_event_data_ProcessName":"C:\\Windows\\CCM\\UpdateTrustedSites.exe","winlogbeat_task":"Process Termination","winlogbeat_event_data_SubjectUserSid":"S-1-5-21-469633003-3481618796-1645009692-1002","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 13:00:40.211","winlogbeat_log_name":"Security","winlogbeat_event_data_SubjectLogonId":"0x1ff3d0","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_SubjectUserName":"westadmin","winlogbeat_thread_id":4544,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"A process has exited.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-469633003-3481618796-1645009692-1002\n\tAccount Name:\t\twestadmin\n\tAccount Domain:\t\tWINADVWORKS\n\tLogon ID:\t\t0x1FF3D0\n\nProcess Information:\n\tProcess ID:\t0x1d48\n\tProcess Name:\tC:\\Windows\\CCM\\UpdateTrustedSites.exe\n\tExit Status:\t0x0","tags":["windows","iis"],"winlogbeat_event_id":4689,"name":"WINADVWORKS","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#aa7fcdda-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| [2018-06-12 08:52:56,248][ERROR][index.engine ] [graylog] [graylog_1][2] failed to merge | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:580) | |
| at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:177) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:83) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.close(RateLimitedIndexOutput.java:49) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:97) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:84) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.close(CompressingStoredFieldsWriter.java:138) | |
| at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:178) | |
| ... 6 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 11 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.close(RateLimitedIndexOutput.java:49) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsIndexWriter.close(CompressingStoredFieldsIndexWriter.java:210) | |
| ... 10 more | |
| [2018-06-12 08:52:56,262][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [250] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,263][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [already closed by tragic event on the index writer] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:103) | |
| at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4670) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:492) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:459) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| [2018-06-12 08:52:56,264][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [250] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,264][DEBUG][action.bulk ] [graylog] [graylog_1][2] failed to execute bulk item (index) index {[graylog_deflector][message][aa806a0c-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730265","collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_tags":["windows","iis"],"winlogbeat_level":"Information","source":"advadmin","type":"wineventlog","winlogbeat_event_data_ProcessId":"0x1574","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_event_data_Status":"0x0","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"WINADVWORKS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_process_id":4,"winlogbeat_opcode":"Info","winlogbeat_task":"Process Termination","winlogbeat_event_data_ProcessName":"C:\\Windows\\System32\\conhost.exe","winlogbeat_event_data_SubjectUserSid":"S-1-5-21-469633003-3481618796-1645009692-1002","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 13:00:40.211","winlogbeat_event_data_SubjectLogonId":"0x1ff3d0","winlogbeat_log_name":"Security","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_SubjectUserName":"westadmin","winlogbeat_thread_id":4544,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"A process has exited.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-469633003-3481618796-1645009692-1002\n\tAccount Name:\t\twestadmin\n\tAccount Domain:\t\tWINADVWORKS\n\tLogon ID:\t\t0x1FF3D0\n\nProcess Information:\n\tProcess ID:\t0x1574\n\tProcess Name:\tC:\\Windows\\System32\\conhost.exe\n\tExit Status:\t0x0","tags":["windows","iis"],"winlogbeat_event_id":4689,"name":"WINADVWORKS","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#aa806a0c-6e58-11e8-a5a1-525400f3698f]]; nested: AlreadyClosedException[this IndexWriter is closed]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:720) | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:734) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1473) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:103) | |
| at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4670) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:492) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:459) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| ... 3 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| [2018-06-12 08:52:56,265][WARN ][index.shard ] [graylog] [graylog_1][2] Failed to perform scheduled engine refresh | |
| [graylog_1][[graylog_1][2]] RefreshFailedEngineException[Refresh failed]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:677) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:103) | |
| at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4670) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:492) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:459) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| ... 5 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| [2018-06-12 08:52:56,266][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [260] requests] on [[graylog_1][2]] | |
| [graylog_1][[graylog_1][2]] EngineClosedException[CurrentState[CLOSED] Closed]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:103) | |
| at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4670) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:492) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:459) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| ... 3 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| [2018-06-12 08:52:56,267][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [250] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,267][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,267][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [engine failure, reason [already closed by tragic event on the index writer]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:103) | |
| at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4670) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:492) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:459) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| [2018-06-12 08:52:56,268][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [260] requests] on [[graylog_1][2]] | |
| [graylog_1][[graylog_1][2]] ShardNotFoundException[no such shard] | |
| at org.elasticsearch.index.IndexService.shardSafe(IndexService.java:197) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction.getIndexShardOperationsCounter(TransportReplicationAction.java:764) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:656) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,269][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[6], s[STARTED], a[id=14eal36hSFW9NOS4bjhqHg]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [already closed by tragic event on the index writer]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:103) | |
| at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4670) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:492) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:459) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:68) | |
| at org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:105) | |
| ... 21 more | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at java.io.FilterOutputStream.close(FilterOutputStream.java:158) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.close(OutputStreamIndexOutput.java:70) | |
| ... 22 more | |
| [2018-06-12 08:52:56,271][WARN ][indices.memory ] [graylog] failed to set shard [graylog_1][0] index buffer to [14.1mb] | |
| org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:720) | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:734) | |
| at org.apache.lucene.index.IndexWriter.ramBytesUsed(IndexWriter.java:475) | |
| at org.elasticsearch.index.engine.InternalEngine.indexWriterRAMBytesUsed(InternalEngine.java:945) | |
| at org.elasticsearch.index.shard.IndexShard.updateBufferSize(IndexShard.java:1147) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController.updateShardBuffers(IndexingMemoryController.java:232) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController$ShardsIndicesStatusChecker.run(IndexingMemoryController.java:286) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController.forceCheck(IndexingMemoryController.java:245) | |
| at org.elasticsearch.index.shard.IndexShard.markLastWrite(IndexShard.java:1050) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:968) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| ... 3 more | |
| [2018-06-12 08:52:56,273][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [250] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,272][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,291][WARN ][indices.memory ] [graylog] failed to set shard [graylog_1][1] index buffer to [14.1mb] | |
| org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:720) | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:734) | |
| at org.apache.lucene.index.IndexWriter.ramBytesUsed(IndexWriter.java:475) | |
| at org.elasticsearch.index.engine.InternalEngine.indexWriterRAMBytesUsed(InternalEngine.java:945) | |
| at org.elasticsearch.index.shard.IndexShard.updateBufferSize(IndexShard.java:1147) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController.updateShardBuffers(IndexingMemoryController.java:232) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController$ShardsIndicesStatusChecker.run(IndexingMemoryController.java:286) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController.forceCheck(IndexingMemoryController.java:245) | |
| at org.elasticsearch.index.shard.IndexShard.markLastWrite(IndexShard.java:1050) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:968) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| ... 3 more | |
| [2018-06-12 08:52:56,292][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,292][ERROR][index.engine ] [graylog] [graylog_1][1] failed to merge | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.codecs.lucene50.ForUtil.writeBlock(ForUtil.java:175) | |
| at org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter.startDoc(Lucene50PostingsWriter.java:237) | |
| at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:141) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1041) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105) | |
| at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:193) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| [2018-06-12 08:52:56,292][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [250] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,295][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [250] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,295][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,296][WARN ][index.engine ] [graylog] [graylog_1][1] failed engine [refresh failed] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.store.RAMOutputStream.writeTo(RAMOutputStream.java:86) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:1007) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:766) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:1085) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1046) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,296][DEBUG][action.bulk ] [graylog] [graylog_1][1] failed to execute bulk item (index) index {[graylog_deflector][message][aa7b12db-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730003","winlogbeat_version":2,"collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_level":"Information","winlogbeat_tags":["windows","iis"],"source":"advadmin","type":"wineventlog","winlogbeat_event_data_ProcessId":"0x17a4","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"AMS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_event_data_NewProcessName":"C:\\Windows\\CCM\\UpdateTrustedSites.exe","winlogbeat_process_id":4,"winlogbeat_opcode":"Info","winlogbeat_event_data_TargetUserName":"westadmin","winlogbeat_task":"Process Creation","winlogbeat_event_data_SubjectUserSid":"S-1-5-18","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 12:00:40.626","winlogbeat_log_name":"Security","winlogbeat_event_data_SubjectLogonId":"0x3e7","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_SubjectUserName":"WINADVWORKS$","winlogbeat_event_data_TokenElevationType":"%%1937","winlogbeat_thread_id":4544,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"A new process has been created.\n\nCreator Subject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tWINADVWORKS$\n\tAccount Domain:\t\tAMS\n\tLogon ID:\t\t0x3E7\n\nTarget Subject:\n\tSecurity ID:\t\tS-1-5-21-469633003-3481618796-1645009692-1002\n\tAccount Name:\t\twestadmin\n\tAccount Domain:\t\tWINADVWORKS\n\tLogon ID:\t\t0x1FF3D0\n\nProcess Information:\n\tNew Process ID:\t\t0x1634\n\tNew Process Name:\tC:\\Windows\\CCM\\UpdateTrustedSites.exe\n\tToken Elevation Type:\tTokenElevationTypeFull (2)\n\tCreator Process ID:\t0x17a4\n\tProcess Command Line:\t\n\nToken Elevation Type indicates the type of token that was assigned to the new process in accordance with User Account Control policy.\n\nType 1 is a full token with no privileges removed or groups disabled. A full token is only used if User Account Control is disabled or if the user is the built-in Administrator account or a service account.\n\nType 2 is an elevated token with no privileges removed or groups disabled. An elevated token is used when User Account Control is enabled and the user chooses to start the program using Run as administrator. An elevated token is also used when an application is configured to always require administrative privilege or to always require maximum privilege, and the user is a member of the Administrators group.\n\nType 3 is a limited token with administrative privileges removed and administrative groups disabled. The limited token is used when User Account Control is enabled, the application does not require administrative privilege, and the user does not choose to start the program using Run as administrator.","winlogbeat_event_data_NewProcessId":"0x1634","tags":["windows","iis"],"winlogbeat_event_data_TargetLogonId":"0x1ff3d0","winlogbeat_event_data_TargetDomainName":"WINADVWORKS","winlogbeat_event_id":4688,"name":"WINADVWORKS","winlogbeat_event_data_TargetUserSid":"S-1-5-21-469633003-3481618796-1645009692-1002","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#aa7b12db-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| [2018-06-12 08:52:56,296][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [engine failure, reason [refresh failed]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.store.RAMOutputStream.writeTo(RAMOutputStream.java:86) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:1007) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:766) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:1085) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1046) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,296][WARN ][index.shard ] [graylog] [graylog_1][1] Failed to perform scheduled engine refresh | |
| [graylog_1][[graylog_1][1]] RefreshFailedEngineException[Refresh failed]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:677) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.store.RAMOutputStream.writeTo(RAMOutputStream.java:86) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:1007) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:766) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:1085) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1046) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| ... 5 more | |
| [2018-06-12 08:52:56,297][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [240] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] IllegalIndexShardStateException[CurrentState[CLOSED] operation only allowed when started/recovering, origin [PRIMARY]] | |
| at org.elasticsearch.index.shard.IndexShard.ensureWriteAllowed(IndexShard.java:1066) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:597) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,297][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[8], s[STARTED], a[id=t6FbzQmLQXabsKfjNQXPlA]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [refresh failed]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.store.RAMOutputStream.writeTo(RAMOutputStream.java:86) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:1007) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:766) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:1085) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:1046) | |
| at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:456) | |
| at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198) | |
| at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:126) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615) | |
| at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:424) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:286) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:261) | |
| at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:251) | |
| at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104) | |
| at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:137) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:154) | |
| at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58) | |
| at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) | |
| at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) | |
| at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:669) | |
| at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:665) | |
| at org.elasticsearch.index.shard.IndexShard$EngineRefresher$1.run(IndexShard.java:1347) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,297][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [240] requests] on [[graylog_1][1]] | |
| [graylog_1][[graylog_1][1]] ShardNotFoundException[no such shard] | |
| at org.elasticsearch.index.IndexService.shardSafe(IndexService.java:197) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction.getIndexShardOperationsCounter(TransportReplicationAction.java:764) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:656) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,300][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,320][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [263] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] EngineClosedException[CurrentState[CLOSED] Closed] | |
| at org.elasticsearch.index.engine.Engine.ensureOpen(Engine.java:329) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:448) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:56,325][WARN ][indices.memory ] [graylog] failed to set shard [graylog_1][0] index buffer to [12.3mb] | |
| org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:720) | |
| at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:734) | |
| at org.apache.lucene.index.IndexWriter.ramBytesUsed(IndexWriter.java:475) | |
| at org.elasticsearch.index.engine.InternalEngine.indexWriterRAMBytesUsed(InternalEngine.java:945) | |
| at org.elasticsearch.index.shard.IndexShard.updateBufferSize(IndexShard.java:1147) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController.updateShardBuffers(IndexingMemoryController.java:232) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController$ShardsIndicesStatusChecker.run(IndexingMemoryController.java:286) | |
| at org.elasticsearch.indices.memory.IndexingMemoryController.forceCheck(IndexingMemoryController.java:245) | |
| at org.elasticsearch.index.shard.IndexShard.markLastWrite(IndexShard.java:1050) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:968) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| ... 3 more | |
| [2018-06-12 08:52:58,423][ERROR][index.engine ] [graylog] [graylog_1][0] failed to merge | |
| org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception | |
| at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:343) | |
| at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:727) | |
| at org.apache.lucene.index.IndexFileDeleter.deletePendingFiles(IndexFileDeleter.java:516) | |
| at org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:719) | |
| at org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:4690) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4164) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:58,426][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [already closed by tragic event on the index writer] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:58,426][DEBUG][action.bulk ] [graylog] [graylog_1][0] failed to execute bulk item (index) index {[graylog_deflector][message][aa7f3187-6e58-11e8-a5a1-525400f3698f], source[{"winlogbeat_fields_gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_record_number":"9730144","collector_node_id":"Winadvworks-graylog-collector-sidecar","gl2_remote_ip":"162.70.25.127","gl2_remote_port":22098,"winlogbeat_level":"Information","winlogbeat_tags":["windows","iis"],"source":"advadmin","type":"wineventlog","winlogbeat_event_data_ProcessId":"0x15c0","gl2_source_input":"5ae0c7bccb28d942245a86af","winlogbeat_event_data_Status":"0x0","winlogbeat_fields_collector_node_id":"Winadvworks-graylog-collector-sidecar","winlogbeat_event_data_SubjectDomainName":"AMS","winlogbeat_provider_guid":"{54849625-5478-4994-A5BA-3E3B0328C30D}","winlogbeat_process_id":4,"winlogbeat_opcode":"Info","winlogbeat_task":"Process Termination","winlogbeat_event_data_ProcessName":"C:\\Windows\\System32\\wbem\\WmiPrvSE.exe","winlogbeat_event_data_SubjectUserSid":"S-1-5-18","winlogbeat_source_name":"Microsoft-Windows-Security-Auditing","gl2_source_node":"df1af257-7540-4a36-aeb0-4d2b727befe1","winlogbeat_keywords":["Audit Success"],"timestamp":"2018-06-09 12:34:19.396","winlogbeat_event_data_SubjectLogonId":"0x3e7","winlogbeat_log_name":"Security","gl2_source_collector":"9605728d-2955-4b6a-bfc4-ab67cff85147","winlogbeat_event_data_SubjectUserName":"WINADVWORKS$","winlogbeat_thread_id":8068,"streams":["000000000000000000000001"],"winlogbeat_type":"wineventlog","message":"A process has exited.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-18\n\tAccount Name:\t\tWINADVWORKS$\n\tAccount Domain:\t\tAMS\n\tLogon ID:\t\t0x3E7\n\nProcess Information:\n\tProcess ID:\t0x15c0\n\tProcess Name:\tC:\\Windows\\System32\\wbem\\WmiPrvSE.exe\n\tExit Status:\t0x0","tags":["windows","iis"],"winlogbeat_event_id":4689,"name":"WINADVWORKS","facility":"winlogbeat","winlogbeat_computer_name":"WINADVWORKS.ams.com"}]} | |
| [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#aa7f3187-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| ... 16 more | |
| [2018-06-12 08:52:58,427][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [already closed by tragic event on the index writer]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:58,427][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [254] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] IllegalIndexShardStateException[CurrentState[CLOSED] operation only allowed when started/recovering, origin [PRIMARY]] | |
| at org.elasticsearch.index.shard.IndexShard.ensureWriteAllowed(IndexShard.java:1066) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:597) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:58,427][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[16], s[STARTED], a[id=99k00BTrR5eVV1D6UpvmjQ]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [already closed by tragic event on the index writer]], failure [FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat.write(Lucene50FieldInfosFormat.java:260) | |
| at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:136) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:422) | |
| at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503) | |
| at org.apache.lucene.index.DocumentsWriter.preUpdate(DocumentsWriter.java:357) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:436) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:454) | |
| at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:605) | |
| at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) | |
| at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:236) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) | |
| at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:52:58,428][DEBUG][action.bulk ] [graylog] failed to execute [BulkShardRequest to [graylog_1] containing [254] requests] on [[graylog_1][0]] | |
| [graylog_1][[graylog_1][0]] ShardNotFoundException[no such shard] | |
| at org.elasticsearch.index.IndexService.shardSafe(IndexService.java:197) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction.getIndexShardOperationsCounter(TransportReplicationAction.java:764) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:656) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) | |
| at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) | |
| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) | |
| at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:378) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,763][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,765][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,765][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=gwBUD7MiSi6Zz6_jsoO2TA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:56.242Z], details[engine failure, reason [already closed by tragic event on the index writer], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5oh.fnm: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,912][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,914][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,914][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=1Rr8IaKsQaiHuuRFj6YpOA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:58.428Z], details[engine failure, reason [already closed by tragic event on the index writer], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:04,914][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:04,915][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=1Rr8IaKsQaiHuuRFj6YpOA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:58.428Z], details[engine failure, reason [already closed by tragic event on the index writer], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f60t.fnm: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#a4124bc8-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:05,003][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.l[2018-06-12 08:53:05,017][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:05,018][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=kC97QeIMTC-zslHIUH2qBA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:04.765Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:05,018][ERROR][index.engine ] [graylog] [graylog_1][1] failed to merge | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86) | |
| at org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addNumericField(Lucene54DocValuesConsumer.java:245) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedField(Lucene54DocValuesConsumer.java:614) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedSetField(Lucene54DocValuesConsumer.java:653) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:131) | |
| at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:736) | |
| at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:219) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:150) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.getChecksum(RateLimitedIndexOutput.java:59) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.close(Lucene54DocValuesConsumer.java:800) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$ConsumerAndSuffix.close(PerFieldDocValuesFormat.java:94) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:97) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.close(PerFieldDocValuesFormat.java:210) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:151) | |
| ... 6 more | |
| [2018-06-12 08:53:05,018][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:05,019][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=kC97QeIMTC-zslHIUH2qBA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:04.765Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#9e9d0c74-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:05,020][WARN ][index.engine ] [graylog] [graylog_1][1] failed engine [merge failed] | |
| org.apache.lucene.index.MergePolicy$MergeException: java.io.IOException: No space left on device | |
| at org.elasticsearch.index.engine.InternalEngine$EngineMergeScheduler$1.doRun(InternalEngine.java:1237) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86) | |
| at org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addNumericField(Lucene54DocValuesConsumer.java:245) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedField(Lucene54DocValuesConsumer.java:614) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedSetField(Lucene54DocValuesConsumer.java:653) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:131) | |
| at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:736) | |
| at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:219) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:150) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.getChecksum(RateLimitedIndexOutput.java:59) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.close(Lucene54DocValuesConsumer.java:800) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$ConsumerAndSuffix.close(PerFieldDocValuesFormat.java:94) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:97) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.close(PerFieldDocValuesFormat.java:210) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:151) | |
| ... 6 more | |
| [2018-06-12 08:53:05,020][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [engine failure, reason [merge failed]] | |
| org.apache.lucene.index.MergePolicy$MergeException: java.io.IOException: No space left on device | |
| at org.elasticsearch.index.engine.InternalEngine$EngineMergeScheduler$1.doRun(InternalEngine.java:1237) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86) | |
| at org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addNumericField(Lucene54DocValuesConsumer.java:245) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedField(Lucene54DocValuesConsumer.java:614) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedSetField(Lucene54DocValuesConsumer.java:653) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:131) | |
| at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:736) | |
| at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:219) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:150) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.getChecksum(RateLimitedIndexOutput.java:59) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.close(Lucene54DocValuesConsumer.java:800) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$ConsumerAndSuffix.close(PerFieldDocValuesFormat.java:94) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:97) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.close(PerFieldDocValuesFormat.java:210) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:151) | |
| ... 6 more | |
| [2018-06-12 08:53:05,021][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=IzzHP6LHQKiHDT60IHVAzg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:56.298Z], details[engine failure, reason [refresh failed], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [merge failed]], failure [MergeException[java.io.IOException: No space left on device]; nested: IOException[No space left on device]; ] | |
| org.apache.lucene.index.MergePolicy$MergeException: java.io.IOException: No space left on device | |
| at org.elasticsearch.index.engine.InternalEngine$EngineMergeScheduler$1.doRun(InternalEngine.java:1237) | |
| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73) | |
| at org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52) | |
| at org.apache.lucene.util.packed.DirectWriter.flush(DirectWriter.java:86) | |
| at org.apache.lucene.util.packed.DirectWriter.add(DirectWriter.java:78) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addNumericField(Lucene54DocValuesConsumer.java:245) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedField(Lucene54DocValuesConsumer.java:614) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.addSortedSetField(Lucene54DocValuesConsumer.java:653) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:131) | |
| at org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:736) | |
| at org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:219) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:150) | |
| at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105) | |
| at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4086) | |
| at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3666) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588) | |
| at org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94) | |
| at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626) | |
| Suppressed: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.getChecksum(OutputStreamIndexOutput.java:80) | |
| at org.apache.lucene.store.RateLimitedIndexOutput.getChecksum(RateLimitedIndexOutput.java:59) | |
| at org.apache.lucene.codecs.CodecUtil.writeCRC(CodecUtil.java:473) | |
| at org.apache.lucene.codecs.CodecUtil.writeFooter(CodecUtil.java:309) | |
| at org.apache.lucene.codecs.lucene54.Lucene54DocValuesConsumer.close(Lucene54DocValuesConsumer.java:800) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$ConsumerAndSuffix.close(PerFieldDocValuesFormat.java:94) | |
| at org.apache.lucene.util.IOUtils.close(IOUtils.java:97) | |
| at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.close(PerFieldDocValuesFormat.java:210) | |
| at org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:151) | |
| ... 6 more | |
| [2018-06-12 08:53:05,021][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:05,022][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=IzzHP6LHQKiHDT60IHVAzg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:56.298Z], details[engine failure, reason [refresh failed], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#a6b46deb-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:05,024][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:05,024][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:05,025][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=HxsVVQC5QDu1VSPSCOfiRw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:56.270Z], details[engine failure, reason [already closed by tragic event on the index writer], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:05,025][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=HxsVVQC5QDu1VSPSCOfiRw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:52:56.270Z], details[engine failure, reason [already closed by tragic event on the index writer], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:06,580][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:53) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,581][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:53) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,582][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=e02n3vTvQHey6uWX-zaMWg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:04.914Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:53) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,583][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,584][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,584][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,584][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=ULQQhOGsTxinP1RU3qgAkg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.018Z], details[engine failure, reason [index], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,585][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][2]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,585][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=Xbhz5SDWTG6fxoOPdKBW2Q], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.032Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][2]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,585][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,586][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=Xbhz5SDWTG6fxoOPdKBW2Q], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.032Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,586][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,586][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=ULQQhOGsTxinP1RU3qgAkg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.018Z], details[engine failure, reason [index], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/index/_f5ne.fdx: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#a0316822-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,587][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=Xbhz5SDWTG6fxoOPdKBW2Q], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.032Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:06,588][WARN ][index.engine ] [graylog] [graylog_1][1] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,588][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,589][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=FIChgYvjT_uOnXrrSMfKJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.021Z], details[engine failure, reason [merge failed], failure MergeException[java.io.IOException: No space left on device]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,589][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,590][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=FIChgYvjT_uOnXrrSMfKJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.021Z], details[engine failure, reason [merge failed], failure MergeException[java.io.IOException: No space left on device]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#a01813b3-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,590][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=FIChgYvjT_uOnXrrSMfKJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.021Z], details[engine failure, reason [merge failed], failure MergeException[java.io.IOException: No space left on device]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:06,620][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=FIChgYvjT_uOnXrrSMfKJw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:05.021Z], details[engine failure, reason [merge failed], failure MergeException[java.io.IOException: No space left on device]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:06,644][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1833033464812114590.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.ut | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5960647808505802674.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:06,710][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=56Ugy5DQTkGHGXeFcC-wQg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.588Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ffcc38c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:06,968][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:53) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,969][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:53) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,969][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=1qe3SUS8ToKv8f2WMWqi2g], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.710Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5960647808505802674.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:53) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,969][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eba3161-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eba3161-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch. | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,991][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,992][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,994][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,995][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=_a4AgzpQQ2KEMLs6H8_eLQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.696Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-3766937495325510428.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,995][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,995][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=_a4AgzpQQ2KEMLs6H8_eLQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.696Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-3766937495325510428.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,996][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,996][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=GH3nPg0bTG2eg7ZbnEu9cg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.696Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1833033464812114590.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#9ec81415-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,997][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,997][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=kw0GnwzHSzKtQAHdwZzrcg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.634Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:06,998][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:06,998][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=GH3nPg0bTG2eg7ZbnEu9cg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.696Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1833033464812114590.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:07,000][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:07,001][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=kw0GnwzHSzKtQAHdwZzrcg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.634Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:07,003][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=kw0GnwzHSzKtQAHdwZzrcg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.634Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,003][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=GH3nPg0bTG2eg7ZbnEu9cg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.696Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1833033464812114590.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,016][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,017][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=cag13y12SiGVVjd85NC-wg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.969Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,020][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=cag13y12SiGVVjd85NC-wg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.969Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,030][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=cag13y12SiGVVjd85NC-wg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.969Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,046][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1653751826428749383.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1653751826428749383.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch[2018-06-12 08:53:07,064][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,065][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=wElCaTT5Tw6XnSVzt-gr6Q], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.995Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,072][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=wElCaTT5Tw6XnSVzt-gr6Q], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:06.995Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec554f0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,072][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,073][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=-JnELcHpRTWC88rbUxlamQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.004Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,073][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=-JnELcHpRTWC88rbUxlamQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.004Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ece55a1-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,085][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,086][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=1Up5eeYQQleZ8FOmjHi3JQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.046Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-4960586520940595545.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,086][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=1Up5eeYQQleZ8FOmjHi3JQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.046Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5200705728406059691.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,096][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,096][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=zJcEq41bRnWqJidk3cNTzQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.055Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1653751826428749383.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-5030565277357268938.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,108][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=zJcEq41bRnWqJidk3cNTzQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.055Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-1653751826428749383.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,109][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,109][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=keCN0WpPShKwiARK2tYEzQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.073Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-220395668376172085.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,110][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=keCN0WpPShKwiARK2tYEzQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.073Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6858143706016241598.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,120][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,123][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=Aoq0WSTKRzu2_9DHUCKfAQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.074Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-4627205768961263458.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:07,124][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=Aoq0WSTKRzu2_9DHUCKfAQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:07.074Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6757995248174097162.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:07,137][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5477157462059431760.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5477157462059431760.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-5477157462059431760.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:13,356][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=2ApUWSfIQr6VLo6_0ZV5Ng], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:13.288Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-3162721357219213738.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6047313021642618975.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6047313021642618975.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6047313021642618975.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-6047313021642618975.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelpe | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-8774064081661770015.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-8774064081661770015.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:52,366][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:52,366][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=ud-uzcrHSg62WLH1H0yxKQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.302Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-9181105645711129829.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:52,366][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:52,366][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=tOrFeO08REaF9iDJdhBCmQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.302Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-1083772395208740898.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:52,367][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=tOrFeO08REaF9iDJdhBCmQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.302Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-1083772395208740898.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:52,367][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=ud-uzcrHSg62WLH1H0yxKQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.302Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-9181105645711129829.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:52,367][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=rVxarcj5SGuXB50rv_bQvQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.302Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-3537794451146174878.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:54,482][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,483][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,483][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=qafpCgr3RNOdWJeZ4vssKg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.368Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-8774064081661770015.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,484][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,484][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=qafpCgr3RNOdWJeZ4vssKg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.368Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-8774064081661770015.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#a057dbe5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,641][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,642][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,648][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=4qqcffZZTuyX30zLuQ0UsQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.484Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]] | |
| java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,648][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,648][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=4qqcffZZTuyX30zLuQ0UsQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.484Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9e9d0c71-6e58-11e8-a5a1-525400f3698f]]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:434) | |
| at java.nio.file.Files.newOutputStream(Files.java:216) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) | |
| at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) | |
| at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) | |
| at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) | |
| at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) | |
| at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) | |
| at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:108) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) | |
| at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) | |
| at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) | |
| at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,668][WARN ][index.engine ] [graylog] [graylog_1][1] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,668][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,669][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,669][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=SuthM03-SByYcpVs1oxIHQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.368Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-6637011017589669655.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,669][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,670][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=rLt5prG0QL64yLIleRZQvg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.368Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,670][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,670][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=rLt5prG0QL64yLIleRZQvg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.368Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.common.bytes.PagedBytesReference.writeTo(PagedBytesReference.java:132) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:57) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,670][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=rLt5prG0QL64yLIleRZQvg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.368Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4153032090495436471.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:54,672][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,677][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][2]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,678][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=2FePdFkCSqGgP-fmzSOPpA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.351Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-8233997380500837855.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][2]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:54,678][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,678][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=2FePdFkCSqGgP-fmzSOPpA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.351Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-8233997380500837855.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:95) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeByte(OutputStreamIndexOutput.java:47) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:176) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:54,679][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=2FePdFkCSqGgP-fmzSOPpA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:52.351Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-8233997380500837855.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:55,115][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,115][WARN ][index.engine ] [graylog] [graylog_1][1] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,115][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,116][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,116][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=PP0gUas-QbGk7p-TUoZacA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.684Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a078d15c-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,116][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,116][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,117][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=8ov4PhVLQ5qQTYosP2YHxg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.648Z], details[engine failure, reason [index], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,117][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,117][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=aypOcicfQS-Oyo_s7OUC9g], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.671Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,118][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,118][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=8ov4PhVLQ5qQTYosP2YHxg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.648Z], details[engine failure, reason [index], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9f0e44dd-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,118][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=aypOcicfQS-Oyo_s7OUC9g], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.671Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:55,118][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=8ov4PhVLQ5qQTYosP2YHxg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.648Z], details[engine failure, reason [index], failure FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/index/_f5zk.fdx: No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:55,118][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [engine failure, reason [index]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,119][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=2i4_t7igQsKTukJH3ynoxg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.669Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [engine failure, reason [index]], failure [IOException[No space left on device]] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,119][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,119][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=2i4_t7igQsKTukJH3ynoxg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.669Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#9f24da13-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,119][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,119][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=aypOcicfQS-Oyo_s7OUC9g], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:54.671Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#a07045e0-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][3]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][3]] IndexFailedEngineException[Index failed for [message#9f1ac7f5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,469][WARN ][index.engine ] [graylog] [graylog_1][1] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,470][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due [2018-06-12 08:53:55,483][WARN ][index.engine ] [graylog] [graylog_1][3] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,484][WARN ][index.engine ] [graylog] [graylog_1][0] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,484][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,485][WARN ][index.engine ] [graylog] [graylog_1][2] failed engine [index] | |
| java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| [2018-06-12 08:53:55,485][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=mCgDavQ5Sg2KZxSL7oVqHw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.119Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][1]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][1]] IndexFailedEngineException[Index failed for [message#9ec5ca21-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at java.nio.channels.Channels.writeFullyImpl(Channels.java:78) | |
| at java.nio.channels.Channels.writeFully(Channels.java:101) | |
| at java.nio.channels.Channels.access$000(Channels.java:61) | |
| at java.nio.channels.Channels$1.write(Channels.java:174) | |
| at org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:278) | |
| at java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:73) | |
| at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) | |
| at java.io.BufferedOutputStream.write(BufferedOutputStream.java:126) | |
| at org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:157) | |
| at org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:170) | |
| at org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:243) | |
| at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:237) | |
| at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:164) | |
| at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:291) | |
| at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:332) | |
| at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) | |
| at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) | |
| at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) | |
| at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:527) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,487][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][2]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,487][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=O8PBNOimSvS4gTUFfKBWhQ], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.116Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][2]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][2]] IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,487][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,487][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=bbGrcAFJQ6uHwKLKP04EWw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.119Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:174) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: [graylog_1][[graylog_1][0]] EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:237) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:171) | |
| ... 11 more | |
| Caused by: [graylog_1][[graylog_1][0]] IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:459) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.performRecoveryOperation(TranslogRecoveryPerformer.java:194) | |
| at org.elasticsearch.index.shard.TranslogRecoveryPerformer.recoveryFromSnapshot(TranslogRecoveryPerformer.java:107) | |
| at org.elasticsearch.index.shard.IndexShard$1.recoveryFromSnapshot(IndexShard.java:1582) | |
| at org.elasticsearch.index.engine.InternalEngine.recoverFromTranslog(InternalEngine.java:235) | |
| ... 12 more | |
| Caused by: java.io.IOException: No space left on device | |
| at sun.nio.ch.FileDispatcherImpl.write0(Native Method) | |
| at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) | |
| at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) | |
| at sun.nio.ch.IOUtil.write(IOUtil.java:65) | |
| at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) | |
| at org.elasticsearch.common.io.Channels.writeToChannel(Channels.java:208) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.flush(BufferingTranslogWriter.java:84) | |
| at org.elasticsearch.index.translog.BufferingTranslogWriter.add(BufferingTranslogWriter.java:66) | |
| at org.elasticsearch.index.translog.Translog.add(Translog.java:545) | |
| at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) | |
| at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:451) | |
| ... 16 more | |
| [2018-06-12 08:53:55,488][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ec1f993-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[grayl | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-3229101608038799523.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-3229101608038799523.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,558][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,558][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=Nmjh5NICTOCtT8rL4rv2tA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.511Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-2449429189550322665.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,559][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=Nmjh5NICTOCtT8rL4rv2tA], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.511Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9ebc0622-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:55,560][WARN ][indices.cluster ] [graylog] [[graylog_1][1]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,560][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=AatbfZ1sTbCTaf49cvOPww], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.484Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/1/translog/translog-8455492086544392316.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,564][WARN ][cluster.action.shard ] [graylog] [graylog_1][1] received shard failed for target shard [[graylog_1][1], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[9], s[INITIALIZING], a[id=AatbfZ1sTbCTaf49cvOPww], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.484Z], details[engine failure, reason [index], failure IOException[No space left on device]]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:55,570][WARN ][indices.cluster ] [graylog] [[graylog_1][0]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,571][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=zSsxdou_RUGgTKJOB-Zdfw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.511Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/0/translog/translog-7872689147045621710.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,576][WARN ][cluster.action.shard ] [graylog] [graylog_1][0] received shard failed for target shard [[graylog_1][0], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[17], s[INITIALIZING], a[id=zSsxdou_RUGgTKJOB-Zdfw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.511Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to recover from translog]; nested: EngineException[failed to recover from translog]; nested: IndexFailedEngineException[Index failed for [message#9eb109a5-6e58-11e8-a5a1-525400f3698f]]; nested: IOException[No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1:9300} marked shard as initializing, but shard is marked as failed, resend shard failure] | |
| [2018-06-12 08:53:55,590][WARN ][indices.cluster ] [graylog] [[graylog_1][3]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,590][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=ZAUsPD0JSTOEITtb8fRClg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.558Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-3229101608038799523.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device]; ] | |
| [graylog_1][[graylog_1][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-4022412570249669140.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:55,590][WARN ][cluster.action.shard ] [graylog] [graylog_1][3] received shard failed for target shard [[graylog_1][3], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=ZAUsPD0JSTOEITtb8fRClg], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:55.558Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/3/translog/translog-3229101608038799523.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [master {graylog}{jj7Zg4ErRs2Kbkmn9W_M_Q}{127.0.0.1}{127.0.0.1 | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:56,910][WARN ][indices.cluster ] [graylog] [[graylog_1][2]] marking and sending shard failed due to [failed recovery] | |
| [graylog_1][[graylog_1][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-3670991638515275605.tlog: No space left on device]; | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56) | |
| at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: [graylog_1][[graylog_1][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-3670991638515275605.tlog: No space left on device]; | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:152) | |
| at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) | |
| at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1513) | |
| at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1497) | |
| at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:970) | |
| at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:942) | |
| at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241) | |
| ... 5 more | |
| Caused by: java.nio.file.FileSystemException: /var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-3670991638515275605.tlog: No space left on device | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) | |
| at java.nio.file.Files.newByteChannel(Files.java:361) | |
| at java.nio.file.Files.createFile(Files.java:632) | |
| at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) | |
| at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) | |
| at java.nio.file.Files.createTempFile(Files.java:852) | |
| at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:320) | |
| at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179) | |
| at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:205) | |
| at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:148) | |
| ... 11 more | |
| [2018-06-12 08:53:56,910][WARN ][cluster.action.shard ] [graylog] [graylog_1][2] received shard failed for target shard [[graylog_1][2], node[jj7Zg4ErRs2Kbkmn9W_M_Q], [P], v[7], s[INITIALIZING], a[id=Bl3vCMP0S2WPfIVhNWUsEw], unassigned_info[[reason=ALLOCATION_FAILED], at[2018-06-12T15:53:56.823Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/var/lib/elasticsearch/elasticsearch/nodes/0/indices/graylog_1/2/translog/translog-7211004683116399940.tlog: No space left on device]; ]]], indexUUID [EOO_UrbqS06n2kK_YJRnYg], message [failed recovery], failure [I[2018-06-12 08:54:16,331][INFO ][cluster.routing.allocation] [graylog] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[graylog_1][2]] ...]). | |
| [2018-06-12 08:54:51,315][INFO ][cluster.routing.allocation.decider] [graylog] rerouting shards: [one or more nodes has gone under the high or low watermark] | |
| [2018-06-12 09:06:28,674][INFO ][cluster.metadata ] [graylog] [graylog_2] creating index, cause [api], templates [graylog-internal], shards [4]/[0], mappings [message] | |
| [2018-06-12 09:06:28,704][INFO ][cluster.routing.allocation] [graylog] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[graylog_2][0], [graylog_2][1], [graylog_2][0]] ...]). | |
| [2018-06-12 11:18:53,191][INFO ][node ] [graylog] stopping ... | |
| [2018-06-12 11:18:53,229][INFO ][node ] [graylog] stopped | |
| [2018-06-12 11:18:53,229][INFO ][node ] [graylog] closing ... | |
| [2018-06-12 11:18:53,236][INFO ][node ] [graylog] closed | |
| [2018-06-12 12:08:28,676][INFO ][node ] [graylog] version[2.4.6], pid[6857], build[5376dca/2017-07-18T12:17:44Z] | |
| [2018-06-12 12:08:28,677][INFO ][node ] [graylog] initializing ... | |
| [2018-06-12 12:08:29,226][INFO ][plugins ] [graylog] modules [reindex, lang-expression, lang-groovy], plugins [], sites [] | |
| [2018-06-12 12:08:29,274][INFO ][env ] [graylog] using [1] data paths, mounts [[/var (/dev/mapper/rhel-var)]], net usable_space [10.2gb], net total_space [20.3gb], spins? [no], types [xfs] | |
| [2018-06-12 12:08:29,274][INFO ][env ] [graylog] heap size [989.8mb], compressed ordinary object pointers [true] | |
| [2018-06-12 12:08:30,579][INFO ][node ] [graylog] initialized | |
| [2018-06-12 12:08:30,579][INFO ][node ] [graylog] starting ... | |
| [2018-06-12 12:08:30,646][INFO ][transport ] [graylog] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300} | |
| [2018-06-12 12:08:30,650][INFO ][discovery ] [graylog] elasticsearch/Uf0LDtCPR_-lA4339SpFhQ | |
| [2018-06-12 12:08:33,673][INFO ][cluster.service ] [graylog] new_master {graylog}{Uf0LDtCPR_-lA4339SpFhQ}{127.0.0.1}{127.0.0.1:9300}, reason: zen-disco-join(elected_as_master, [0] joins received) | |
| [2018-06-12 12:08:33,688][INFO ][http ] [graylog] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200} | |
| [2018-06-12 12:08:33,688][INFO ][node ] [graylog] started | |
| [2018-06-12 12:08:33,769][INFO ][gateway ] [graylog] recovered [2] indices into cluster_state | |
| [2018-06-12 12:08:34,912][INFO ][cluster.routing.allocation] [graylog] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[graylog_1][3]] ...]). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Graylog v2.4.3+2c41897 | |
| elasticsearch-2.4.6-1 | |
| metricbeat version 6.2.4 (amd64), libbeat 6.2.4 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment