Skip to content

Instantly share code, notes, and snippets.

@wesruv

wesruv/gist:e87f2c670d64cf68967e7f02fc9bdc64

Created February 14, 2017 17:07
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save wesruv/e87f2c670d64cf68967e7f02fc9bdc64 to your computer and use it in GitHub Desktop.
Save wesruv/e87f2c670d64cf68967e7f02fc9bdc64 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-1158164249911420948.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,948][WARN ][cluster.action.shard ] [Power Broker] [lullabot][4] received shard failed for target shard [[lullabot][4], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4882], s[INITIALIZING], a[id=JkCAHu3jTLqxDBa1drpqKw], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.944Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-7537693276651332805.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [master {Power Broker}{rMO1Wry9R6GHFRpWcuUyrg}{172.17.0.5}{172.17.0.5:9300} marked shard as initializing, but shard is marked as failed, resend shard failure]
elasticsearch_1 | [2017-02-14 17:03:30,948][WARN ][cluster.action.shard ] [Power Broker] [lullabot][3] received shard failed for target shard [[lullabot][3], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4574], s[INITIALIZING], a[id=MFmDylV4QIW82ET2TyMMSg], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.943Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-7514800275861179072.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [master {Power Broker}{rMO1Wry9R6GHFRpWcuUyrg}{172.17.0.5}{172.17.0.5:9300} marked shard as initializing, but shard is marked as failed, resend shard failure]
elasticsearch_1 | [2017-02-14 17:03:30,948][WARN ][index.translog ] [Power Broker] [lullabot][1] failed to delete temp file /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog
elasticsearch_1 | java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
elasticsearch_1 | at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
elasticsearch_1 | at java.nio.file.Files.delete(Files.java:1126)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:358)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | [2017-02-14 17:03:30,949][WARN ][indices.cluster ] [Power Broker] [[lullabot][1]] marking and sending shard failed due to [failed recovery]
elasticsearch_1 | [lullabot][[lullabot][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,949][WARN ][cluster.action.shard ] [Power Broker] [lullabot][1] received shard failed for target shard [[lullabot][1], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4675], s[INITIALIZING], a[id=BXQDexyhSei8VXajH-448w], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.944Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-5446418442225293953.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device]; ]
elasticsearch_1 | [lullabot][[lullabot][1]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][1]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-3306427097789871479.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,949][WARN ][cluster.action.shard ] [Power Broker] [lullabot][1] received shard failed for target shard [[lullabot][1], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4675], s[INITIALIZING], a[id=BXQDexyhSei8VXajH-448w], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.944Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/1/translog/translog-5446418442225293953.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [master {Power Broker}{rMO1Wry9R6GHFRpWcuUyrg}{172.17.0.5}{172.17.0.5:9300} marked shard as initializing, but shard is marked as failed, resend shard failure]
elasticsearch_1 | [2017-02-14 17:03:30,949][WARN ][index.translog ] [Power Broker] [lullabot][0] failed to delete temp file /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog
elasticsearch_1 | java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
elasticsearch_1 | at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
elasticsearch_1 | at java.nio.file.Files.delete(Files.java:1126)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:358)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | [2017-02-14 17:03:30,950][WARN ][indices.cluster ] [Power Broker] [[lullabot][0]] marking and sending shard failed due to [failed recovery]
elasticsearch_1 | [lullabot][[lullabot][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,950][WARN ][cluster.action.shard ] [Power Broker] [lullabot][0] received shard failed for target shard [[lullabot][0], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4452], s[INITIALIZING], a[id=HEbjkIXFSj68NhHYmyyrTg], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.946Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-2337260916562090791.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device]; ]
elasticsearch_1 | [lullabot][[lullabot][0]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][0]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-5816283059281948328.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,950][WARN ][cluster.action.shard ] [Power Broker] [lullabot][0] received shard failed for target shard [[lullabot][0], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4452], s[INITIALIZING], a[id=HEbjkIXFSj68NhHYmyyrTg], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.946Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/0/translog/translog-2337260916562090791.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [master {Power Broker}{rMO1Wry9R6GHFRpWcuUyrg}{172.17.0.5}{172.17.0.5:9300} marked shard as initializing, but shard is marked as failed, resend shard failure]
elasticsearch_1 | [2017-02-14 17:03:30,951][WARN ][index.translog ] [Power Broker] [lullabot][2] failed to delete temp file /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog
elasticsearch_1 | java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
elasticsearch_1 | at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
elasticsearch_1 | at java.nio.file.Files.delete(Files.java:1126)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:358)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | [2017-02-14 17:03:30,952][WARN ][index.translog ] [Power Broker] [lullabot][4] failed to delete temp file /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog
elasticsearch_1 | java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
elasticsearch_1 | at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
elasticsearch_1 | at java.nio.file.Files.delete(Files.java:1126)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:358)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | [2017-02-14 17:03:30,952][WARN ][indices.cluster ] [Power Broker] [[lullabot][4]] marking and sending shard failed due to [failed recovery]
elasticsearch_1 | [lullabot][[lullabot][4]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][4]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,952][WARN ][cluster.action.shard ] [Power Broker] [lullabot][4] received shard failed for target shard [[lullabot][4], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4883], s[INITIALIZING], a[id=EnX7YJV6S3CaZ8hP4kpVzg], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.949Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-3718453561548921589.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device]; ]
elasticsearch_1 | [lullabot][[lullabot][4]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][4]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-6946483988443985597.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,952][WARN ][indices.cluster ] [Power Broker] [[lullabot][2]] marking and sending shard failed due to [failed recovery]
elasticsearch_1 | [lullabot][[lullabot][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,953][WARN ][cluster.action.shard ] [Power Broker] [lullabot][2] received shard failed for target shard [[lullabot][2], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4507], s[INITIALIZING], a[id=aWlxpxapSTSKbMjNXsBQEw], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.945Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-9082933407280619053.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [failed recovery], failure [IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device]; ]
elasticsearch_1 | [lullabot][[lullabot][2]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][2]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-4174149389863621087.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:344)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | ... 11 more
elasticsearch_1 | [2017-02-14 17:03:30,953][WARN ][cluster.action.shard ] [Power Broker] [lullabot][4] received shard failed for target shard [[lullabot][4], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4883], s[INITIALIZING], a[id=EnX7YJV6S3CaZ8hP4kpVzg], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.949Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/4/translog/translog-3718453561548921589.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [master {Power Broker}{rMO1Wry9R6GHFRpWcuUyrg}{172.17.0.5}{172.17.0.5:9300} marked shard as initializing, but shard is marked as failed, resend shard failure]
elasticsearch_1 | [2017-02-14 17:03:30,953][WARN ][cluster.action.shard ] [Power Broker] [lullabot][2] received shard failed for target shard [[lullabot][2], node[rMO1Wry9R6GHFRpWcuUyrg], [P], v[4507], s[INITIALIZING], a[id=aWlxpxapSTSKbMjNXsBQEw], unassigned_info[[reason=ALLOCATION_FAILED], at[2017-02-14T17:03:30.945Z], details[failed recovery, failure IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/2/translog/translog-9082933407280619053.tlog: No space left on device]; ]]], indexUUID [pyONpeifSV6b_Ef8JCxQVg], message [master {Power Broker}{rMO1Wry9R6GHFRpWcuUyrg}{172.17.0.5}{172.17.0.5:9300} marked shard as initializing, but shard is marked as failed, resend shard failure]
elasticsearch_1 | [2017-02-14 17:03:30,953][WARN ][index.translog ] [Power Broker] [lullabot][3] failed to delete temp file /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-815805006160605036.tlog
elasticsearch_1 | java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-815805006160605036.tlog
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
elasticsearch_1 | at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
elasticsearch_1 | at java.nio.file.Files.delete(Files.java:1126)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.recoverFromFiles(Translog.java:358)
elasticsearch_1 | at org.elasticsearch.index.translog.Translog.<init>(Translog.java:179)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.openTranslog(InternalEngine.java:208)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:151)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | [2017-02-14 17:03:30,954][WARN ][indices.cluster ] [Power Broker] [[lullabot][3]] marking and sending shard failed due to [failed recovery]
elasticsearch_1 | [lullabot][[lullabot][3]] IndexShardRecoveryException[failed to recovery from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-815805006160605036.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:250)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.access$100(StoreRecoveryService.java:56)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService$1.run(StoreRecoveryService.java:129)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
elasticsearch_1 | at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
elasticsearch_1 | at java.lang.Thread.run(Thread.java:745)
elasticsearch_1 | Caused by: [lullabot][[lullabot][3]] EngineCreationFailureException[failed to create engine]; nested: FileSystemException[/usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-815805006160605036.tlog: No space left on device];
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:155)
elasticsearch_1 | at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.newEngine(IndexShard.java:1515)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.createNewEngine(IndexShard.java:1499)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:972)
elasticsearch_1 | at org.elasticsearch.index.shard.IndexShard.performTranslogRecovery(IndexShard.java:944)
elasticsearch_1 | at org.elasticsearch.index.shard.StoreRecoveryService.recoverFromStore(StoreRecoveryService.java:241)
elasticsearch_1 | ... 5 more
elasticsearch_1 | Caused by: java.nio.file.FileSystemException: /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog.ckp -> /usr/share/elasticsearch/data/elasticsearch/nodes/0/indices/lullabot/3/translog/translog-815805006160605036.tlog: No space left on device
elasticsearch_1 | at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
elasticsearch_1 | at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copyFile(UnixCopyFile.java:253)
elasticsearch_1 | at sun.nio.fs.UnixCopyFile.copy(UnixCopyFile.java:581)
elasticsearch_1 | at sun.nio.fs.UnixFileSystemProvider.copy(UnixFileSystemProvider.java:253)
elasticsearch_1 | at java.nio.file.Files.copy(Files.java:1274)
^CGracefully stopping... (press Ctrl+C again to force)
Stopping lullabotcomedit_web_1 ... done
Stopping lullabotcomedit_couchdb_1 ... done
Stopping lullabotcomedit_elasticsearch_1 ...
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment