-
-
Save ronjakoi/5ef1eecb58907fc791bb176123259996 to your computer and use it in GitHub Desktop.
{ | |
"responseHeader": { | |
"status": 500, | |
"QTime": 17 | |
}, | |
"Operation backup caused exception:": "java.nio.file.NoSuchFileException:java.nio.file.NoSuchFileException: /opt/solr-backup/testi", | |
"exception": { | |
"msg": "/opt/solr-backup/testi", | |
"rspCode": -1 | |
}, | |
"error": { | |
"metadata": [ | |
"error-class", | |
"org.apache.solr.common.SolrException", | |
"root-error-class", | |
"org.apache.solr.common.SolrException" | |
], | |
"msg": "/opt/solr-backup/testi", | |
"trace": "org.apache.solr.common.SolrException: /opt/solr-backup/testi | |
at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:300) | |
at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:237) | |
at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:215) | |
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173) | |
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748) | |
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729) | |
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510) | |
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361) | |
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305) | |
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691) | |
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) | |
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) | |
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) | |
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226) | |
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) | |
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) | |
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) | |
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) | |
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) | |
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213) | |
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119) | |
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) | |
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335) | |
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) | |
at org.eclipse.jetty.server.Server.handle(Server.java:534) | |
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) | |
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) | |
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273) | |
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95) | |
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) | |
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) | |
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) | |
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) | |
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) | |
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) | |
at java.lang.Thread.run(Thread.java:748) | |
", | |
"code": 500 | |
} | |
} |
2018-03-07 14:36:02.577 INFO (qtp985934102-15) [ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :backup with params name=testi&action=BACKUP&location=/opt/solr-backup/&collection=intranet_2018-03-07T115247Z&wt=json and sendToOCPQueue=true | |
2018-03-07 14:36:02.594 INFO (qtp985934102-15) [ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=testi&action=BACKUP&location=/opt/solr-backup/&collection=intranet_2018-03-07T115247Z&wt=json} status=500 QTime=17 | |
2018-03-07 14:36:02.595 ERROR (qtp985934102-15) [ ] o.a.s.s.HttpSolrCall null:org.apache.solr.common.SolrException: /opt/solr-backup/testi | |
at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:300) | |
at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:237) | |
at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:215) | |
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173) | |
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748) | |
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729) | |
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510) | |
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361) | |
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305) | |
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691) | |
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) | |
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) | |
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) | |
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226) | |
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) | |
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) | |
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) | |
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) | |
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) | |
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213) | |
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119) | |
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) | |
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335) | |
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) | |
at org.eclipse.jetty.server.Server.handle(Server.java:534) | |
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) | |
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) | |
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273) | |
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95) | |
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) | |
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) | |
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) | |
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) | |
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) | |
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) | |
at java.lang.Thread.run(Thread.java:748) |
Ok wow.
I found it.
The solr documentation states that the backup location should be a "shared filesystem". It should really clarify what exactly it means by that.
It means: The backup location must literally be the exact same across all solrs.
In other words: it has to be an NFS mount or some other mount.
And, the most important bit: it has to be mounted on every single machine the solr cloud exists on.
Because it's not because the command to do the backup was executed on the first of X machines, that only that machine will execute it.
The job is spread over all machines, and all of them do checks to see if directories they expect to exists, can be found.
Machine 2 will expect to find what machine 1 has written. And if it doesn't, this kind of error happens.
Thank you, internet stranger from the year 2018, for being my rubber debugging duck. Even if you didn't know it.
Hello @ronjakoi
Did you ever found out the cause for this?
I'm running into something similar, though the location I'm writing to is an NFS mount. Doing manual writes as the solr user works as expected, but backup gives this exception.
Did you ever fix it?