Last active
May 10, 2019 21:08
-
-
Save clockfort/4789976034dfa2db38e16bfe3631c1be to your computer and use it in GitHub Desktop.
log for atlasdb issue 3999
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| WARN [2019-05-09T21:07:47.271Z] com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer: Error occurred talking to host '{}': {} (host: <redacted>-cassandra-20.<redacted>) (exception: com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException: This batch mutate operation requires ALL Cassandra nodes to be up and available.) | |
| WARN [2019-05-09T21:07:48.644Z] com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPool: Randomly redirected a query intended for host {} to {}. (previousHost: <redacted>-cassandra-20.<redacted>, randomHost: <redacted>-cassandra-21.<redacted>) | |
| WARN [2019-05-09T21:07:48.645Z] com.palantir.atlasdb.keyvalue.cassandra.CassandraKeyValueService: A call to table(s) liveMigration.consensus_store failed with an exception. (0: liveMigration.consensus_store) | |
| WARN [2019-05-09T21:07:48.645Z] com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer: Not reusing resource {} due to {} of host {} (host: <redacted>-cassandra-21.<redacted>) (exception: com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException: This batch mutate operation requires ALL Cassandra nodes to be up and available., resource: map[valid:true outputProtocol:map[scheme:org.apache.thrift.scheme.StandardScheme transport:map[bytesRemainingInBuffer:0 open:false buffer:<redacted> bufferPosition:29]] inputProtocol:map[transport:map[buffer:<redacted> bufferPosition:29 bytesRemainingInBuffer:0 open:false] scheme:org.apache.thrift.scheme.StandardScheme]], throwable0_message: This batch mutate operation requires ALL Cassandra nodes to be up and available., throwable1_message: <nil>) | |
| com.palantir.atlasdb.keyvalue.api.InsufficientConsistencyException: This batch mutate operation requires ALL Cassandra nodes to be up and available. | |
| at com.palantir.atlasdb.keyvalue.cassandra.WrappingQueryRunner.batchMutate(WrappingQueryRunner.java:55) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.insertTombstones(CellRangeDeleter.java:138) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.insertRangeTombstones(CellRangeDeleter.java:120) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.access$000(CellRangeDeleter.java:40) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter$1.apply(CellRangeDeleter.java:92) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter$1.apply(CellRangeDeleter.java:88) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.runWithGoodResource(CassandraClientPoolingContainer.java:143) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.runWithPooledResource(CassandraClientPoolingContainer.java:109) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolImpl.runWithPooledResourceRecordingMetrics(CassandraClientPoolImpl.java:447) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolImpl.runWithRetryOnHost(CassandraClientPoolImpl.java:394) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.deleteAllTimestampsOnSingleHost(CellRangeDeleter.java:88) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.deleteAllTimestamps(CellRangeDeleter.java:70) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraKeyValueServiceImpl.deleteAllTimestamps(CassandraKeyValueServiceImpl.java:1563) | |
| at com.palantir.atlasdb.keyvalue.impl.ProfilingKeyValueService.lambda$deleteAllTimestamps$12(ProfilingKeyValueService.java:185) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.lambda$maybeLog$0(KvsProfilingLogger.java:95) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:124) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:101) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:94) | |
| at com.palantir.atlasdb.keyvalue.impl.ProfilingKeyValueService.maybeLog(ProfilingKeyValueService.java:452) | |
| at com.palantir.atlasdb.keyvalue.impl.ProfilingKeyValueService.deleteAllTimestamps(ProfilingKeyValueService.java:185) | |
| at com.palantir.atlasdb.keyvalue.api.AutoDelegate_KeyValueService.deleteAllTimestamps(AutoDelegate_KeyValueService.java:195) | |
| at com.palantir.atlasdb.keyvalue.impl.TracingKeyValueService.deleteAllTimestamps(TracingKeyValueService.java:174) | |
| at sun.reflect.GeneratedMethodAccessor182.invoke(Unknown Source) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.execute(InvocationEventProxy.java:140) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.instrumentInvocation(InvocationEventProxy.java:117) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.handleInvocation(InvocationEventProxy.java:97) | |
| at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86) | |
| at com.sun.proxy.$Proxy81.deleteAllTimestamps(Unknown Source) | |
| at com.palantir.atlasdb.keyvalue.api.AutoDelegate_KeyValueService.deleteAllTimestamps(AutoDelegate_KeyValueService.java:195) | |
| at com.palantir.atlasdb.sweep.queue.SweepQueueDeleter.lambda$sweep$0(SweepQueueDeleter.java:73) | |
| at java.lang.Iterable.forEach(Iterable.java:75) | |
| at com.palantir.atlasdb.sweep.queue.SweepQueueDeleter.sweep(SweepQueueDeleter.java:65) | |
| at com.palantir.atlasdb.sweep.queue.SweepQueue.sweepNextBatch(SweepQueue.java:125) | |
| at com.palantir.atlasdb.sweep.queue.TargetedSweeper.sweepNextBatch(TargetedSweeper.java:177) | |
| at com.palantir.atlasdb.sweep.queue.TargetedSweeper$BackgroundSweepScheduler.lambda$runOneIteration$0(TargetedSweeper.java:223) | |
| at java.util.Optional.ifPresent(Optional.java:159) | |
| at com.palantir.atlasdb.sweep.queue.TargetedSweeper$BackgroundSweepScheduler.runOneIteration(TargetedSweeper.java:223) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.cassandra.thrift.UnavailableException: <nil> | |
| at org.apache.cassandra.thrift.Cassandra$batch_mutate_result$batch_mutate_resultStandardScheme.read(Cassandra.java:31809) | |
| at org.apache.cassandra.thrift.Cassandra$batch_mutate_result$batch_mutate_resultStandardScheme.read(Cassandra.java:31786) | |
| at org.apache.cassandra.thrift.Cassandra$batch_mutate_result.read(Cassandra.java:31720) | |
| at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:78) | |
| at org.apache.cassandra.thrift.Cassandra$Client.recv_batch_mutate(Cassandra.java:1190) | |
| at org.apache.cassandra.thrift.Cassandra$Client.batch_mutate(Cassandra.java:1176) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientImpl.lambda$batch_mutate$5(CassandraClientImpl.java:130) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientImpl.executeHandlingExceptions(CassandraClientImpl.java:299) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientImpl.batch_mutate(CassandraClientImpl.java:130) | |
| at com.palantir.atlasdb.keyvalue.cassandra.ProfilingCassandraClient.lambda$batch_mutate$11(ProfilingCassandraClient.java:171) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:124) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:106) | |
| at com.palantir.atlasdb.keyvalue.cassandra.ProfilingCassandraClient.batch_mutate(ProfilingCassandraClient.java:168) | |
| at com.palantir.atlasdb.keyvalue.cassandra.TracingCassandraClient.batch_mutate(TracingCassandraClient.java:133) | |
| at sun.reflect.GeneratedMethodAccessor148.invoke(Unknown Source) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.execute(InvocationEventProxy.java:140) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.instrumentInvocation(InvocationEventProxy.java:117) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.handleInvocation(InvocationEventProxy.java:97) | |
| at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86) | |
| at com.sun.proxy.$Proxy64.batch_mutate(Unknown Source) | |
| at com.palantir.atlasdb.keyvalue.cassandra.InstrumentedCassandraClient.batch_mutate(InstrumentedCassandraClient.java:56) | |
| at com.palantir.atlasdb.keyvalue.cassandra.QosCassandraClient.lambda$batch_mutate$3(QosCassandraClient.java:111) | |
| at com.palantir.atlasdb.keyvalue.cassandra.QosCassandraClient.execute(QosCassandraClient.java:151) | |
| at com.palantir.atlasdb.keyvalue.cassandra.QosCassandraClient.executeWrite(QosCassandraClient.java:140) | |
| at com.palantir.atlasdb.keyvalue.cassandra.QosCassandraClient.batch_mutate(QosCassandraClient.java:109) | |
| at com.palantir.atlasdb.keyvalue.cassandra.WrappingQueryRunner.lambda$batchMutate$0(WrappingQueryRunner.java:51) | |
| at com.palantir.atlasdb.keyvalue.cassandra.TracingQueryRunner.run(TracingQueryRunner.java:50) | |
| at com.palantir.atlasdb.keyvalue.cassandra.WrappingQueryRunner.batchMutate(WrappingQueryRunner.java:50) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.insertTombstones(CellRangeDeleter.java:138) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.insertRangeTombstones(CellRangeDeleter.java:120) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.access$000(CellRangeDeleter.java:40) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter$1.apply(CellRangeDeleter.java:92) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter$1.apply(CellRangeDeleter.java:88) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.runWithGoodResource(CassandraClientPoolingContainer.java:143) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolingContainer.runWithPooledResource(CassandraClientPoolingContainer.java:109) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolImpl.runWithPooledResourceRecordingMetrics(CassandraClientPoolImpl.java:447) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraClientPoolImpl.runWithRetryOnHost(CassandraClientPoolImpl.java:394) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.deleteAllTimestampsOnSingleHost(CellRangeDeleter.java:88) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CellRangeDeleter.deleteAllTimestamps(CellRangeDeleter.java:70) | |
| at com.palantir.atlasdb.keyvalue.cassandra.CassandraKeyValueServiceImpl.deleteAllTimestamps(CassandraKeyValueServiceImpl.java:1563) | |
| at com.palantir.atlasdb.keyvalue.impl.ProfilingKeyValueService.lambda$deleteAllTimestamps$12(ProfilingKeyValueService.java:185) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.lambda$maybeLog$0(KvsProfilingLogger.java:95) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:124) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:101) | |
| at com.palantir.atlasdb.logging.KvsProfilingLogger.maybeLog(KvsProfilingLogger.java:94) | |
| at com.palantir.atlasdb.keyvalue.impl.ProfilingKeyValueService.maybeLog(ProfilingKeyValueService.java:452) | |
| at com.palantir.atlasdb.keyvalue.impl.ProfilingKeyValueService.deleteAllTimestamps(ProfilingKeyValueService.java:185) | |
| at com.palantir.atlasdb.keyvalue.api.AutoDelegate_KeyValueService.deleteAllTimestamps(AutoDelegate_KeyValueService.java:195) | |
| at com.palantir.atlasdb.keyvalue.impl.TracingKeyValueService.deleteAllTimestamps(TracingKeyValueService.java:174) | |
| at sun.reflect.GeneratedMethodAccessor182.invoke(Unknown Source) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.execute(InvocationEventProxy.java:140) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.instrumentInvocation(InvocationEventProxy.java:117) | |
| at com.palantir.tritium.proxy.InvocationEventProxy.handleInvocation(InvocationEventProxy.java:97) | |
| at com.google.common.reflect.AbstractInvocationHandler.invoke(AbstractInvocationHandler.java:86) | |
| at com.sun.proxy.$Proxy81.deleteAllTimestamps(Unknown Source) | |
| at com.palantir.atlasdb.keyvalue.api.AutoDelegate_KeyValueService.deleteAllTimestamps(AutoDelegate_KeyValueService.java:195) | |
| at com.palantir.atlasdb.sweep.queue.SweepQueueDeleter.lambda$sweep$0(SweepQueueDeleter.java:73) | |
| at java.lang.Iterable.forEach(Iterable.java:75) | |
| at com.palantir.atlasdb.sweep.queue.SweepQueueDeleter.sweep(SweepQueueDeleter.java:65) | |
| at com.palantir.atlasdb.sweep.queue.SweepQueue.sweepNextBatch(SweepQueue.java:125) | |
| at com.palantir.atlasdb.sweep.queue.TargetedSweeper.sweepNextBatch(TargetedSweeper.java:177) | |
| at com.palantir.atlasdb.sweep.queue.TargetedSweeper$BackgroundSweepScheduler.lambda$runOneIteration$0(TargetedSweeper.java:223) | |
| at java.util.Optional.ifPresent(Optional.java:159) | |
| at com.palantir.atlasdb.sweep.queue.TargetedSweeper$BackgroundSweepScheduler.runOneIteration(TargetedSweeper.java:223) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) | |
| at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment