Cassandra 2.1 - Repair issue (2)

gistfile1.txt

INFO  [AntiEntropyStage:6] 2014-09-28 22:27:45,482 RepairSession.java:171 - [repair #e4bf3e70-474d-11e4-a221-2fe2626539ec] Received merkle tree for domain from /172.16.66.81
ERROR [RepairJobTask:1] 2014-09-28 22:27:45,484 RepairJob.java:127 - Error occurred during snapshot phase
java.lang.RuntimeException: Could not create snapshot at /172.16.66.81
	at org.apache.cassandra.repair.SnapshotTask$SnapshotCallback.onFailure(SnapshotTask.java:77) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:48) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:62) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_67]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_67]
	at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
ERROR [AntiEntropyStage:6] 2014-09-28 22:27:45,484 CassandraDaemon.java:166 - Exception in thread Thread[AntiEntropyStage:6,5,main]
java.lang.ClassCastException: java.math.BigInteger cannot be cast to java.nio.ByteBuffer
	at org.apache.cassandra.db.marshal.UUIDType.compare(UUIDType.java:44) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.LocalToken.compareTo(LocalToken.java:44) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.LocalToken.compareTo(LocalToken.java:24) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.Range.contains(Range.java:71) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.Range.contains(Range.java:111) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.Range.intersects(Range.java:142) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.Range.intersects(Range.java:129) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.dht.AbstractBounds.intersects(AbstractBounds.java:83) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.repair.RepairMessageVerbHandler$1.apply(RepairMessageVerbHandler.java:83) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.repair.RepairMessageVerbHandler$1.apply(RepairMessageVerbHandler.java:80) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.db.ColumnFamilyStore.snapshotWithoutFlush(ColumnFamilyStore.java:2152) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.db.ColumnFamilyStore.snapshot(ColumnFamilyStore.java:2215) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:79) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:62) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_67]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.7.0_67]
	at java.lang.Thread.run(Thread.java:745) ~[na:1.7.0_67]
ERROR [AntiEntropySessions:6] 2014-09-28 22:27:45,485 RepairSession.java:303 - [repair #e4bf3e70-474d-11e4-a221-2fe2626539ec] session completed with the following error
java.io.IOException: Failed during snapshot creation.
	at org.apache.cassandra.repair.RepairSession.failedSnapshot(RepairSession.java:344) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.repair.RepairJob$2.onFailure(RepairJob.java:128) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) ~[guava-16.0.jar:na]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_67]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_67]
	at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
INFO  [AntiEntropySessions:7] 2014-09-28 22:27:45,486 RepairSession.java:260 - [repair #e7caf7d0-474d-11e4-a221-2fe2626539ec] new session: will sync /172.16.66.81, /172.16.66.3 on range (8412985223068976986243322671985114129,8512706148121614197056821748995514375] for m2mp.[registrynode, generalsettings, domain, user, registrynodechildren, timeseries_index, timeseries, registrynodedata]
ERROR [AntiEntropySessions:6] 2014-09-28 22:27:45,487 CassandraDaemon.java:166 - Exception in thread Thread[AntiEntropySessions:6,5,RMI Runtime]
java.lang.RuntimeException: java.io.IOException: Failed during snapshot creation.
	at com.google.common.base.Throwables.propagate(Throwables.java:160) ~[guava-16.0.jar:na]
	at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_67]
	at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_67]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_67]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_67]
	at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
Caused by: java.io.IOException: Failed during snapshot creation.
	at org.apache.cassandra.repair.RepairSession.failedSnapshot(RepairSession.java:344) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at org.apache.cassandra.repair.RepairJob$2.onFailure(RepairJob.java:128) ~[apache-cassandra-2.1.0.jar:2.1.0]
	at com.google.common.util.concurrent.Futures$4.run(Futures.java:1172) ~[guava-16.0.jar:na]
	... 3 common frames omitted
INFO  [AntiEntropyStage:7] 2014-09-28 22:27:45,489 ColumnFamilyStore.java:856 - Enqueuing flush of registrynode: 2230 (0%) on-heap, 0 (0%) off-heap

I might have solved it

I got rid of this error and it seems the repair is back on track...

I did a nodetool flush, stopped cassandra, removed the logs, restarted it.

Some other things I did that might also have solved the issue:

• I removed the "saved_cache" files on the "source" server and deleted two empty column families
• I tried a repair on some specific tables before re-starting it on the complete keyspace