Example of cassandra verify ( #8703 ) for cassandra-8703

gistfile1.txt

# Data actively being written - 100,000 counter updates
bash-3.2# ./bin/nodetool flush
bash-3.2# ./bin/sstableverify test metrics
WARN  09:43:52 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') (1987623 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') succeeded. All 20077 rows read successfully
bash-3.2# ./bin/nodetool flush
bash-3.2# ./bin/sstableverify test metrics
WARN  09:44:02 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-2-big-Data.db') (1315710 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-2-big-Data.db') succeeded. All 13290 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') (1987623 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') succeeded. All 20077 rows read successfully
bash-3.2# ./bin/nodetool flush
bash-3.2# ./bin/sstableverify test metrics
WARN  09:44:16 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-2-big-Data.db') (1315710 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-2-big-Data.db') succeeded. All 13290 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-3-big-Data.db') (1442628 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-3-big-Data.db') succeeded. All 14572 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') (1987623 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') succeeded. All 20077 rows read successfully
bash-3.2# ./bin/nodetool  verify
bash-3.2# ./bin/nodetool  verify test metrics

# At this point, all writing is done
bash-3.2# ./bin/nodetool  flush
bash-3.2# ./bin/sstableverify test metrics
WARN  09:44:58 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') (1987623 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') succeeded. All 20077 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-4-big-Data.db') (1860210 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-4-big-Data.db') succeeded. All 18790 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-2-big-Data.db') (1315710 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-2-big-Data.db') succeeded. All 13290 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-3-big-Data.db') (1442628 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-3-big-Data.db') succeeded. All 14572 rows read successfully
bash-3.2# ./bin/nodetool  scrub
bash-3.2# ./bin/sstableverify test metrics
WARN  09:45:14 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-5-big-Data.db') (1442628 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-5-big-Data.db') succeeded. All 14572 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-7-big-Data.db') (1860210 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-7-big-Data.db') succeeded. All 18790 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-8-big-Data.db') (1987623 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-8-big-Data.db') succeeded. All 20077 rows read successfully
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-6-big-Data.db') (1315710 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-6-big-Data.db') succeeded. All 13290 rows read successfully
bash-3.2# ./bin/nodetool  compact
bash-3.2# ./bin/sstableverify test metrics
WARN  09:45:30 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-9-big-Data.db') (6606171 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-9-big-Data.db') succeeded. All 66729 rows read successfully

# Now we intentionally corrupt the data, a single byte in the middle of the sstable
bash-3.2# dd if=/dev/zero of=/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-9-big-Data.db bs=1 count=1 seek=657 conv=notrunc 
1+0 records in
1+0 records out
1 bytes transferred in 0.000022 secs (45590 bytes/sec)
bash-3.2# ./bin/sstableverify test metrics
WARN  09:46:05 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-9-big-Data.db') (6606171 bytes)
Error verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-9-big-Data.db'): java.lang.Exception: Invalid SSTable, please force repair
bash-3.2# ./bin/nodetool verify
error: Invalid SSTable, please force repair
-- StackTrace --
java.lang.Exception: Invalid SSTable, please force repair
	at org.apache.cassandra.db.compaction.Verifier.verify(Verifier.java:190)
	at org.apache.cassandra.db.compaction.CompactionManager.verifyOne(CompactionManager.java:689)
	at org.apache.cassandra.db.compaction.CompactionManager.access$400(CompactionManager.java:98)
	at org.apache.cassandra.db.compaction.CompactionManager$4.execute(CompactionManager.java:348)
	at org.apache.cassandra.db.compaction.CompactionManager$2.call(CompactionManager.java:284)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:724)

# To recover, a full repair is required. Should discuss the 'right' way to trigger this (set sstable as unrepaired? immediately force repair? Unsure
# For now, we kill the sstable and restart the node
bash-3.2# ccm node1 stop
bash-3.2# rm /Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-9-big-Data.db
bash-3.2# ccm node1 start
bash-3.2# ./bin/nodetool  repair
[2015-01-31 01:47:10,304] Starting repair command #1, repairing keyspace test with repair options (parallelism: parallel, primary range: false, incremental: true, job threads: 1, ColumnFamilies: [], dataCenters: [], hosts: [], # of ranges: 2)
[2015-01-31 01:47:11,539] Repair session 2048cff0-a92e-11e4-bf65-8ff7b1eb6f10 for range (3074457345618258602,-9223372036854775808] finished
[2015-01-31 01:47:11,805] Repair session 204a2f80-a92e-11e4-bf65-8ff7b1eb6f10 for range (-3074457345618258603,3074457345618258602] finished
[2015-01-31 01:47:11,827] Repair command #1 finished in 1 second
[2015-01-31 01:47:11,836] Nothing to repair for keyspace 'system_auth'
[2015-01-31 01:47:11,844] Starting repair command #2, repairing keyspace system_traces with repair options (parallelism: parallel, primary range: false, incremental: true, job threads: 1, ColumnFamilies: [], dataCenters: [], hosts: [], # of ranges: 2)
[2015-01-31 01:47:12,909] Repair session 21d31970-a92e-11e4-bf65-8ff7b1eb6f10 for range (3074457345618258602,-9223372036854775808] finished
[2015-01-31 01:47:12,914] Repair session 21d31971-a92e-11e4-bf65-8ff7b1eb6f10 for range (-3074457345618258603,3074457345618258602] finished
[2015-01-31 01:47:12,915] Repair command #2 finished in 1 second
bash-3.2# ./bin/nodetool verify test metrics
bash-3.2# ./bin/sstableverify test metrics
WARN  09:47:39 JNA link failure, one or more native method will be unavailable.
Verifying BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') (6606171 bytes)
Verify of BigTableReader(path='/Users/jeff/.ccm/snapshot/node1/data/test/metrics-963efb90a92d11e48cd58ff7b1eb6f10/la-1-big-Data.db') succeeded. All 66729 rows read successfully