Last active
February 19, 2023 17:36
-
-
Save spullara/8150289 to your computer and use it in GitHub Desktop.
Server-side count of keys for FoundationDB. Still has to scan the complete database. Modified with input from the FDB team to use getKey.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// 1) Stride through the database using KeySelectors until you pass the end of the database | |
// 2) Then, back off until you find the last key | |
final AtomicBoolean retry = new AtomicBoolean(); | |
Function<Transaction, Long> function = new Function<Transaction, Long>() { | |
private long start = System.currentTimeMillis(); | |
private long count = 0; | |
private int offset = 1000000; | |
private KeySelector keySelector = KeySelector.firstGreaterOrEqual(new byte[0]); | |
private boolean narrowing = false; | |
@Override | |
public Long apply(Transaction transaction) { | |
// Just in case it tries to load the data | |
transaction.options().setReadYourWritesDisable(); | |
if (retry.get()) { | |
offset = Math.max(1, offset /= 2); | |
System.out.println("Retry, reducing offset size to: " + offset); | |
} else { | |
retry.set(true); | |
} | |
byte[] key = transaction.getKey(keySelector.add(offset)).get(); | |
if (isSystemKey(key)) { | |
offset = Math.max(1, offset /= 2); | |
narrowing = true; | |
} else { | |
keySelector = KeySelector.firstGreaterThan(key); | |
count += offset; | |
if (narrowing) { | |
key = transaction.getKey(keySelector).get(); | |
if (isSystemKey(key)) { | |
return count; | |
} | |
} | |
} | |
System.out.println(count + " @ " + (1000l * count / (System.currentTimeMillis() - start)) + " per second"); | |
return null; | |
} | |
private boolean isSystemKey(byte[] key) { | |
return key.length == 1 && key[0] == -1; | |
} | |
}; | |
Long count; | |
while ((count = database.run(function)) == null) { | |
retry.set(false); | |
} | |
System.out.println(count + " keys"); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment