tbg

E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245  a panic has occurred!
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +‹can't leave a non-joint config›
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +(1) attached stack trace
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +  -- stack trace:
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +  | runtime.gopanic
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +  |      GOROOT/src/runtime/panic.go:884
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +  | go.etcd.io/raft/v3.(*raft).applyConfChange
E230706 18:27:30.787545 188 1@util/log/logcrash/crash_reporting.go:188 ⋮ [T1,n9] 2245 +  |      go.etcd.io/raft/v3/external/io_etcd_go_raft_v3/raft.go:1870
E230706 18:27:30.787545 188 1@util/log/logcrash/crash

tbg

goroutine 408771655 [running]:
runtime/pprof.writeGoroutineStacks({0x7f2f2fac2818, 0xc09ec5aa00})
	GOROOT/src/runtime/pprof/pprof.go:692 +0x70
runtime/pprof.writeGoroutine({0x7f2f2fac2818?, 0xc09ec5aa00?}, 0x0?)
	GOROOT/src/runtime/pprof/pprof.go:681 +0x2b
runtime/pprof.(*Profile).WriteTo(0x55d8d00?, {0x7f2f2fac2818?, 0xc09ec5aa00?}, 0xc?)
	GOROOT/src/runtime/pprof/pprof.go:330 +0x14b
net/http/pprof.handler.ServeHTTP({0xc01417a4c1, 0x9}, {0x6f81db0, 0xc09ec5aa00}, 0x702f67756265642f?)
	GOROOT/src/net/http/pprof/pprof.go:253 +0x4a5
net/http/pprof.Index({0x6f81db0?, 0xc09ec5aa00}, 0xc387927600)

tbg

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,

tbg

=== RUN   TestWriteWithStuckLeaseholderDisk
    test_log_scope.go:161: test logs captured to: /var/folders/bw/w49dhnhx61xdh_t25_zscjpc0000gq/T/logTestWriteWithStuckLeaseholderDisk2758821677
    test_log_scope.go:79: use -show-logs to present logs inline
    client_stuck_disk_test.go:423: write 0: <nil>
    client_stuck_disk_test.go:423: write 1: <nil>
    client_stuck_disk_test.go:423: write 2: <nil>
    client_stuck_disk_test.go:423: write 3: <nil>
    client_stuck_disk_test.go:423: write 4: <nil>
    client_stuck_disk_test.go:423: write 5: <nil>
    client_stuck_disk_test.go:423: write 6: <nil>

tbg

// DumpMemSST iterates through the SST represented by the input bytes and returns
// all MVCC point and range key values in it.
//
// Not optimized.
// For testing only.
func DumpMemSST(sst []byte) (ps []MVCCKeyValue, rs []MVCCRangeKeyValue, _ error) {
	it, err := NewMemSSTIterator(sst, false, IterOptions{
		LowerBound: keys.MinKey,
		UpperBound: keys.MaxKey,
		KeyTypes:   IterKeyTypePointsAndRanges,

tbg

$ benchstat benchdiff/50c064e/artifacts/out.2022-12-21T13_05_49Z benchdiff/52671d3/artifacts/out.2022-12-21T13_05_49Z
name                                                 old time/op    new time/op    delta
ReplicaProposal/bytes=256_B,withFollower=false-24       108µs ± 4%     109µs ± 5%     ~     (p=0.286 n=30+30)
ReplicaProposal/bytes=256_B,withFollower=true-24        139µs ± 4%     140µs ± 6%     ~     (p=0.273 n=30+30)
ReplicaProposal/bytes=512_B,withFollower=false-24       112µs ± 5%     113µs ± 5%     ~     (p=0.217 n=30+27)
ReplicaProposal/bytes=512_B,withFollower=true-24        146µs ± 4%     145µs ± 3%     ~     (p=0.255 n=30+27)
ReplicaProposal/bytes=1.0_KiB,withFollower=false-24     123µs ± 7%     124µs ± 6%     ~     (p=0.227 n=29+29)
ReplicaProposal/bytes=1.0_KiB,withFollower=true-24      156µs ± 3%     157µs ± 4%     ~     (p=0.362 n=29+30)
ReplicaProposal/bytes=256_KiB,withFollower=false-24    2.48ms ± 7%    2.56ms ± 7%   +3.39%  (p=0.002 n=30+29)
ReplicaProposal/bytes=256_KiB,withFollower=true

tbg

WriteBatch gets created internally in a storage.Batch // instance 1

https://github.com/cockroachdb/cockroach/blob/f089b43803b1d59d4752238e5b4bb37cfe0fbf8d/pkg/kv/kvserver/replica_proposal.go#L713-L717

The impl actually copies, womp womp (the batch reuses the original memory but still): // instance 2

https://github.com/cockroachdb/cockroach/blob/5bafe901cd6fd4ceee00ee922e7f13ed95c4134b/pkg/storage/pebble_batch.go#L567-L576

Aside: If we kept the batch around with the proposal, we could conceivably use its backing memory. We could also apply the batch directly, possibly saving overhead on the leaseholder.

tbg

Advent of protobuf hackery

You have this proto message.

message RequestHeader {
   // some fields
}

You want to add a field to it that is "there" only during crdb_test.

tbg

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,

tbg

AVG_DURATION_SEC	TEST_NAME
88.719246032000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestReplicateQueueSwapVotersWithNonVoters
67.954845606000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestReplicateQueueRebalanceMultiStore
59.736863905000	github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptstorage: TestStorage
52.481829026000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestMergeQueueSeesNonVoters
49.185198413000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestReplicateQueueDecommissioningNonVoters
48.835447316000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestResetQuorum
42.682879684000	github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptstorage: TestStorage/Protect_-_unlimited_bytes
41.607149644000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestReplicateQueueRebalanceMultiStore/simple
41.130417495000	github.com/cockroachdb/cockroach/pkg/kv/kvserver: TestStoreRangeMergeTimestampCache