DRBD debug

global config

global
{
  usage-count yes;
}

common
{
  protocol C;

  handlers
  {
    pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
    pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
    local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
    split-brain "/usr/lib/drbd/send_email.php";
    out-of-sync "/usr/lib/mcvirt/mcvirt-drbd.py > /dev/null 2>&1";
  }

  startup
  {
    wfc-timeout 15;
    degr-wfc-timeout 60;
  }

  net
  {
    ko-count 50;
    cram-hmac-alg sha1;
    shared-secret "*******";
    after-sb-0pri discard-zero-changes;
    after-sb-1pri discard-secondary;
    after-sb-2pri disconnect;
  }

  disk 
  {
    c-plan-ahead 0;
  }

  syncer
  {
    verify-alg sha1;
    rate 10M;
  }
}

invalidate + verify logs

Oct  5 09:52:49 node2 kernel: [9596315.312642] block drbd2: Resync done (total 833 sec; paused 0 sec; 10068 K/sec)
Oct  5 09:52:49 node2 kernel: [9596315.312652] block drbd2: updated UUIDs 0397F8B93DD2BF6E:0000000000000000:1BA75983ECCF8BC0:6949E223AE7A57C0
Oct  5 09:52:49 node2 kernel: [9596315.312662] block drbd2: conn( SyncTarget -> Connected ) disk( Inconsistent -> UpToDate ) 
Oct  5 09:52:49 node2 kernel: [9596315.462073] block drbd2: helper command: /sbin/drbdadm after-resync-target minor-2
Oct  5 09:52:49 node2 kernel: [9596315.465274] block drbd2: helper command: /sbin/drbdadm after-resync-target minor-2 exit code 0 (0x0)
Oct  5 10:13:36 node2 kernel: [9597562.957033] block drbd2: conn( Connected -> StartingSyncT ) disk( UpToDate -> Outdated ) 
Oct  5 10:13:36 node2 kernel: [9597563.246609] block drbd2: bitmap WRITE of 64 pages took 3 jiffies
Oct  5 10:13:36 node2 kernel: [9597563.267543] block drbd2: 8192 MB (2097152 bits) marked out-of-sync by on disk bit-map.
Oct  5 10:13:36 node2 kernel: [9597563.415797] block drbd2: conn( StartingSyncT -> WFSyncUUID ) 
Oct  5 10:13:36 node2 kernel: [9597563.415807] block drbd2: updated sync uuid F1A22A083E4F7CD6:0000000000000000:1BA75983ECCF8BC0:6949E223AE7A57C0
Oct  5 10:13:36 node2 kernel: [9597563.567902] block drbd2: helper command: /sbin/drbdadm before-resync-target minor-2
Oct  5 10:13:36 node2 kernel: [9597563.570977] block drbd2: helper command: /sbin/drbdadm before-resync-target minor-2 exit code 0 (0x0)
Oct  5 10:13:36 node2 kernel: [9597563.571007] block drbd2: conn( WFSyncUUID -> SyncTarget ) disk( Outdated -> Inconsistent ) 
Oct  5 10:13:36 node2 kernel: [9597563.571022] block drbd2: Began resync as SyncTarget (will sync 8388608 KB [2097152 bits set]).
Oct  5 10:27:28 node2 kernel: [9598396.293991] block drbd2: Resync done (total 832 sec; paused 0 sec; 10080 K/sec)
Oct  5 10:27:28 node2 kernel: [9598396.294002] block drbd2: updated UUIDs 0397F8B93DD2BF6E:0000000000000000:F1A22A083E4F7CD6:1BA75983ECCF8BC0
Oct  5 10:27:28 node2 kernel: [9598396.294011] block drbd2: conn( SyncTarget -> Connected ) disk( Inconsistent -> UpToDate ) 
Oct  5 10:27:28 node2 kernel: [9598396.327158] block drbd2: helper command: /sbin/drbdadm after-resync-target minor-2
Oct  5 10:27:28 node2 kernel: [9598396.330325] block drbd2: helper command: /sbin/drbdadm after-resync-target minor-2 exit code 0 (0x0)
Oct  5 10:27:49 node2 kernel: [9598416.517565] block drbd2: conn( Connected -> VerifyS ) 
Oct  5 10:27:49 node2 kernel: [9598416.517571] block drbd2: Starting Online Verify from sector 0
Oct  5 10:41:15 node2 kernel: [9599223.447735] block drbd2: Out of sync: start=16512744, size=8 (sectors)
Oct  5 10:41:15 node2 kernel: [9599223.849300] block drbd2: Out of sync: start=16519168, size=8 (sectors)
Oct  5 10:41:15 node2 kernel: [9599223.853084] block drbd2: Out of sync: start=16519648, size=8 (sectors)
Oct  5 10:41:15 node2 kernel: [9599223.855322] block drbd2: Out of sync: start=16519720, size=8 (sectors)
Oct  5 10:41:16 node2 kernel: [9599224.653895] block drbd2: Out of sync: start=16536160, size=8 (sectors)
Oct  5 10:41:22 node2 kernel: [9599230.875010] block drbd2: Out of sync: start=16663464, size=8 (sectors)
Oct  5 10:41:28 node2 kernel: [9599236.374961] block drbd2: Online verify done (total 819 sec; paused 0 sec; 10240 K/sec)
Oct  5 10:41:28 node2 kernel: [9599236.374967] block drbd2: Online verify found 6 4k block out of sync!
Oct  5 10:41:28 node2 kernel: [9599236.376993] block drbd2: conn( VerifyS -> Connected ) 
Oct  5 10:41:28 node2 kernel: [9599236.377017] block drbd2: helper command: /sbin/drbdadm out-of-sync minor-2
Oct  5 10:41:28 node2 kernel: [9599236.444131] block drbd2: helper command: /sbin/drbdadm out-of-sync minor-2 exit code 0 (0x0)

proc-drbd

root@node2:~# cat /proc/drbd
version: 8.4.3 (api:1/proto:86-101)
srcversion: 6551AD2C98F533733BE558C 

 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:357716533 dw:357716533 dr:1677721600 al:0 bm:7 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
 2: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:460489344 dw:460489344 dr:167772160 al:0 bm:2585 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:24
 3: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r-----
    ns:1827692563 nr:22438544 dw:1850131239 dr:325127543 al:7640 bm:51 lo:1 pe:0 ua:0 ap:1 ep:1 wo:f oos:32
 4: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:44290019 dw:44290019 dr:125829120 al:0 bm:516 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
 5: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:16648402 dw:16648402 dr:125829120 al:0 bm:9 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
 6: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:62010007 dw:62010007 dr:157286400 al:0 bm:23 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
 7: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:59509457 dw:59509457 dr:235929600 al:0 bm:947 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:24
 8: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:44401359 dw:44401359 dr:139149312 al:0 bm:15 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
 9: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:49905368 dw:49905368 dr:204472320 al:0 bm:27 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:4
10: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:726642383 dw:726642383 dr:293601280 al:0 bm:40 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
11: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:73045002 dw:73045002 dr:109051904 al:0 bm:514 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
12: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:43356937 dw:43356937 dr:109051904 al:0 bm:520 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:132
13: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:792194920 dw:792194920 dr:1633681408 al:0 bm:15 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
14: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:85522440 dw:85522440 dr:113287696 al:0 bm:3032 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:8
15: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:1794357008 dw:1794357008 dr:1363148800 al:0 bm:12597 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:8
16: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r-----
    ns:36916022 nr:0 dw:28527802 dr:130065197 al:2072 bm:519 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
17: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r-----
    ns:11155880 nr:0 dw:3084436 dr:42319863 al:378 bm:499 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
18: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r-----
    ns:401437834 nr:0 dw:200058934 dr:923647515 al:10186 bm:12502 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:4084

Resource config

resource mcvirt_vm-vm-name-disk-1
{
  on node1
  {
    device /dev/drbd2;
    disk /dev/node1/mcvirt_vm-vm-name-disk-1-drbd-raw;
    address 1.2.3.3:7790;
    flexible-meta-disk /dev/node1/mcvirt_vm-vm-name-disk-1-drbd-meta;
  }
  on node2
  {
    device /dev/drbd2;
    disk /dev/node2/mcvirt_vm-vm-name-disk-1-drbd-raw;
    address 1.2.3.4:7790;
    flexible-meta-disk /dev/node2/mcvirt_vm-vm-name-disk-1-drbd-meta;
  }
}

drbd -V:

DRBDADM_BUILDTAG=GIT-hash:\ 599f286440bd633d15d5ff985204aff4bccffadd\ build\ by\ phil@fat-tyre\,\ 2013-10-11\ 16:42:48
DRBDADM_API_VERSION=1
DRBD_KERNEL_VERSION_CODE=0x080403
DRBDADM_VERSION_CODE=0x080404
DRBDADM_VERSION=8.4.4