Using dev-run-ovndb to see ovn trace when things were broken

ovn-trace.txt

❯ cp ./must-gather.local.4438226141351332503/quay-io-openshift-release-dev-ocp-v4-0-art-dev-sha256-e3b54a445db4bcf97ad6fef54e39a6da3b055e137c86eade9cfd815854b20c09/network_logs/ovnkube-master-rnsww_sbdb.gz \
  ~/sbdb.db.gz
❯ cd ; gunzip sbdb.db.gz
❯ git clone git@github.com:openshift/dev-knowledge-base.git
❯ cd dev-knowledge-base/scripts
❯ mv ~/sbdb.db .
❯ ./dev-run-ovndb sbdb.db s

[root@8346d29bf786 ~]# ovn-trace --ct new anurag47a-bwr7f-master-0 \
   'inport=="openshift-ingress-operator_ingress-operator-79c596f44f-z2rmz" &&
   eth.src==0a:58:0a:82:00:16 && eth.dst==0a:58:0a:82:00:01 && tcp && 
   tcp.src==56572 && tcp.dst==80 && ip4.src==10.130.0.22 && ip4.dst==192.168.0.7 && ip.ttl==64'
   
[root@8346d29bf786 ~]# ovn-trace --ct new anurag47a-bwr7f-master-0 'inport=="openshift-ingress-operator_ingress-operator-79c596f44f-z2rmz" &&
> eth.src==0a:58:0a:82:00:16 && eth.dst==0a:58:0a:82:00:01 &&
> tcp && tcp.src==56572 && tcp.dst==80 && ip4.src==10.130.0.22 && ip4.dst==192.168.0.7 && ip.ttl==64'

# tcp,reg14=0x82,vlan_tci=0x0000,dl_src=0a:58:0a:82:00:16,dl_dst=0a:58:0a:82:00:01,nw_src=10.130.0.22,nw_dst=192.168.0.7,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=56572,tp_dst=80,tcp_flags=0

ingress(dp="anurag47a-bwr7f-master-0", inport="openshift-ingress-operator_ingress-operator-79c596f44f-z2rmz")
-------------------------------------------------------------------------------------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:5036): inport == "openshift-ingress-operator_ingress-operator-79c596f44f-z2rmz" && eth.src == {0a:58:0a:82:00:16}, priority 50, uuid 179643bb
    next;
 1. ls_in_port_sec_ip (ovn-northd.c:4684): inport == "openshift-ingress-operator_ingress-operator-79c596f44f-z2rmz" && eth.src == 0a:58:0a:82:00:16 && ip4.src == {10.130.0.22}, priority 90, uuid 4cdf5def
    next;
 5. ls_in_pre_acl (ovn-northd.c:5237): ip, priority 100, uuid a2dadb42
    reg0[0] = 1;
    next;
 6. ls_in_pre_lb (ovn-northd.c:5405): ip, priority 100, uuid c839d81f
    reg0[2] = 1;
    next;
 7. ls_in_pre_stateful (ovn-northd.c:5432): reg0[2] == 1 && ip4 && tcp, priority 120, uuid 1ddee10d
    reg1 = ip4.dst;
    reg2[0..15] = tcp.dst;
    ct_lb;

ct_lb
-----
 8. ls_in_acl_hint (ovn-northd.c:5525): !ct.trk, priority 5, uuid 5c6a21ff
    reg0[8] = 1;
    reg0[9] = 1;
    next;
22. ls_in_l2_lkup (ovn-northd.c:7609): eth.dst == 0a:58:0a:82:00:01, priority 50, uuid b9ac2d3c
    outport = "stor-anurag47a-bwr7f-master-0";
    output;

egress(dp="anurag47a-bwr7f-master-0", inport="openshift-ingress-operator_ingress-operator-79c596f44f-z2rmz", outport="stor-anurag47a-bwr7f-master-0")
-----------------------------------------------------------------------------------------------------------------------------------------------------
 0. ls_out_pre_lb (ovn-northd.c:5182): ip && outport == "stor-anurag47a-bwr7f-master-0", priority 110, uuid 26df4ff1
    next;
 1. ls_out_pre_acl (ovn-northd.c:5182): ip && outport == "stor-anurag47a-bwr7f-master-0", priority 110, uuid fac03d41
    next;
 3. ls_out_acl_hint (ovn-northd.c:5525): !ct.trk, priority 5, uuid a4fe6d1b
    reg0[8] = 1;
    reg0[9] = 1;
    next;
 9. ls_out_port_sec_l2 (ovn-northd.c:5131): outport == "stor-anurag47a-bwr7f-master-0", priority 50, uuid 4eef0b81
    output;
    /* output to "stor-anurag47a-bwr7f-master-0", type "patch" */

ingress(dp="ovn_cluster_router", inport="rtos-anurag47a-bwr7f-master-0")
------------------------------------------------------------------------
 0. lr_in_admission (ovn-northd.c:9657): eth.dst == 0a:58:0a:82:00:01 && inport == "rtos-anurag47a-bwr7f-master-0", priority 50, uuid c914755c
    xreg0[0..47] = 0a:58:0a:82:00:01;
    next;
 1. lr_in_lookup_neighbor (ovn-northd.c:9736): 1, priority 0, uuid c8d6c6d3
    reg9[2] = 1;
    next;
 2. lr_in_learn_neighbor (ovn-northd.c:9745): reg9[2] == 1, priority 100, uuid 2f61bd8b
    next;
10. lr_in_ip_routing (ovn-northd.c:8704): ip4.src == 10.130.0.0/23, priority 46, uuid 573d0e79
    ip.ttl--;
    reg8[0..15] = 0;
    reg0 = 100.64.0.3;
    reg1 = 100.64.0.1;
    eth.src = 0a:58:64:40:00:01;
    outport = "rtoj-ovn_cluster_router";
    flags.loopback = 1;
    next;
11. lr_in_ip_routing_ecmp (ovn-northd.c:10003): reg8[0..15] == 0, priority 150, uuid cb1f5764
    next;
12. lr_in_policy (ovn-northd.c:10128): 1, priority 0, uuid b088970d
    reg8[0..15] = 0;
    next;
13. lr_in_policy_ecmp (ovn-northd.c:10130): reg8[0..15] == 0, priority 150, uuid 9a008e04
    next;
14. lr_in_arp_resolve (ovn-northd.c:10507): outport == "rtoj-ovn_cluster_router" && reg0 == 100.64.0.3, priority 100, uuid 34c47a1d
    eth.dst = 0a:58:64:40:00:03;
    next;
18. lr_in_arp_request (ovn-northd.c:10753): 1, priority 0, uuid 97b1574b
    output;

egress(dp="ovn_cluster_router", inport="rtos-anurag47a-bwr7f-master-0", outport="rtoj-ovn_cluster_router")
----------------------------------------------------------------------------------------------------------
 3. lr_out_delivery (ovn-northd.c:10801): outport == "rtoj-ovn_cluster_router", priority 100, uuid 5b5c01e0
    output;
    /* output to "rtoj-ovn_cluster_router", type "patch" */

ingress(dp="join", inport="jtor-ovn_cluster_router")
----------------------------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:5036): inport == "jtor-ovn_cluster_router", priority 50, uuid ead93428
    next;
 6. ls_in_pre_lb (ovn-northd.c:5179): ip && inport == "jtor-ovn_cluster_router", priority 110, uuid 6e2c69c1
    next;
22. ls_in_l2_lkup (ovn-northd.c:7674): eth.dst == 0a:58:64:40:00:03, priority 50, uuid 3f6ea834
    outport = "jtor-GR_anurag47a-bwr7f-master-0";
    output;

egress(dp="join", inport="jtor-ovn_cluster_router", outport="jtor-GR_anurag47a-bwr7f-master-0")
-----------------------------------------------------------------------------------------------
 0. ls_out_pre_lb (ovn-northd.c:5182): ip && outport == "jtor-GR_anurag47a-bwr7f-master-0", priority 110, uuid 611d3186
    next;
 9. ls_out_port_sec_l2 (ovn-northd.c:5131): outport == "jtor-GR_anurag47a-bwr7f-master-0", priority 50, uuid 9f9ff2b8
    output;
    /* output to "jtor-GR_anurag47a-bwr7f-master-0", type "l3gateway" */

ingress(dp="GR_anurag47a-bwr7f-master-0", inport="rtoj-GR_anurag47a-bwr7f-master-0")
------------------------------------------------------------------------------------
 0. lr_in_admission (ovn-northd.c:9657): eth.dst == 0a:58:64:40:00:03 && inport == "rtoj-GR_anurag47a-bwr7f-master-0", priority 50, uuid 0770c63e
    xreg0[0..47] = 0a:58:64:40:00:03;
    next;
 1. lr_in_lookup_neighbor (ovn-northd.c:9736): 1, priority 0, uuid c8d6c6d3
    reg9[2] = 1;
    next;
 2. lr_in_learn_neighbor (ovn-northd.c:9745): reg9[2] == 1 || reg9[3] == 0, priority 100, uuid 46eb2b98
    next;
 6. lr_in_dnat (ovn-northd.c:11967): ip, priority 50, uuid 8599befb
    flags.loopback = 1;
    ct_dnat;

ct_dnat /* assuming no un-dnat entry, so no change */
-----------------------------------------------------
10. lr_in_ip_routing (ovn-northd.c:8704): ip4.dst == 192.168.0.0/18, priority 37, uuid 2836eb71
    ip.ttl--;
    reg8[0..15] = 0;
    reg0 = ip4.dst;
    reg1 = 192.168.2.202;
    eth.src = fa:16:3e:ae:47:55;
    outport = "rtoe-GR_anurag47a-bwr7f-master-0";
    flags.loopback = 1;
    next;
11. lr_in_ip_routing_ecmp (ovn-northd.c:10003): reg8[0..15] == 0, priority 150, uuid cb1f5764
    next;
12. lr_in_policy (ovn-northd.c:10128): 1, priority 0, uuid b088970d
    reg8[0..15] = 0;
    next;
13. lr_in_policy_ecmp (ovn-northd.c:10130): reg8[0..15] == 0, priority 150, uuid 9a008e04
    next;
14. lr_in_arp_resolve (ovn-northd.c:10164): ip4, priority 0, uuid 3f9a8ec8
    get_arp(outport, reg0);
    /* MAC binding to fa:16:3e:75:de:a7. */
    next;
18. lr_in_arp_request (ovn-northd.c:10753): 1, priority 0, uuid 97b1574b
    output;

egress(dp="GR_anurag47a-bwr7f-master-0", inport="rtoj-GR_anurag47a-bwr7f-master-0", outport="rtoe-GR_anurag47a-bwr7f-master-0")
-------------------------------------------------------------------------------------------------------------------------------
 1. lr_out_snat (ovn-northd.c:11589): ip && ip4.src == 10.128.0.0/14, priority 15, uuid efefa2ba
    ct_snat(192.168.2.202);

ct_snat(ip4.src=192.168.2.202)
------------------------------
 3. lr_out_delivery (ovn-northd.c:10801): outport == "rtoe-GR_anurag47a-bwr7f-master-0", priority 100, uuid 3858ab01
    output;
    /* output to "rtoe-GR_anurag47a-bwr7f-master-0", type "l3gateway" */

ingress(dp="ext_anurag47a-bwr7f-master-0", inport="etor-GR_anurag47a-bwr7f-master-0")
-------------------------------------------------------------------------------------
 0. ls_in_port_sec_l2 (ovn-northd.c:5036): inport == "etor-GR_anurag47a-bwr7f-master-0", priority 50, uuid 660288f6
    next;
 6. ls_in_pre_lb (ovn-northd.c:5179): ip && inport == "etor-GR_anurag47a-bwr7f-master-0", priority 110, uuid 414ea939
    next;
22. ls_in_l2_lkup (ovn-northd.c:6994): 1, priority 0, uuid 70903136
    outport = get_fdb(eth.dst);
    next;
23. ls_in_l2_unknown (ovn-northd.c:6999): outport == "none", priority 50, uuid e5652419
    outport = "_MC_unknown";
    output;

multicast(dp="ext_anurag47a-bwr7f-master-0", mcgroup="_MC_unknown")
-------------------------------------------------------------------

    egress(dp="ext_anurag47a-bwr7f-master-0", inport="etor-GR_anurag47a-bwr7f-master-0", outport="br-ex_anurag47a-bwr7f-master-0")
    ------------------------------------------------------------------------------------------------------------------------------
         0. ls_out_pre_lb (ovn-northd.c:5182): ip && outport == "br-ex_anurag47a-bwr7f-master-0", priority 110, uuid 6d7bce40
            next;
         9. ls_out_port_sec_l2 (ovn-northd.c:5131): outport == "br-ex_anurag47a-bwr7f-master-0", priority 50, uuid 3b030225
            output;
            /* output to "br-ex_anurag47a-bwr7f-master-0", type "localnet" */