I'm trying to get --topology-network 3_nets_4_leafs_openvswitch to work, but it fails during overcloud introspection. The rest of this process works fine if I remove that option (and the associated -e provison_virsh_network_name). I've tried both -e provison_virsh_network_name="external" and -e provison_virsh_network_name="ctlplane1 ctlplane2" (see notes below the interface details)
$ infrared virsh -vvv -o outputs/provision.yml \
--topology-nodes undercloud:1,controller:3,compute:2,ceph:3 \
--host-address 127.0.0.2 \
--host-key /root/.ssh/id_rsa \
--image-url <...>/rhel-guest-image-8.4-992.x86_64.qcow2 \
--host-memory-overcommit True \
--topology-network 3_nets_4_leafs_openvswitch \
--serial-files True
[...]
$ infrared tripleo-undercloud \
-vv \
-o outputs/undercloud-install.yml \
--mirror "rdu2" \
--version "16.2" \
--splitstack no \
--shade-host undercloud-0 \
--ssl yes \
--build "passed_phase2" \
--images-task rpm \
--images-update no \
--tls-ca <...>/RH-IT-Root-CA.crt \
--overcloud-domain "osp-cloudops-02" \
--config-options DEFAULT.undercloud_timezone=UTC
[...]
PLAY RECAP **********************************************************************************************************************
ceph-0 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
ceph-1 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
ceph-2 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
compute-0 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
compute-1 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
controller-0 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
controller-1 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
controller-2 : ok=0 changed=0 unreachable=0 failed=0 skipped=44 rescued=0 ignored=0
hypervisor : ok=7 changed=3 unreachable=0 failed=0 skipped=69 rescued=0 ignored=0
localhost : ok=14 changed=6 unreachable=0 failed=0 skipped=13 rescued=0 ignored=0
undercloud-0 : ok=144 changed=77 unreachable=0 failed=0 skipped=384 rescued=0 ignored=1
Everything works fine up until this point. All the VM network interfaces look okay to me:
$ for VM in $(virsh list | grep running | awk '{print $2}'); do echo "==$VM=="; virsh dumpxml $VM | grep -B2 portgroup; done
==ceph-0==
<interface type='bridge'>
<mac address='52:54:00:cf:ca:a3'/>
<source network='data' portgroup='ctlplane2' portid='177d8dd8-6cda-4df5-bff5-df74130ba3bd' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:c7:e3:90'/>
<source network='data' portgroup='trunk' portid='682f6bdd-7dac-4e18-8ebe-b4638fa2007d' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:90:ea:93'/>
<source network='data' portgroup='external' portid='fe630258-9ac3-41f6-a6f2-d28c9ff482aa' bridge='data'/>
==ceph-1==
<interface type='bridge'>
<mac address='52:54:00:43:78:68'/>
<source network='data' portgroup='ctlplane2' portid='c11fd55e-ae33-49eb-9909-360752c2dca8' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:02:6b:5f'/>
<source network='data' portgroup='trunk' portid='38ebbc04-9c99-44c0-888f-a2640182a3dc' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:d5:ab:3f'/>
<source network='data' portgroup='external' portid='fcaae195-0ce9-4262-a58b-5d313e24bed4' bridge='data'/>
==ceph-2==
<interface type='bridge'>
<mac address='52:54:00:af:8d:08'/>
<source network='data' portgroup='ctlplane2' portid='5191b975-34cb-452f-943c-0ea72891e98b' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:a2:8a:a1'/>
<source network='data' portgroup='trunk' portid='7572b150-6fcd-4a64-bef9-f1d9a5ce5d70' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:e3:35:15'/>
<source network='data' portgroup='external' portid='6fd82e90-7458-4819-9379-268f7790bad9' bridge='data'/>
==compute-0==
<interface type='bridge'>
<mac address='52:54:00:8e:ba:e1'/>
<source network='data' portgroup='ctlplane2' portid='b68baf7a-fcc7-4f6e-87c8-f2e2b08db605' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:49:53:73'/>
<source network='data' portgroup='trunk' portid='912183fe-e466-4e26-963f-6e30ea15b998' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:e3:79:f4'/>
<source network='data' portgroup='external' portid='ae5389ad-3897-4a52-b4cc-33b9897f392c' bridge='data'/>
==compute-1==
<interface type='bridge'>
<mac address='52:54:00:bb:27:78'/>
<source network='data' portgroup='ctlplane2' portid='bc894912-dd05-493e-b735-c00d3198a0e1' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:88:f1:c1'/>
<source network='data' portgroup='trunk' portid='7ef9187b-3195-4bfd-bdd7-09d4e5265bb8' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:7a:28:58'/>
<source network='data' portgroup='external' portid='b8cc3339-41a3-4721-aa02-12a65392588b' bridge='data'/>
==controller-0==
<interface type='bridge'>
<mac address='52:54:00:b8:57:a8'/>
<source network='data' portgroup='ctlplane1' portid='b3c90664-6159-495b-a17b-e2d38fd38c84' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:e8:73:c1'/>
<source network='data' portgroup='trunk' portid='aadb18dc-e20e-49ad-95d8-975ca5d75a85' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:d1:47:09'/>
<source network='data' portgroup='external' portid='c9e24fab-6c2a-4f93-be47-6dea00703387' bridge='data'/>
==controller-1==
<interface type='bridge'>
<mac address='52:54:00:5e:dd:1c'/>
<source network='data' portgroup='ctlplane1' portid='da337639-8002-44ca-a040-fa49c4d6d604' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:7e:83:91'/>
<source network='data' portgroup='trunk' portid='03ca433a-487a-4772-b74f-2900df88fb80' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:63:df:d7'/>
<source network='data' portgroup='external' portid='be454d3f-9434-43d9-80b4-bae39ff12216' bridge='data'/>
==controller-2==
<interface type='bridge'>
<mac address='52:54:00:83:b3:9d'/>
<source network='data' portgroup='ctlplane1' portid='b9b493c5-4a28-4cee-9796-0ede022d1b63' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:e5:9c:92'/>
<source network='data' portgroup='trunk' portid='ceff69c5-8fcd-4653-a517-81841cc3563e' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:1f:09:e5'/>
<source network='data' portgroup='external' portid='62ce46d6-8f25-457e-94e7-970e8267d377' bridge='data'/>
==undercloud-0==
<interface type='bridge'>
<mac address='52:54:00:c6:d9:ba'/>
<source network='data' portgroup='ctlplane0' portid='cd8d7dc9-39b9-48e1-8b71-b3a5d663f600' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:3e:43:79'/>
<source network='data' portgroup='external' portid='731ab56f-4d2e-422a-8bcd-7b7dabab72ba' bridge='data'/>
I originally deployed it with -e provison_virsh_network_name="ctlplane1 ctlplane2" (since these appear to be the "first" interfaces on every VM) but it failed in introspection and I noticed that the only DHCP traffic I could see on the undercloud was showing as coming from the undercloud's own external interface MAC (not the MAC of the VMs). Since there were no DHCP responses and I didn't see any problems in the DHCP logs (see below), I have retried with -e provison_virsh_network_name="external" even though I'm less certain that it's correct.
$ infrared tripleo-overcloud \
-vv \
-o outputs/overcloud-install.yml \
--version "16.2" \
--deployment-files virt \
--overcloud-debug yes \
--network-backend geneve \
--network-protocol ipv4 \
--network-dvr yes \
--storage-backend ceph \
--storage-external no \
--overcloud-ssl no \
--introspect yes \
--tagging yes \
--deploy yes \
--ntp-server "clock.redhat.com,10.5.27.10,10.11.160.238" \
--overcloud-domain "osp-cloudops-02" \
-e provison_virsh_network_name="external" \
--containers yes
[...]
TASK [print UUID's of nodes that failed introspection] **********************************************************************************************************************************
ESC[1;30mtask path: /root/.infrared/plugins/tripleo-overcloud/tasks/introspection_recovery.yml:65ESC[0m
ESC[0;32mok: [undercloud-0] => {ESC[0m
ESC[0;32m "msg": "Introspection failed on these nodes: ['2d15bc0b-326e-4be3-9ee4-89366dbb316f', 'd1ba89b7-43ae-42fe-a8e3-7cba3dc6e159', '218241ec-9c12-4d96-9916-754f820e9025', '63587462-118f-49cf-b8d8-52485735582c', '706ee761-927d-40ad-9bef-83bc4ad6ee1a', '123c4ed0-28bd-416e-839b-e1e0290a860f', '4d4466c2-2a96-4cb9-8ea7-c48f8b9c8e91', '54c038c0-552d-4d5a-87c7-864166418c9c']"ESC[0m
So it fails on introspection. After this runs, I can see that the correct "external" MACs made it into instackenv.json
[stack@undercloud-0 ~]$ grep -B1 mac instackenv.json
"name": "ceph-0",
"mac": ["52:54:00:90:ea:93"],
--
"name": "ceph-1",
"mac": ["52:54:00:d5:ab:3f"],
--
"name": "ceph-2",
"mac": ["52:54:00:e3:35:15"],
--
"name": "compute-0",
"mac": ["52:54:00:e3:79:f4"],
--
"name": "compute-1",
"mac": ["52:54:00:7a:28:58"],
--
"name": "controller-0",
"mac": ["52:54:00:d1:47:09"],
--
"name": "controller-1",
"mac": ["52:54:00:63:df:d7"],
--
"name": "controller-2",
"mac": ["52:54:00:1f:09:e5"],
I sniffed for DHCP traffic on the undercloud while "TASK [start node introspection]" is running, this is what I see:
[stack@undercloud-0 ~]$ sudo tcpdump -i any port 67 or port 68 or port 69
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on any, link-type LINUX_SLL (Linux cooked v1), capture size 262144 bytes
17:27:09.904427 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:27:12.101143 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:27:16.692569 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:27:22.762333 IP undercloud-0.osp-cloudops-02.50519 > undercloud-0.osp-cloudops-02.tftp: 47 RRQ "map-file" octet tsize 0 blksize 512 timeout 3
17:27:24.698427 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:27:40.951016 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:27:54.908730 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:27:57.014409 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:28:01.901525 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:28:10.528748 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:28:27.007019 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:28:39.912461 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:28:42.435568 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
17:28:42.778209 IP undercloud-0.osp-cloudops-02.59939 > undercloud-0.osp-cloudops-02.tftp: 47 RRQ "map-file" octet tsize 0 blksize 512 timeout 3
17:28:46.993527 IP 0.0.0.0.bootpc > 255.255.255.255.bootps: BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290
[... continues while introspection retries ...]
I thought perhaps they all have the undercloud MAC address because of DHCP relaying or something, but if I fully decode, they all have "undercloud-0" in the Hostname Option. AFAICT I never see DHCP traffic from my VMs, whichever provison_virsh_network_name setting I use.
17:52:55.786413 IP (tos 0xc0, ttl 64, id 0, offset 0, flags [DF], proto UDP (17), length 318)
0.0.0.0.bootpc > 255.255.255.255.bootps: [udp sum ok] BOOTP/DHCP, Request from 52:54:00:3e:43:79 (oui Unknown), length 290, xid 0x1e5bf5cc, secs 15, Flags [none] (0x0000)
Client-Ethernet-Address 52:54:00:3e:43:79 (oui Unknown)
Vendor-rfc1048 Extensions
Magic Cookie 0x63825363
DHCP-Message Option 53, length 1: Discover
Client-ID Option 61, length 7: ether 52:54:00:3e:43:79
Parameter-Request Option 55, length 17:
Subnet-Mask, Time-Zone, Domain-Name-Server, Hostname
Domain-Name, MTU, BR, Classless-Static-Route
Default-Gateway, Static-Route, YD, YS
NTP, Option 119, Classless-Static-Route-Microsoft, Option 252
RP
MSZ Option 57, length 2: 576
Hostname Option 12, length 12: "undercloud-0"
Other debugging based on https://docs.openstack.org/ironic-inspector/latest/user/troubleshooting.html
(undercloud) [stack@undercloud-0 ~]$ openstack baremetal node list
+--------------------------------------+--------------+---------------+-------------+--------------------+-------------+
| UUID | Name | Instance UUID | Power State | Provisioning State | Maintenance |
+--------------------------------------+--------------+---------------+-------------+--------------------+-------------+
| 2d15bc0b-326e-4be3-9ee4-89366dbb316f | ceph-0 | None | power on | manageable | False |
| d1ba89b7-43ae-42fe-a8e3-7cba3dc6e159 | ceph-1 | None | power on | manageable | False |
| 218241ec-9c12-4d96-9916-754f820e9025 | ceph-2 | None | power on | manageable | False |
| 63587462-118f-49cf-b8d8-52485735582c | compute-0 | None | power on | manageable | False |
| 706ee761-927d-40ad-9bef-83bc4ad6ee1a | compute-1 | None | power on | manageable | False |
| 123c4ed0-28bd-416e-839b-e1e0290a860f | controller-0 | None | power on | manageable | False |
| 4d4466c2-2a96-4cb9-8ea7-c48f8b9c8e91 | controller-1 | None | power on | manageable | False |
| 54c038c0-552d-4d5a-87c7-864166418c9c | controller-2 | None | power on | manageable | False |
+--------------------------------------+--------------+---------------+-------------+--------------------+-------------+
[stack@undercloud-0 ~]$ sudo journalctl -u openstack-ironic-inspector
-- Logs begin at Tue 2022-01-25 15:58:22 UTC, end at Tue 2022-01-25 17:31:18 UTC. --
-- No entries --
[stack@undercloud-0 ~]$ sudo journalctl -u openstack-ironic-inspector-dnsmasq
-- Logs begin at Tue 2022-01-25 15:58:22 UTC, end at Tue 2022-01-25 17:32:12 UTC. --
-- No entries --
[root@undercloud-0 ironic-inspector]# pwd
/var/log/containers/ironic-inspector
[root@undercloud-0 ironic-inspector]# cat dnsmasq.log
Jan 25 16:20:52 dnsmasq[6]: started, version 2.79 DNS disabled
Jan 25 16:20:52 dnsmasq[6]: compile time options: IPv6 GNU-getopt DBus no-i18n IDN2 DHCP DHCPv6 no-Lua TFTP no-conntrack ipset auth DNSSEC loop-detect inotify
Jan 25 16:20:52 dnsmasq-dhcp[6]: DHCP, IP range 192.168.24.100 -- 192.168.24.120, lease time 10m
Jan 25 16:20:52 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/unknown_hosts_filter
Jan 25 16:20:52 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/unknown_hosts_filter
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:35:15
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:35:15
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:7a:28:58
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:7a:28:58
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:79:f4
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:79:f4
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:1f:09:e5
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:1f:09:e5
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d5:ab:3f
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d5:ab:3f
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:90:ea:93
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:90:ea:93
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d1:47:09
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d1:47:09
Jan 25 17:25:37 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:63:df:d7
Jan 25 17:25:37 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:63:df:d7
Jan 25 17:26:10 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d5:ab:3f
Jan 25 17:26:10 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d5:ab:3f
Jan 25 17:26:10 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/unknown_hosts_filter
Jan 25 17:26:10 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/unknown_hosts_filter
Jan 25 17:26:10 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d1:47:09
Jan 25 17:26:10 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:d1:47:09
Jan 25 17:26:10 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:90:ea:93
Jan 25 17:26:10 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:90:ea:93
Jan 25 17:26:11 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:79:f4
Jan 25 17:26:11 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:79:f4
Jan 25 17:26:12 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:7a:28:58
Jan 25 17:26:12 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:7a:28:58
Jan 25 17:26:12 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:63:df:d7
Jan 25 17:26:12 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:63:df:d7
Jan 25 17:26:13 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:35:15
Jan 25 17:26:13 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:e3:35:15
Jan 25 17:26:14 dnsmasq[6]: inotify, new or changed file /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:1f:09:e5
Jan 25 17:26:14 dnsmasq-dhcp[6]: read /var/lib/ironic-inspector/dhcp-hostsdir/52:54:00:1f:09:e5
[root@undercloud-0 ironic-inspector]# grep ERROR ironic-inspector.log
2022-01-25 16:20:50.990 7 DEBUG ironic_inspector.common.service_utils [-] logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s log_opt_values /usr/lib/python3.6/site-packages/oslo_config/cfg.py:2581
2022-01-25 16:20:51.035 7 DEBUG oslo_service.service [-] logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s log_opt_values /usr/lib/python3.6/site-packages/oslo_config/cfg.py:2581
2022-01-25 16:20:51.088 7 DEBUG oslo_db.sqlalchemy.engines [-] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION _check_effective_sql_mode /usr/lib/python3.6/site-packages/oslo_db/sqlalchemy/engines.py:307
[root@undercloud-0 ironic-inspector]# grep WARN ironic-inspector.log
2022-01-25 16:20:50.987 7 DEBUG ironic_inspector.common.service_utils [-] default_log_levels = ['sqlalchemy=WARNING', 'iso8601=WARNING', 'requests=WARNING', 'urllib3.connectionpool=WARNING', 'keystonemiddleware=WARNING', 'keystoneauth=WARNING', 'ironicclient=WARNING', 'amqp=WARNING', 'amqplib=WARNING', 'oslo.messaging=WARNING', 'oslo_messaging=WARNING'] log_opt_values /usr/lib/python3.6/site-packages/oslo_config/cfg.py:2581
2022-01-25 16:20:51.023 7 WARNING keystonemiddleware.auth_token [-] AuthToken middleware is set with keystone_authtoken.service_token_roles_required set to False. This is backwards compatible but deprecated behaviour. Please set this to True.
2022-01-25 16:20:51.033 7 DEBUG oslo_service.service [-] default_log_levels = ['sqlalchemy=WARNING', 'iso8601=WARNING', 'requests=WARNING', 'urllib3.connectionpool=WARNING', 'keystonemiddleware=WARNING', 'keystoneauth=WARNING', 'ironicclient=WARNING', 'amqp=WARNING', 'amqplib=WARNING', 'oslo.messaging=WARNING', 'oslo_messaging=WARNING'] log_opt_values /usr/lib/python3.6/site-packages/oslo_config/cfg.py:2581
(undercloud) [stack@undercloud-0 ~]$ openstack baremetal node show controller-0 | grep ipmi_address
| driver_info | {'deploy_kernel': 'file:///var/lib/ironic/httpboot/agent.kernel', 'rescue_kernel': 'file:///var/lib/ironic/httpboot/agent.kernel', 'deploy_ramdisk': 'file:///var/lib/ironic/httpboot/agent.ramdisk', 'rescue_ramdisk': 'file:///var/lib/ironic/httpboot/agent.ramdisk', 'ipmi_address': '172.16.0.7', 'ipmi_username': 'admin', 'ipmi_password': '******', 'ipmi_port': '6235'} |
(undercloud) [stack@undercloud-0 ~]$ ipmitool -I lanplus -H 172.16.0.7 -L ADMINISTRATOR -p 6235 -U admin -P password -R 1 -N 5 power status
Chassis Power is on
Finally, if I wait long enough, the interface information goes missing out of virsh as if something removed them, and only undercloud-0 remains with connected portgroups.
[root@osp-cloudops-02 16.2]# for VM in $(virsh list | grep running | awk '{print $2}'); do echo "==$VM=="; virsh dumpxml $VM | grep -B2 portgroup; done
==undercloud-0==
<interface type='bridge'>
<mac address='52:54:00:c6:d9:ba'/>
<source network='data' portgroup='ctlplane0' portid='cd8d7dc9-39b9-48e1-8b71-b3a5d663f600' bridge='data'/>
--
<interface type='bridge'>
<mac address='52:54:00:3e:43:79'/>
<source network='data' portgroup='external' portid='731ab56f-4d2e-422a-8bcd-7b7dabab72ba' bridge='data'/>
==compute-0==
==controller-0==
==ceph-2==
==controller-1==
==ceph-1==
==controller-2==
==compute-1==
==ceph-0==
[root@osp-cloudops-02 16.2]# virsh net-port-list data
UUID
---------------------------------------
731ab56f-4d2e-422a-8bcd-7b7dabab72ba
cd8d7dc9-39b9-48e1-8b71-b3a5d663f600
[root@osp-cloudops-02 16.2]# virsh net-port-dumpxml data cd8d7dc9-39b9-48e1-8b71-b3a5d663f600
<networkport>
<uuid>cd8d7dc9-39b9-48e1-8b71-b3a5d663f600</uuid>
<owner>
<name>undercloud-0</name>
<uuid>2236b7b4-4165-419f-abf0-33b7d84f2608</uuid>
</owner>
<group>ctlplane0</group>
<mac address='52:54:00:c6:d9:ba'/>
<virtualport type='openvswitch'>
<parameters interfaceid='994631d1-8dc4-45cc-96b7-0980309b993c'/>
</virtualport>
<vlan>
<tag id='4'/>
</vlan>
<plug type='bridge' bridge='data'/>
</networkport>
[root@osp-cloudops-02 16.2]# virsh net-port-dumpxml data 731ab56f-4d2e-422a-8bcd-7b7dabab72ba
<networkport>
<uuid>731ab56f-4d2e-422a-8bcd-7b7dabab72ba</uuid>
<owner>
<name>undercloud-0</name>
<uuid>2236b7b4-4165-419f-abf0-33b7d84f2608</uuid>
</owner>
<group>external</group>
<mac address='52:54:00:3e:43:79'/>
<virtualport type='openvswitch'>
<parameters interfaceid='50e2e68b-0400-4387-952b-1c0f3d3f7433'/>
</virtualport>
<vlan>
<tag id='10'/>
</vlan>
<plug type='bridge' bridge='data'/>
</networkport>
I'm thinking there is another network config step required that I am missing? I'm not sure exactly what to try next.