VM is not able to get IP address from tenant network

logs during vm creation and status afterwards

pike installation using openstack ansible .   In this set up  neutron and kvm are  in  managment LAN POD2 with ip as 10.0.1.x . They also have 2nd NIC for vxlan and VALN 
which is connected to external router. whereas rest of the controller and HAproxy  etc. are in managnment LAN POD1.  

I have the following network : 
root@ansiblenew:/opt/openstack-ansible/playbooks# openstack network list
+--------------------------------------+---------------+--------------------------------------+
| ID                                   | Name          | Subnets                              |
+--------------------------------------+---------------+--------------------------------------+
| 6abf6249-2c8f-467f-b56d-d426dcdae6f1 | Net-30        | 26ad78d6-25e2-4f4b-bd8d-3469fae3e989 |
| fb143a1f-d84b-4650-8e52-b369810f4190 | Provider-Flat | 54e0f637-1235-4a35-aaed-7e49f8fb68f5 |
+--------------------------------------+---------------+--------------------------------------+

Whe a VM is created in provider-Flat it is reachable from external network however when the VM is in Net-30 it is not even getting the IP.
Here are some of the important logs . 


openstack server create --nic net-id=6abf6249-2c8f-467f-b56d-d426dcdae6f1 --image cirrosraw --flavor small --key-name k2 --security-group ssh-ping cirrosk1




Nov 16 17:04:45 neutron1-neutron-agents-container-f6882a35 neutron-dhcp-agent: 2017-11-16 17:04:40.658 4886 INFO neutron.agent.dhcp.agent [req-74fd514c-c7e8-4c0a-b4c2-29961bcfa1e9 30313bbe61fe452680c6c2ee9d8475fc 8e13cb3b05ec4c7d87bac8ffe9f2bd19 - - -] Trigger reload_allocations for port admin_state_up=True, allowed_address_pairs=[], binding:host_id=, binding:profile=, binding:vif_details=, binding:vif_type=unbound, binding:vnic_type=normal, created_at=2017-11-16T11:34:39Z, description=, device_id=e7cc4603-e3eb-45b4-992c-f1de2f9bbb57, device_owner=, dns_assignment=[{u'hostname': u'host-30-1-1-13', u'ip_address': u'30.1.1.13', u'fqdn': u'host-30-1-1-13.itc.in.'}], dns_name=, extra_dhcp_opts=[], fixed_ips=[{u'subnet_id': u'26ad78d6-25e2-4f4b-bd8d-3469fae3e989', u'ip_address': u'30.1.1.13'}], id=916f615d-bb3c-4dbf-8a0a-ae4feb9522fb, mac_address=fa:16:3e:97:a2:37, name=, network_id=6abf6249-2c8f-467f-b56d-d426dcdae6f1, port_security_enabled=True, project_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, qos_policy_id=None, revision_number=3, security_groups=[u'479b2ddc-3de3-43e6-b501-f05d009894d0'], status=DOWN, tags=[], tenant_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, updated_at=2017-11-16T11:34:39Z
Nov 16 17:04:45 neutron1-neutron-agents-container-f6882a35 neutron-dhcp-agent: 2017-11-16 17:04:42.171 4886 INFO neutron.agent.dhcp.agent [req-60056d84-834e-48bb-8f57-071e09392f3b 49659e0aa5f94358a843b8968ecd87a3 7bb422c109ed49108919341d4bbb7e33 - - -] Trigger reload_allocations for port admin_state_up=True, allowed_address_pairs=[], binding:host_id=kvm1, binding:profile=, binding:vif_details=, binding:vif_type=unbound, binding:vnic_type=normal, created_at=2017-11-16T11:34:39Z, description=, device_id=e7cc4603-e3eb-45b4-992c-f1de2f9bbb57, device_owner=compute:nova, dns_assignment=[{u'hostname': u'cirrosk1', u'ip_address': u'30.1.1.13', u'fqdn': u'cirrosk1.itc.in.'}], dns_name=cirrosk1, extra_dhcp_opts=[], fixed_ips=[{u'subnet_id': u'26ad78d6-25e2-4f4b-bd8d-3469fae3e989', u'ip_address': u'30.1.1.13'}], id=916f615d-bb3c-4dbf-8a0a-ae4feb9522fb, mac_address=fa:16:3e:97:a2:37, name=, network_id=6abf6249-2c8f-467f-b56d-d426dcdae6f1, port_security_enabled=True, project_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, qos_policy_id=None, revision_number=4, security_groups=[u'479b2ddc-3de3-43e6-b501-f05d009894d0'], status=DOWN, tags=[], tenant_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, updated_at=2017-11-16T11:34:41Z
Nov 16 17:04:45 neutron1-neutron-agents-container-f6882a35 neutron-dhcp-agent: 2017-11-16 17:04:43.304 4886 INFO neutron.agent.dhcp.agent [req-60056d84-834e-48bb-8f57-071e09392f3b 49659e0aa5f94358a843b8968ecd87a3 7bb422c109ed49108919341d4bbb7e33 - - -] Trigger reload_allocations for port admin_state_up=True, allowed_address_pairs=[], binding:host_id=kvm1, binding:profile=, binding:vif_details=port_filter=True, binding:vif_type=bridge, binding:vnic_type=normal, created_at=2017-11-16T11:34:39Z, description=, device_id=e7cc4603-e3eb-45b4-992c-f1de2f9bbb57, device_owner=compute:nova, dns_assignment=[{u'hostname': u'cirrosk1', u'ip_address': u'30.1.1.13', u'fqdn': u'cirrosk1.itc.in.'}], dns_name=cirrosk1, extra_dhcp_opts=[], fixed_ips=[{u'subnet_id': u'26ad78d6-25e2-4f4b-bd8d-3469fae3e989', u'ip_address': u'30.1.1.13'}], id=916f615d-bb3c-4dbf-8a0a-ae4feb9522fb, mac_address=fa:16:3e:97:a2:37, name=, network_id=6abf6249-2c8f-467f-b56d-d426dcdae6f1, port_security_enabled=True, project_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, qos_policy_id=None, revision_number=6, security_groups=[u'479b2ddc-3de3-43e6-b501-f05d009894d0'], status=DOWN, tags=[], tenant_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, updated_at=2017-11-16T11:34:42Z
 


root@neutron1-neutron-agents-container-f6882a35:~# cat /var/lib/neutron/dhcp/6abf6249-2c8f-467f-b56d-d426dcdae6f1/host
fa:16:3e:67:34:53,host-30-1-1-10.itc.in.,30.1.1.10
fa:16:3e:b2:00:0d,host-30-1-1-1.itc.in.,30.1.1.1
fa:16:3e:98:f2:de,ten1.itc.in.,30.1.1.15
fa:16:3e:09:e2:5f,vm1-kvm3.itc.in.,30.1.1.20
fa:16:3e:97:a2:37,cirrosk1.itc.in.,30.1.1.13




root@kvm1:~# virsh list
 Id    Name                           State
----------------------------------------------------
 5     instance-00000127              running
 6     instance-0000012d              running

root@kvm1:~# virsh console 6
Connected to domain instance-0000012d
Escape character is ^]

login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root.
cirrosk1 login: cirros
Password:
$
$
$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether fa:16:3e:97:a2:37 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::f816:3eff:fe97:a237/64 scope link
       valid_lft forever preferred_lft forever



openstack server create --nic net-id=fb143a1f-d84b-4650-8e52-b369810f4190  --image cirrosraw --flavor small --key-name k2 --security-group ssh-ping cirrosk1-provider

*** /var/log/log-storage/neutron1-neutron-agents-container-f6882a35/neutron-dhcp-agent.log ***
Nov 16 17:15:25 neutron1-neutron-agents-container-f6882a35 neutron-dhcp-agent: 2017-11-16 17:15:20.103 4886 INFO neutron.agent.dhcp.agent [req-927c6e50-ff1a-4dcb-bbe8-9530034df4ce 30313bbe61fe452680c6c2ee9d8475fc 8e13cb3b05ec4c7d87bac8ffe9f2bd19 - - -] Trigger reload_allocations for port admin_state_up=True, allowed_address_pairs=[], binding:host_id=, binding:profile=, binding:vif_details=, binding:vif_type=unbound, binding:vnic_type=normal, created_at=2017-11-16T11:45:18Z, description=, device_id=34884ab3-3473-4db1-9221-47929bde57ca, device_owner=, dns_assignment=[{u'hostname': u'host-2-2-2-14', u'ip_address': u'2.2.2.14', u'fqdn': u'host-2-2-2-14.itc.in.'}], dns_name=, extra_dhcp_opts=[], fixed_ips=[{u'subnet_id': u'54e0f637-1235-4a35-aaed-7e49f8fb68f5', u'ip_address': u'2.2.2.14'}], id=50872ef4-9fcf-4700-9e69-7cf8228759bd, mac_address=fa:16:3e:e9:9f:38, name=, network_id=fb143a1f-d84b-4650-8e52-b369810f4190, port_security_enabled=True, project_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, qos_policy_id=None, revision_number=3, security_groups=[u'479b2ddc-3de3-43e6-b501-f05d009894d0'], status=DOWN, tags=[], tenant_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, updated_at=2017-11-16T11:45:19Z
Nov 16 17:15:25 neutron1-neutron-agents-container-f6882a35 neutron-dhcp-agent: 2017-11-16 17:15:21.606 4886 INFO neutron.agent.dhcp.agent [req-62acae7d-7671-4dab-9d5a-1d9a90431fa6 49659e0aa5f94358a843b8968ecd87a3 7bb422c109ed49108919341d4bbb7e33 - - -] Trigger reload_allocations for port admin_state_up=True, allowed_address_pairs=[], binding:host_id=kvm1, binding:profile=, binding:vif_details=, binding:vif_type=unbound, binding:vnic_type=normal, created_at=2017-11-16T11:45:18Z, description=, device_id=34884ab3-3473-4db1-9221-47929bde57ca, device_owner=compute:nova, dns_assignment=[{u'hostname': u'cirrosk1-provider', u'ip_address': u'2.2.2.14', u'fqdn': u'cirrosk1-provider.itc.in.'}], dns_name=cirrosk1-provider, extra_dhcp_opts=[], fixed_ips=[{u'subnet_id': u'54e0f637-1235-4a35-aaed-7e49f8fb68f5', u'ip_address': u'2.2.2.14'}], id=50872ef4-9fcf-4700-9e69-7cf8228759bd, mac_address=fa:16:3e:e9:9f:38, name=, network_id=fb143a1f-d84b-4650-8e52-b369810f4190, port_security_enabled=True, project_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, qos_policy_id=None, revision_number=4, security_groups=[u'479b2ddc-3de3-43e6-b501-f05d009894d0'], status=DOWN, tags=[], tenant_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, updated_at=2017-11-16T11:45:20Z
Nov 16 17:15:25 neutron1-neutron-agents-container-f6882a35 neutron-dhcp-agent: 2017-11-16 17:15:22.803 4886 INFO neutron.agent.dhcp.agent [req-62acae7d-7671-4dab-9d5a-1d9a90431fa6 49659e0aa5f94358a843b8968ecd87a3 7bb422c109ed49108919341d4bbb7e33 - - -] Trigger reload_allocations for port admin_state_up=True, allowed_address_pairs=[], binding:host_id=kvm1, binding:profile=, binding:vif_details=port_filter=True, binding:vif_type=bridge, binding:vnic_type=normal, created_at=2017-11-16T11:45:18Z, description=, device_id=34884ab3-3473-4db1-9221-47929bde57ca, device_owner=compute:nova, dns_assignment=[{u'hostname': u'cirrosk1-provider', u'ip_address': u'2.2.2.14', u'fqdn': u'cirrosk1-provider.itc.in.'}], dns_name=cirrosk1-provider, extra_dhcp_opts=[], fixed_ips=[{u'subnet_id': u'54e0f637-1235-4a35-aaed-7e49f8fb68f5', u'ip_address': u'2.2.2.14'}], id=50872ef4-9fcf-4700-9e69-7cf8228759bd, mac_address=fa:16:3e:e9:9f:38, name=, network_id=fb143a1f-d84b-4650-8e52-b369810f4190, port_security_enabled=True, project_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, qos_policy_id=None, revision_number=6, security_groups=[u'479b2ddc-3de3-43e6-b501-f05d009894d0'], status=DOWN, tags=[], tenant_id=8e13cb3b05ec4c7d87bac8ffe9f2bd19, updated_at=2017-11-16T11:45:22Z


*** /var/log/log-storage/kvm1/neutron-linuxbridge-agent.log ***
Nov 16 17:16:23 kvm1 neutron-linuxbridge-agent: 2017-11-16 17:16:17.171 12764 INFO neutron.agent.securitygroups_rpc [req-1d5dbebc-4c47-4637-ae4d-13841b8b0810 - - - - -] Preparing filters for devices set(['tap50872ef4-9f'])
Nov 16 17:16:23 kvm1 neutron-linuxbridge-agent: 2017-11-16 17:16:20.306 12764 INFO neutron.plugins.ml2.drivers.agent._common_agent [req-1d5dbebc-4c47-4637-ae4d-13841b8b0810 - - - - -] Port tap50872ef4-9f updated. Details: {u'profile': {}, u'network_qos_policy_id': None, u'qos_policy_id': None, u'allowed_address_pairs': [], u'admin_state_up': True, u'network_id': u'fb143a1f-d84b-4650-8e52-b369810f4190', u'segmentation_id': None, u'mtu': 1500, u'device_owner': u'compute:nova', u'physical_network': u'flat', u'mac_address': u'fa:16:3e:e9:9f:38', u'device': u'tap50872ef4-9f', u'port_security_enabled': True, u'port_id': u'50872ef4-9fcf-4700-9e69-7cf8228759bd', u'fixed_ips': [{u'subnet_id': u'54e0f637-1235-4a35-aaed-7e49f8fb68f5', u'ip_address': u'2.2.2.14'}], u'network_type': u'flat'}
Nov 16 17:16:23 kvm1 neutron-linuxbridge-agent: 2017-11-16 17:16:20.352 12764 INFO neutron.agent.securitygroups_rpc [req-1d5dbebc-4c47-4637-ae4d-13841b8b0810 - - - - -] Security group member updated [u'479b2ddc-3de3-43e6-b501-f05d009894d0']


*** /var/log/log-storage/kvm1/nova-compute.log ***
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:24.431 1761 WARNING nova.virt.libvirt.driver [req-f381f59c-510d-4064-a1e1-d534febc1a2a 30313bbe61fe452680c6c2ee9d8475fc 8e13cb3b05ec4c7d87bac8ffe9f2bd19 - default default] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] Timeout waiting for vif plugging callback for instance with vm_state building and task_state spawning.: Timeout: 10 seconds
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:24.437 1761 INFO nova.compute.manager [req-1a2168cf-e17f-49b0-a476-1eb64f21bda4 - - - - -] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] VM Resumed (Lifecycle Event)
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:24.451 1761 INFO nova.virt.libvirt.driver [-] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] Instance spawned successfully.
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:24.452 1761 INFO nova.compute.manager [req-f381f59c-510d-4064-a1e1-d534febc1a2a 30313bbe61fe452680c6c2ee9d8475fc 8e13cb3b05ec4c7d87bac8ffe9f2bd19 - default default] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] Took 24.62 seconds to spawn the instance on the hypervisor.
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:24.879 1761 INFO nova.compute.manager [req-1a2168cf-e17f-49b0-a476-1eb64f21bda4 - - - - -] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] During sync_power_state the instance has a pending task (spawning). Skip.
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:24.881 1761 INFO nova.compute.manager [req-1a2168cf-e17f-49b0-a476-1eb64f21bda4 - - - - -] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] VM Resumed (Lifecycle Event)
Nov 16 17:16:33 kvm1 nova-compute: 2017-11-16 17:16:25.110 1761 INFO nova.compute.manager [req-f381f59c-510d-4064-a1e1-d534febc1a2a 30313bbe61fe452680c6c2ee9d8475fc 8e13cb3b05ec4c7d87bac8ffe9f2bd19 - default default] [instance: 34884ab3-3473-4db1-9221-47929bde57ca] Took 27.74 seconds to build instance.


root@neutron1-neutron-agents-container-f6882a35:~# cat /var/lib/neutron/dhcp/fb143a1f-d84b-4650-8e52-b369810f4190/host
fa:16:3e:e8:83:d0,host-2-2-2-10.itc.in.,2.2.2.10
fa:16:3e:88:ff:06,host-2-2-2-19.itc.in.,2.2.2.19
fa:16:3e:e9:9f:38,cirrosk1-provider.itc.in.,2.2.2.14
root@neutron1-neutron-agents-container-f6882a35:~# ip netns list
qrouter-69476894-6c9b-4106-8431-e4ef5bfcb14e (id: 3)
qdhcp-fb143a1f-d84b-4650-8e52-b369810f4190 (id: 2)
qdhcp-6abf6249-2c8f-467f-b56d-d426dcdae6f1 (id: 1)
root@neutron1-neutron-agents-container-f6882a35:~# ip netns exec qdhcp-fb143a1f-d84b-4650-8e52-b369810f4190  ping 2.2.2.10
PING 2.2.2.10 (2.2.2.10) 56(84) bytes of data.
64 bytes from 2.2.2.10: icmp_seq=1 ttl=64 time=0.055 ms
64 bytes from 2.2.2.10: icmp_seq=2 ttl=64 time=0.029 ms
^C
--- 2.2.2.10 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 999ms
rtt min/avg/max/mdev = 0.029/0.042/0.055/0.013 ms
root@neutron1-neutron-agents-container-f6882a35:~# ip netns exec qdhcp-fb143a1f-d84b-4650-8e52-b369810f4190  ping 2.2.2.19
PING 2.2.2.19 (2.2.2.19) 56(84) bytes of data.
64 bytes from 2.2.2.19: icmp_seq=1 ttl=64 time=0.144 ms
64 bytes from 2.2.2.19: icmp_seq=2 ttl=64 time=0.055 ms