gistfile1.txt

/usr/local/bin/octavi ^Ma-worker  --config-file /etc/octavia/octavia.conf & echo $! >/opt/stack/status/s ^Mtack/o-cw.pid; fg || echo "o-cw failed to start" | tee "/opt/stack/status/stack/ ^Mo-cw.failure"
[1] 22987
/usr/local/bin/octavia-worker --config-file /etc/octavia/octavia.conf
2016-07-21 10:20:08.843 22987 INFO octavia.common.config [-] Logging enabled!
2016-07-21 10:20:08.844 22987 WARNING oslo_reports.guru_meditation_report [-] Guru meditation now registers SIGUSR1 and SIGUSR2 by default for backward compatibility. SIGUSR1 will no longer be registered in a future release, so please use SIGUSR2 to generate reports.
2016-07-21 10:20:09.558 22987 INFO root [-] Generating grammar tables from /usr/lib/python2.7/lib2to3/Grammar.txt
2016-07-21 10:20:09.577 22987 INFO root [-] Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
2016-07-21 10:20:09.853 22987 INFO octavia.controller.queue.consumer [-] Starting consumer...
2016-07-21 10:23:14.150 22987 INFO octavia.controller.queue.endpoint [-] Creating load balancer '3e0105ef-b4da-4f89-bd37-3fd6ee71a048'...
2016-07-21 10:23:14.403 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension security-group found enabled
2016-07-21 10:23:14.433 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension dns-integration is not enabled
2016-07-21 10:23:14.447 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension allowed-address-pairs found enabled
2016-07-21 10:23:14.463 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension security-group found enabled
2016-07-21 10:23:14.479 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension dns-integration is not enabled
2016-07-21 10:23:14.492 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension allowed-address-pairs found enabled
2016-07-21 10:23:14.511 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension security-group found enabled
2016-07-21 10:23:14.527 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension dns-integration is not enabled
2016-07-21 10:23:14.539 22987 INFO octavia.network.drivers.neutron.base [-] Neutron extension allowed-address-pairs found enabled
2016-07-21 10:23:14.593 22987 INFO octavia.controller.worker.tasks.database_tasks [-] Created Amphora in DB with id 764bb6ad-1b23-4c47-a51a-fe130ce3757e
2016-07-21 10:23:14.907 22987 INFO octavia.certificates.generator.local [-] Signing a certificate request using OpenSSL locally.
2016-07-21 10:23:14.907 22987 INFO octavia.certificates.generator.local [-] Using CA Certificate from config.
2016-07-21 10:23:14.908 22987 INFO octavia.certificates.generator.local [-] Using CA Private Key from config.
2016-07-21 10:23:14.908 22987 INFO octavia.certificates.generator.local [-] Using CA Private Key Passphrase from config.
2016-07-21 10:23:53.035 22987 INFO octavia.controller.worker.tasks.database_tasks [-] Mark ALLOCATED in DB for amphora: 764bb6ad-1b23-4c47-a51a-fe130ce3757e with compute id e3d12041-851b-4336-a37f-0d82d726c649 for load balancer: 3e0105ef-b4da-4f89-bd37-3fd6ee71a048
2016-07-21 10:23:53.111 22987 INFO octavia.network.drivers.neutron.allowed_address_pairs [-] Port 5fa916d3-1cf0-4430-a252-92186f0c9d3b already exists. Nothing to be done.
2016-07-21 10:24:00.897 22987 WARNING octavia.amphorae.drivers.haproxy.rest_api_driver [-] Could not connect to instance. Retrying.
2016-07-21 10:24:03.897 22987 WARNING octavia.amphorae.drivers.haproxy.rest_api_driver [-] Could not connect to instance. Retrying.
2016-07-21 10:33:15.825 22987 WARNING octavia.amphorae.drivers.haproxy.rest_api_driver [-] Could not connect to instance. Retrying.
2016-07-21 10:33:26.829 22987 WARNING octavia.amphorae.drivers.haproxy.rest_api_driver [-] Could not connect to instance. Retrying.
2016-07-21 10:33:37.837 22987 WARNING octavia.amphorae.drivers.haproxy.rest_api_driver [-] Could not connect to instance. Retrying.
2016-07-21 10:33:48.847 22987 WARNING octavia.amphorae.drivers.haproxy.rest_api_driver [-] Could not connect to instance. Retrying.
2016-07-21 10:33:58.293 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.amphora_driver_tasks.AmphoraPostVIPPlug' (6f15db4a-a1fd-4abd-864d-f045cb647337) transitioned into state 'FAILURE' from state 'RUNNING'
9 predecessors (most recent first):
 
9 predecessors (most recent first):
  Atom 'octavia.controller.worker.tasks.network_tasks.GetAmphoraeNetworkConfigs' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'loadbalancer': <octavia.common.data_models.LoadBalancer object at 0x7f4eb2337c50>}, 'provides': {u'764bb6ad-1b23-4c47-a51a-fe130ce3757e': <octavia.network.data_models.AmphoraNetworkConfig object at 0x7f4eb23375d0>}}
  |__Atom 'reload-lb-after-plug-vip' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'loadbalancer_id': u'3e0105ef-b4da-4f89-bd37-3fd6ee71a048'}, 'provides': <octavia.common.data_models.LoadBalancer object at 0x7f4eb2337c50>}
     |__Atom 'octavia.controller.worker.tasks.database_tasks.UpdateAmphoraVIPData' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'amps_data': [<octavia.common.data_models.Amphora object at 0x7f4eb2379dd0>]}, 'provides': None}
        |__Atom 'octavia.controller.worker.tasks.network_tasks.PlugVIP' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'loadbalancer': <octavia.common.data_models.LoadBalancer object at 0x7f4eb2601bd0>}, 'provides': [<octavia.common.data_models.Amphora object at 0x7f4eb2379dd0>]}
           |__Atom 'octavia.controller.worker.tasks.database_tasks.UpdateVIPAfterAllocation' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'vip': <octavia.common.data_models.Vip object at 0x7f4eb2337750>, 'loadbalancer_id': u'3e0105ef-b4da-4f89-bd37-3fd6ee71a048'}, 'provides': <octavia.common.data_models.LoadBalancer object at 0x7f4eb2601bd0>}
              |__Atom 'octavia.controller.worker.tasks.network_tasks.AllocateVIP' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'loadbalancer': <octavia.common.data_models.LoadBalancer object at 0x7f4eb2cb2bd0>}, 'provides': <octavia.common.data_models.Vip object at 0x7f4eb2337750>}
                 |__Flow 'octavia-new-loadbalancer-net-subflow'
                    |__Atom 'post-amphora-association-octavia-post-loadbalancer-amp_association-subflow-reload-lb-after-amp-assoc' {'intention': 'EXECUTE', 'state': 'SUCCESS', 'requires': {'loadbalancer_id': u'3e0105ef-b4da-4f89-bd37-3fd6ee71a048'}, 'provides': <octavia.common.data_models.LoadBalancer object at 0x7f4eb2cb2bd0>}
                       |__Flow 'post-amphora-association-octavia-post-loadbalancer-amp_association-subflow'
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker Traceback (most recent call last):
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker   File "/usr/local/lib/python2.7/dist-packages/taskflow/engines/action_engine/executor.py", line 56, in _execute_task
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker     result = task.execute(**arguments)
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker   File "/opt/stack/octavia/octavia/controller/worker/tasks/amphora_driver_tasks.py", line 229, in execute
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker     loadbalancer, amphorae_network_config)
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker   File "/opt/stack/octavia/octavia/amphorae/drivers/haproxy/rest_api_driver.py", line 132, in post_vip_plug
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker     net_info)
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker   File "/opt/stack/octavia/octavia/amphorae/drivers/haproxy/rest_api_driver.py", line 327, in plug_vip
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker     return exc.check_exception(r)
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker   File "/opt/stack/octavia/octavia/amphorae/drivers/haproxy/exceptions.py", line 32, in check_exception
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker     raise responses[status_code]()
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker NotFound: Not Found
2016-07-21 10:33:58.293 22987 ERROR octavia.controller.worker.controller_worker
2016-07-21 10:33:58.298 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.amphora_driver_tasks.AmphoraPostVIPPlug' (6f15db4a-a1fd-4abd-864d-f045cb647337) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.300 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.network_tasks.GetAmphoraeNetworkConfigs' (4753b864-07c4-4aa3-b38a-8acfe45b0a64) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.304 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'reload-lb-after-plug-vip' (979fe690-c5be-4db9-924c-3a098edafa3d) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.306 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.database_tasks.UpdateAmphoraVIPData' (cab95830-f4f3-47b3-a9a8-5e81d1e8e6f7) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.309 22987 WARNING octavia.controller.worker.tasks.network_tasks [-] Unable to plug VIP for loadbalancer id 3e0105ef-b4da-4f89-bd37-3fd6ee71a048
2016-07-21 10:33:58.880 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.network_tasks.PlugVIP' (c4263cfa-b422-4145-9316-46396b459622) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.306 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.database_tasks.UpdateAmphoraVIPData' (cab95830-f4f3-47b3-a9a8-5e81d1e8e6f7) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.309 22987 WARNING octavia.controller.worker.tasks.network_tasks [-] Unable to plug VIP for loadbalancer id 3e0105ef-b4da-4f89-bd37-3fd6ee71a048
2016-07-21 10:33:58.880 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.network_tasks.PlugVIP' (c4263cfa-b422-4145-9316-46396b459622) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.882 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.database_tasks.UpdateVIPAfterAllocation' (c5d649cd-86c5-4d16-8562-cb0b406c52f5) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:33:58.883 22987 WARNING octavia.controller.worker.tasks.network_tasks [-] Deallocating vip 172.24.4.2
2016-07-21 10:33:58.941 22987 INFO octavia.network.drivers.neutron.allowed_address_pairs [-] Port 5fa916d3-1cf0-4430-a252-92186f0c9d3b will not be deleted by Octavia as it was not created by Octavia.
2016-07-21 10:33:58.983 22987 INFO octavia.network.drivers.neutron.allowed_address_pairs [-] Removing security group 3cc56699-324d-453a-b11a-2be87e79fe46 from port 5fa916d3-1cf0-4430-a252-92186f0c9d3b
2016-07-21 10:33:59.275 22987 WARNING octavia.network.drivers.neutron.allowed_address_pairs [-] Attempt 1 to remove security group 3cc56699-324d-453a-b11a-2be87e79fe46 failed.
2016-07-21 10:34:00.341 22987 INFO octavia.network.drivers.neutron.allowed_address_pairs [-] Deleted security group 3cc56699-324d-453a-b11a-2be87e79fe46
2016-07-21 10:34:00.342 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'octavia.controller.worker.tasks.network_tasks.AllocateVIP' (63b38844-15c4-4583-81bc-5d81155026f0) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:34:00.344 22987 WARNING octavia.controller.worker.controller_worker [-] Task 'post-amphora-association-octavia-post-loadbalancer-amp_association-subflow-reload-lb-after-amp-assoc' (54bd1d3b-ace4-4bc9-b5f7-b15a0a40be32) transitioned into state 'REVERTED' from state 'REVERTING'
2016-07-21 10:34:00.346 22987 WARNING octavia.controller.worker.controller_worker [-] Flow 'post-amphora-association-octavia-post-loadbalancer-amp_association-subflow' (75285439-a550-40d4-9f0e-b3195bf2f8a3) transitioned into state 'REVERTED' from state 'RUNNING'
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server [-] Exception during message handling
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server Traceback (most recent call last):
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/oslo_messaging/rpc/server.py", line 133, in _process_incoming
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     res = self.dispatcher.dispatch(message)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 150, in dispatch
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     return self._do_dispatch(endpoint, method, ctxt, args)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/oslo_messaging/rpc/dispatcher.py", line 121, in _do_dispatch
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     result = func(ctxt, **new_args)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/opt/stack/octavia/octavia/controller/queue/endpoint.py", line 45, in create_load_balancer
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     self.worker.create_load_balancer(load_balancer_id)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/opt/stack/octavia/octavia/controller/worker/controller_worker.py", line 317, in create_load_balancer
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     post_lb_amp_assoc.run()
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/taskflow/engines/action_engine/engine.py", line 247, in run
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/taskflow/engines/action_engine/engine.py", line 247, in run
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     for _state in self.run_iter(timeout=timeout):
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/taskflow/engines/action_engine/engine.py", line 340, in run_iter
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     failure.Failure.reraise_if_any(fails)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/taskflow/types/failure.py", line 336, in reraise_if_any
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     failures[0].reraise()
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/taskflow/types/failure.py", line 343, in reraise
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     six.reraise(*self._exc_info)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/usr/local/lib/python2.7/dist-packages/taskflow/engines/action_engine/executor.py", line 56, in _execute_task
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     result = task.execute(**arguments)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/opt/stack/octavia/octavia/controller/worker/tasks/amphora_driver_tasks.py", line 229, in execute
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     loadbalancer, amphorae_network_config)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/opt/stack/octavia/octavia/amphorae/drivers/haproxy/rest_api_driver.py", line 132, in post_vip_plug
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     net_info)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/opt/stack/octavia/octavia/amphorae/drivers/haproxy/rest_api_driver.py", line 327, in plug_vip
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     return exc.check_exception(r)
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server   File "/opt/stack/octavia/octavia/amphorae/drivers/haproxy/exceptions.py", line 32, in check_exception
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server     raise responses[status_code]()
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server NotFound: Not Found
2016-07-21 10:34:00.347 22987 ERROR oslo_messaging.rpc.server