root@a2s41:/var/log/contrail# contrail-status
== Contrail Control ==
supervisor-control: active
contrail-control active
contrail-control-nodemgr active
contrail-dns active
contrail-named active
== Contrail Analytics ==
supervisor-analytics: active
contrail-alarm-gen initializing (Collector connection down)
contrail-analytics-api initializing (Collector connection down)
contrail-analytics-nodemgr active
contrail-collector inactive
contrail-query-engine initializing (Collector connection down)
contrail-snmp-collector active
contrail-topology active
== Contrail Config ==
supervisor-config: active
contrail-api:0 active
contrail-config-nodemgr active
contrail-device-manager active
contrail-discovery:0 active
contrail-schema active
contrail-svc-monitor active
ifmap active
== Contrail Web UI ==
supervisor-webui: active
contrail-webui active
contrail-webui-middleware active
== Contrail Database ==
supervisor-database: active
contrail-database active
contrail-database-nodemgr failed
kafka active
== Contrail Support Services ==
supervisor-support-service: active
rabbitmq-server active #####Environment
- Test was executed as user with non-admin role.
- HAProxy timeout is set to 60 seconds
- API server consumes ~100% in all scenarios
No of creates per second - 10 (600 concurrent creates in a minute)
#####api-server
127.0.0.1 - - [2015-03-10 00:52:21] "GET /virtual-network/17a16792-d654-47cd-8255-06bef89e3542?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1630 0.009637
10.84.7.36 - - [2015-03-10 00:52:21] "POST /neutron/network HTTP/1.1" 200 589 0.016121
127.0.0.1 - - [2015-03-10 00:52:21] "GET /virtual-machine-interfaces?count=True&parent_id=dd922057d0c74869a95f7429af7755aa&detail=False HTTP/1.1" 200 152 0.024557
10.84.7.36 - - [2015-03-10 00:52:21] "POST /neutron/port HTTP/1.1" 200 120 0.030464
127.0.0.1 - - [2015-03-10 00:52:21] "GET /virtual-network/17a16792-d654-47cd-8255-06bef89e3542?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1630 0.006907
127.0.0.1 - - [2015-03-10 00:52:21] "GET /project/dd922057-d0c7-4869-a95f-7429af7755aa?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 929 0.007722
127.0.0.1 - - [2015-03-10 00:52:21] "POST /virtual-machine-interfaces HTTP/1.1" 200 559 0.099715
127.0.0.1 - - [2015-03-10 00:52:21] "POST /instance-ips HTTP/1.1" 200 357 0.036252
127.0.0.1 - - [2015-03-10 00:52:21] "GET /virtual-machine-interface/3adcced5-54f7-4e7e-96e8-2c3df4a63b0b?fields=logical_router_back_refs%2Cinstance_ip_back_refs%2Cfloating_ip_back_refs HTTP/1.1" 200 1857 0.005765
127.0.0.1 - - [2015-03-10 00:52:21] "GET /virtual-network/17a16792-d654-47cd-8255-06bef89e3542?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1630 0.005678
127.0.0.1 - - [2015-03-10 00:52:21] "GET /instance-ip/b5bd5ac7-c790-43f2-86e4-30cacf07196f?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1405 0.004105
10.84.7.36 - - [2015-03-10 00:52:21] "POST /neutron/port HTTP/1.1" 200 674 0.202694#####Neutron
2015-03-10 00:52:21.645 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] get_network(): {u'status': u'ACTIVE', u'router:external': False, u'subnets': [u'd6fd40dc-98bc-46b0-b6aa-4413d75569c7'], u'name': u'VN0', u'admin_state_up': True, u'tenant_id': u'dd922057d0c74869a95f7429af7755aa', u'shared': False, u'contrail:subnet_ipam': [{u'subnet_cidr': u'10.1.0.0/16', u'ipam_fq_name': [u'default-domain', u'default-project', u'default-network-ipam']}], u'contrail:instance_count': 0, u'id': u'17a16792-d654-47cd-8255-06bef89e3542', u'contrail:fq_name': [u'default-domain', u'test-project', u'VN0']}
2015-03-10 00:52:21.682 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] get_port_count(): {u'count': 0}
2015-03-10 00:52:21.891 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] create_port(): {u'status': u'DOWN', u'name': u'3adcced5-54f7-4e7e-96e8-2c3df4a63b0b', u'admin_state_up': True, u'network_id': u'17a16792-d654-47cd-8255-06bef89e3542', u'tenant_id': u'dd922057d0c74869a95f7429af7755aa', u'device_owner': u'', u'mac_address': u'02:3a:dc:ce:d5:54', u'device_id': u'', u'fixed_ips': [{u'subnet_id': u'd6fd40dc-98bc-46b0-b6aa-4413d75569c7', u'ip_address': u'10.1.0.3'}], u'id': u'3adcced5-54f7-4e7e-96e8-2c3df4a63b0b', u'security_groups': [u'41ea2dba-9f40-4249-b596-f445951a46f1'], u'contrail:fq_name': [u'default-domain', u'test-project', u'3adcced5-54f7-4e7e-96e8-2c3df4a63b0b']}No of creates per second - ~3.5 (200 concurrent creates in a minute)
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-networks?count=True&parent_id=dd922057d0c74869a95f7429af7755aa&detail=False HTTP/1.1" 200 142 0.007704
10.84.7.36 - - [2015-03-10 10:08:44] "POST /neutron/network HTTP/1.1" 200 120 0.011882
127.0.0.1 - - [2015-03-10 10:08:44] "GET /project/dd922057-d0c7-4869-a95f-7429af7755aa?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 929 0.005608
127.0.0.1 - - [2015-03-10 10:08:44] "POST /virtual-networks HTTP/1.1" 200 473 0.051777
10.84.7.36 - - [2015-03-10 10:08:44] "POST /neutron/network HTTP/1.1" 200 414 0.068692
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-network/c7e4d042-b037-424d-b027-55b253d4e6fb?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 982 0.004423
10.84.7.36 - - [2015-03-10 10:08:44] "POST /neutron/network HTTP/1.1" 200 414 0.009890
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-network/c7e4d042-b037-424d-b027-55b253d4e6fb?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 982 0.004793
10.84.7.36 - - [2015-03-10 10:08:44] "POST /neutron/network HTTP/1.1" 200 414 0.008636
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-networks?count=False&parent_id=dd922057-d0c7-4869-a95f-7429af7755aa&detail=True HTTP/1.1" 200 970 0.010081
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-networks?count=False&detail=True HTTP/1.1" 200 3495 0.018526
10.84.7.36 - - [2015-03-10 10:08:44] "POST /neutron/subnet HTTP/1.1" 200 120 0.038926
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-network/c7e4d042-b037-424d-b027-55b253d4e6fb?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 982 0.003996
127.0.0.1 - - [2015-03-10 10:08:44] "POST /fqname-to-id HTTP/1.1" 404 196 0.002417
127.0.0.1 - - [2015-03-10 10:08:44] "PUT /virtual-network/c7e4d042-b037-424d-b027-55b253d4e6fb HTTP/1.1" 200 264 0.007497
127.0.0.1 - - [2015-03-10 10:08:44] "POST /ref-update HTTP/1.1" 200 156 0.050200
127.0.0.1 - - [2015-03-10 10:08:44] "GET /virtual-network/c7e4d042-b037-424d-b027-55b253d4e6fb?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1608 0.003650
10.84.7.36 - - [2015-03-10 10:08:44] "POST /neutron/subnet HTTP/1.1" 200 547 0.0994302015-03-10 10:08:44.81 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] get_network_count(): {u'count': 0}
2015-03-10 10:08:44.155 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] create_network(): {u'status': u'ACTIVE', u'router:external': False, u'subnets': [], u'name': u'VN0', u'admin_state_up': True, u'tenant_id': u'dd922057d0c74869a95f7429af7755aa', u'shared': False, u'contrail:instance_count': 0, u'id': u'c7e4d042-b037-424d-b027-55b253d4e6fb', u'contrail:fq_name': [u'default-domain', u'test-project', u'VN0']}No of creates per second - 6.5 (390 concurrent creates in a minute)
#####Api-Server
127.0.0.1 - - [2015-03-10 13:39:38] "GET /project/dd922057-d0c7-4869-a95f-7429af7755aa?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 929 0.009031
127.0.0.1 - - [2015-03-10 13:39:38] "POST /security-groups HTTP/1.1" 200 471 0.071420
127.0.0.1 - - [2015-03-10 13:39:38] "GET /security-group/9188176d-43d9-4e94-aa8b-4cdeef2a7aa0?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 932 0.004714
127.0.0.1 - - [2015-03-10 13:39:39] "PUT /security-group/9188176d-43d9-4e94-aa8b-4cdeef2a7aa0 HTTP/1.1" 200 262 0.064535
10.84.7.36 - - [2015-03-10 13:39:39] "POST /neutron/security_group HTTP/1.1" 200 329 0.182283
10.84.7.36 - - [2015-03-10 13:39:39] "PUT /access-control-list/da7367bb-6d3e-4dba-a66b-4fcc355bbec2 HTTP/1.1" 200 274 0.008406
10.84.7.36 - - [2015-03-10 13:39:39] "PUT /access-control-list/f248a704-0865-4c73-be33-5b6c9b942400 HTTP/1.1" 200 274 0.009055#####Neutron
2015-03-10 13:39:39.8 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] create_security_group(): {u'name': u'SG0', u'contrail:fq_name': [u'default-domain', u'test-project', u'SG0'], u'tenant_id': u'dd922057d0c74869a95f7429af7755aa', u'security_group_rules': [], u'id': u'9188176d-43d9-4e94-aa8b-4cdeef2a7aa0', u'description': u''}#####Issues observed
API server got reset when we do 500 concurrent creates(However all SGs got added)
WARNING:api-0:Connection dropped: socket connection broken
WARNING:api-0:Transition to CONNECTING
INFO:api-0:Zookeeper connection lost
INFO:api-0:Connecting to 10.84.7.41:2181
10.84.7.36 - - [2015-03-10 11:26:11] "POST /neutron/security_group HTTP/1.1" 200 333 56.107959
INFO:api-0:Sending request(xid=None): Connect(protocol_version=0, last_zxid_seen=4295374650, time_out=20000, session_id=93452113789976580, passwd='T\x97\x98\tu\xaa\x83\xac\xfb\xbc\x9a\x0c\x15\xfd(l', read_only=None)
WARNING:api-0:Session has expired
INFO:api-0:Zookeeper session lost, state: EXPIRED_SESSION
Few of the access-control-lists are not cleaned up on SG deletionNo of creates per second - Bug (- concurrent creates in a minute)
root@a2s36:~# neutron security-group-rule-list
+--------------------------------------+----------------+-----------+----------+------------------+--------------+
| id | security_group | direction | protocol | remote_ip_prefix | remote_group |
+--------------------------------------+----------------+-----------+----------+------------------+--------------+
| 3f18d27b-cfb5-4000-a36a-405faa6094cc | default | ingress | any | | default |
| 5708e834-c452-4134-856b-019653b97b2c | default | egress | any | 0.0.0.0/0 | |
| b83898d4-8b0b-4957-a95c-0c7e64c4d3ea | default | egress | any | ::/0 | |
| 3a0c5549-8f9f-43b8-a07e-49cefcca73a9 | SG1 | egress | any | 0.0.0.0/0 | |
| ee5ed025-eac9-4ebb-8556-ee420e92810d | SG0 | egress | any | 0.0.0.0/0 | |
+--------------------------------------+----------------+-----------+----------+------------------+--------------+
root@a2s36:~# python scale.py --neutron --n_threads 2 --tenant test-project --n_sg_rules 1 --keystone_ip 10.84.7.41 --api_server_ip 10.84.7.41 --sg_id 91d2c8a5-489b-464c-af41-e5f37656bf3e
scale.py:247: SyntaxWarning: name 'alloc_addr_list' is used prior to global declaration
global alloc_addr_list; alloc_addr_list.append(address)
Time at start 15:00:50:
{'direction': 'ingress', 'protocol': 'tcp', 'remote_ip_prefix': '105.132.189.200/29', 'port_range_max': 1000, 'security_group_id': '91d2c8a5-489b-464c-af41-e5f37656bf3e', 'port_range_min': 1000}
{'direction': 'ingress', 'protocol': 'tcp', 'remote_ip_prefix': '199.90.24.144/29', 'port_range_max': 1001, 'security_group_id': '91d2c8a5-489b-464c-af41-e5f37656bf3e', 'port_range_min': 1001}
Time at End 15:00:51:
Time to create objects 0:00:00.369533
root@a2s36:~# neutron security-group-rule-list
+--------------------------------------+----------------+-----------+----------+--------------------+--------------+
| id | security_group | direction | protocol | remote_ip_prefix | remote_group |
+--------------------------------------+----------------+-----------+----------+--------------------+--------------+
| 3f18d27b-cfb5-4000-a36a-405faa6094cc | default | ingress | any | | default |
| 5708e834-c452-4134-856b-019653b97b2c | default | egress | any | 0.0.0.0/0 | |
| b83898d4-8b0b-4957-a95c-0c7e64c4d3ea | default | egress | any | ::/0 | |
| 3a0c5549-8f9f-43b8-a07e-49cefcca73a9 | SG1 | egress | any | 0.0.0.0/0 | |
| a475c3e7-3de7-4f6b-8c12-ef81cd93c9fd | SG1 | ingress | tcp | 105.132.189.200/29 | |
| ee5ed025-eac9-4ebb-8556-ee420e92810d | SG0 | egress | any | 0.0.0.0/0 | |
+--------------------------------------+----------------+-----------+----------+--------------------+--------------+127.0.0.1 - - [2015-03-10 14:52:00] "GET /security-group/238cb67d-fd06-4879-95ac-0cd407d93414?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 2660 0.010938
127.0.0.1 - - [2015-03-10 14:52:00] "GET /security-group/238cb67d-fd06-4879-95ac-0cd407d93414?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 2660 0.010020
127.0.0.1 - - [2015-03-10 14:52:00] "PUT /security-group/238cb67d-fd06-4879-95ac-0cd407d93414 HTTP/1.1" 200 262 0.080749
127.0.0.1 - - [2015-03-10 14:52:00] "PUT /security-group/238cb67d-fd06-4879-95ac-0cd407d93414 HTTP/1.1" 200 262 0.084527
127.0.0.1 - - [2015-03-10 14:52:00] "GET /security-group/238cb67d-fd06-4879-95ac-0cd407d93414?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 3213 0.006844
10.84.7.36 - - [2015-03-10 14:52:00] "POST /neutron/security_group_rule HTTP/1.1" 200 440 0.122942
127.0.0.1 - - [2015-03-10 14:52:00] "GET /security-group/238cb67d-fd06-4879-95ac-0cd407d93414?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 3213 0.005887
10.84.7.36 - - [2015-03-10 14:52:00] "POST /neutron/security_group_rule HTTP/1.1" 200 441 0.130694
10.84.7.36 - - [2015-03-10 14:52:00] "PUT /access-control-list/f16bc4ec-4af3-43f0-af1d-8cb336b8bdb9 HTTP/1.1" 200 274 0.008433
10.84.7.36 - - [2015-03-10 14:52:00] "PUT /access-control-list/195918dc-3664-4f98-b1e0-9667f3476668 HTTP/1.1" 200 274 0.008891
10.84.7.42 - - [2015-03-10 14:52:20] "GET /virtual-routers HTTP/1.1" 200 320 0.003444
10.84.7.42 - - [2015-03-10 14:52:20] "GET /virtual-router/5a7fa3e7-1a90-4bc3-961b-da788cb80a3d HTTP/1.1" 200 1044 0.005754No of creates per second - 16 (960 concurrent creates in a minute)
Api-server was about 90% loaded (Keystone and cassandra were momentarily ~90%)
127.0.0.1 - - [2015-03-10 15:29:47] "GET /logical-routers?count=True&parent_id=dd922057d0c74869a95f7429af7755aa&detail=False HTTP/1.1" 200 141 0.015550
10.84.7.36 - - [2015-03-10 15:29:47] "POST /neutron/router HTTP/1.1" 200 120 0.024192
127.0.0.1 - - [2015-03-10 15:29:47] "GET /project/dd922057-d0c7-4869-a95f-7429af7755aa?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 929 0.008233
127.0.0.1 - - [2015-03-10 15:29:47] "POST /logical-routers HTTP/1.1" 200 473 0.071435
127.0.0.1 - - [2015-03-10 15:29:47] "GET /logical-router/53e6d66b-a19d-46ab-a7da-edc985ef0311?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 936 0.004451
10.84.7.36 - - [2015-03-10 15:29:47] "POST /neutron/router HTTP/1.1" 200 396 0.1018262015-03-10 15:29:47.821 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] get_router_count(): {u'count': 0}
2015-03-10 15:29:47.929 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] create_router(): {u'status': u'ACTIVE', u'external_gateway_info': None, u'name': u'RTR0', u'gw_port_id': None, u'admin_state_up': True, u'tenant_id': u'dd922057d0c74869a95f7429af7755aa', u'shared': False, u'id': u'53e6d66b-a19d-46ab-a7da-edc985ef0311', u'contrail:fq_name': [u'default-domain', u'test-project', u'RTR0']}No of creates per second - 13 (780 concurrent creates in a minute)
127.0.0.1 - - [2015-03-11 16:26:18] "GET /virtual-networks?count=False&parent_id=dd922057-d0c7-4869-a95f-7429af7755aa&detail=True HTTP/1.1" 200 132 0.009982
127.0.0.1 - - [2015-03-11 16:26:18] "GET /virtual-networks?count=False&detail=True HTTP/1.1" 200 4141 0.018325
127.0.0.1 - - [2015-03-11 16:26:18] "GET /virtual-networks?count=False&detail=True HTTP/1.1" 200 4141 0.023540
10.84.7.36 - - [2015-03-11 16:26:18] "POST /neutron/network HTTP/1.1" 200 596 0.073109
127.0.0.1 - - [2015-03-11 16:26:18] "GET /floating-ips?count=True&detail=False&back_ref_id=dd922057d0c74869a95f7429af7755aa HTTP/1.1" 200 138 0.011456
10.84.7.36 - - [2015-03-11 16:26:18] "POST /neutron/floatingip HTTP/1.1" 200 120 0.017461
127.0.0.1 - - [2015-03-11 16:26:18] "GET /floating-ip-pools?count=False&parent_id=36bccd67-c826-4c2d-b498-785d80088eb9&detail=False HTTP/1.1" 200 339 0.005842
127.0.0.1 - - [2015-03-11 16:26:18] "POST /fqname-to-id HTTP/1.1" 200 156 0.003286
127.0.0.1 - - [2015-03-11 16:26:18] "GET /floating-ip-pool/5b899a59-6dcd-4753-ae85-79427f63891a?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 995 0.005070
127.0.0.1 - - [2015-03-11 16:26:18] "GET /project/dd922057-d0c7-4869-a95f-7429af7755aa?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 929 0.007193
127.0.0.1 - - [2015-03-11 16:26:18] "POST /floating-ips HTTP/1.1" 200 563 0.095288
127.0.0.1 - - [2015-03-11 16:26:18] "GET /floating-ip/d9d4f9db-7a29-4205-a976-7759d422f498?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1305 0.007873
127.0.0.1 - - [2015-03-11 16:26:18] "POST /fqname-to-id HTTP/1.1" 200 156 0.003099
10.84.7.36 - - [2015-03-11 16:26:18] "POST /neutron/floatingip HTTP/1.1" 200 383 0.165882
10.84.7.36 - - [2015-03-11 16:26:18] "POST /fqname-to-id HTTP/1.1" 200 156 0.003060
10.84.7.36 - - [2015-03-11 16:26:18] "GET /floating-ip/d9d4f9db-7a29-4205-a976-7759d422f498?exclude_back_refs=True&exclude_children=True HTTP/1.1" 200 1311 0.0046142015-03-11 16:26:18.291 DEBUG [neutron_plugin_contrail.plugins.opencontrail.contrail_plugin] get_network(): filters: {u'name': [u'FIP-VN']} data: [{u'shared': False, u'router:external': True, u'id': u'36bccd67-c826-4c2d-b498-785d80088eb9', u'tenant_id': u'adcde4930d7748e5be70a89c7adf21d9'}]No of creates per second - 6.5 (390 concurrent creates in a minute)