Skip to content

Instantly share code, notes, and snippets.

This file has been truncated, but you can view the full file.
Jul 10 07:50:48 10.35.0.13 cobalt-compute: 9120 140600130837392 periodic_task run_periodic_tasks Running periodic task CobaltManager._refresh_host
Jul 10 07:50:48 10.35.0.13 rabbitmq.log: =INFO REPORT==== 10-Jul-2014::07:50:48 ===
Jul 10 07:50:48 10.35.0.13 rabbitmq.log: accepting AMQP connection <0.1236.0> (10.35.0.14:52133 -> 10.35.0.3:5672)
Jul 10 07:50:48 10.35.0.14 2014-07-10 07:50:48.072 14453 140340022848080 oslo.messaging._drivers.impl_rabbit impl_rabbit _connect Connected to AMQP server on 10.35.0.3:5672
Jul 10 07:50:48 10.35.0.14 2014-07-10 07:50:48.073 14453 140340022848080 kombu.channel debug __wrapped [Kombu channel:1] exchange_declare(nowait=False, exchange='nova', durable=False, passive=False, arguments=None, type='topic', auto_delete=False) __wrapped /usr/lib64/python2.7/site-packages/kombu/utils/debug.py:56
Jul 10 07:50:48 10.35.0.14 2014-07-10 07:50:48.074 14453 140340022848080 kombu.channel debug __wrapped [Kombu channel:1] prepare_message('{"oslo.message": "{\\"_context_roles\\": [\\"_
@noelbk
noelbk / gist:619426fa88c3bdd0534c
Created July 9, 2014 17:46
cmessages-test_rolling_evac-fail-3
This file has been truncated, but you can view the full file.
Jul 8 19:24:22 10.35.0.14 nova-compute 2014-07-08 19:24:22.946 6347 INFO nova.virt.libvirt.imagecache [-] Removable base files: /mnt/novadisk/nova/instances/_base/afc9137c447f75f367d2a200ace85682280e8375
Jul 8 19:24:22 10.35.0.14 nova-compute 2014-07-08 19:24:22.946 6347 INFO nova.virt.libvirt.imagecache [-] Base file too young to remove: /mnt/novadisk/nova/instances/_base/afc9137c447f75f367d2a200ace85682280e8375
Jul 8 19:24:23 10.35.0.13 [2014-07-08 19:24:23.049] 941/Thread-1 savage.WSGI-FredInterface/INFO: 10.35.0.2 "GET /private/status HTTP/1.1" 200 3035
Jul 8 19:24:24 10.35.0.15 nova-network.log: /usr/lib64/python2.7/site-packages/sqlalchemy/engine/default.py:435: Warning: Incorrect datetime value: '2014-07-08 19:14:24+00:00' for column 'updated_at' at row 1
Jul 8 19:24:24 10.35.0.15 nova-network.log: cursor.execute(statement, parameters)
Jul 8 19:24:26 10.35.0.13 nova-compute 2014-07-08 19:24:26.962 4705 AUDIT nova.compute.resource_tracker [-] Auditing locally available compute resources
Jul 8 1
@noelbk
noelbk / gist:96bdea94ac976828d220
Created July 7, 2014 16:55
oslo.messaging nosetests fail on Icehouse node
[root@node-10-35-0-14 oslo.messaging-1.3.0.5.gc21a2c2]# nosetests tests
nosetests tests
EEETraceback (most recent call last):
File "/usr/lib64/python2.7/site-packages/eventlet/hubs/hub.py", line 346, in fire_timers
timer()
File "/usr/lib64/python2.7/site-packages/eventlet/hubs/timer.py", line 56, in __call__
cb(*args, **kw)
File "/usr/lib64/python2.7/site-packages/eventlet/greenthread.py", line 194, in main
result = function(*args, **kwargs)
File "/root/oslo.messaging-1.3.0.5.gc21a2c2/tests/test_executor.py", line 100, in <lambda>
@noelbk
noelbk / gist:12adcfe188d9f54f971d
Last active August 29, 2015 14:03
test_oslo.py times out waiting for reply
[root@node-10-35-0-14 ~]# python ./test_oslo.py
python ./test_oslo.py
2014-07-07 16:31:33,764 8949 139757228312320 /usr/lib64/python2.7/site-packages/stevedore/extension.py:_load_plugins
found extension EntryPoint.parse('qpid = oslo.messaging._drivers.impl_qpid:QpidDriver')
2014-07-07 16:31:33,764 8949 139757228312320 /usr/lib64/python2.7/site-packages/stevedore/extension.py:_load_plugins
found extension EntryPoint.parse('zmq = oslo.messaging._drivers.impl_zmq:ZmqDriver')
2014-07-07 16:31:33,765 8949 139757228312320 /usr/lib64/python2.7/site-packages/stevedore/extension.py:_load_plugins
found extension EntryPoint.parse('kombu = oslo.messaging._drivers.impl_rabbit:RabbitDriver')
2014-07-07 16:31:33,765 8949 139757228312320 /usr/lib64/python2.7/site-packages/stevedore/extension.py:_load_plugins
found extension EntryPoint.parse('rabbit = oslo.messaging._drivers.impl_rabbit:RabbitDriver')
eval $(sed -n \
-e 's/^auth_uri\s*=\s*/AUTH_URI=/p' \
-e 's/^admin_password\s*=\s*/OS_PASSWORD=/p' \
-e 's/^admin_tenant_name\s*=\s*/OS_TENANT_NAME=/p' \
-e 's/^admin_user\s*=\s*/OS_USERNAME=/p' \
/etc/nova/nova.conf)
OS_AUTH_URL=$AUTH_URI/v2.0
export OS_TENANT_NAME OS_USERNAME OS_PASSWORD OS_AUTH_URL
OS_TENANT_NAME=admin
OS_USERNAME=admin
Jun 27 00:46:16 10.35.0.15 cobalt-compute: hooks INFO Calling hooks for pre_migrate action
Jun 27 00:46:16 10.35.0.15 cobalt-compute: hooks INFO Done calling hooks for pre_migrate action
Jun 27 00:46:16 10.35.0.15 cobalt-compute: amqpdriver DEBUG MSG_ID is 5a138705165b4b8d9a4b3fe374038736
Jun 27 00:46:16 10.35.0.15 cobalt-compute: amqp DEBUG UNIQUE_ID is 39463529d74f49a78d4c16c67dda14c5.
Jun 27 00:46:16 10.35.0.15 cobalt-compute: impl_rabbit ERROR Failed to publish message to topic 'network': [Errno 104] Connection reset by peer
Traceback (most recent call last):
File "/usr/lib64/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 622, in ensure
File "/usr/lib64/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 718, in _publish
File "/usr/lib64/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 379, in __init__
File "/usr/lib64/python2.7/site-packages/oslo/messaging/_drivers/impl_rabbit.py", line 326, in __init__
tcpdump -veplni host host 10.144.15.189
error : ret -1
tcpdump: listening on host, link-type EN10MB (Ethernet), capture size 65535 bytes
22:50:18.908740 00:01:e8:8b:56:47 > 84:2b:2b:5d:d1:9b, ethertype 802.1Q (0x8100), length 78: vlan 2980, p 2, ethertype IPv4, (tos 0x0, ttl 59, id 19821, offset 0, flags [DF], proto TCP (6), length 60)
172.18.47.145.37495 > 10.144.15.189.22: Flags [S], cksum 0x4604 (correct), seq 3357281722, win 25400, options [mss 1270,sackOK,TS val 5652695 ecr 0,nop,wscale 7], length 0
22:50:19.907585 00:01:e8:8b:56:47 > 84:2b:2b:5d:d1:9b, ethertype 802.1Q (0x8100), length 78: vlan 2980, p 2, ethertype IPv4, (tos 0x0, ttl 59, id 19822, offset 0, flags [DF], proto TCP (6), length 60)
172.18.47.145.37495 > 10.144.15.189.22: Flags [S], cksum 0x450a (correct), seq 3357281722, win 25400, options [mss 1270,sackOK,TS val 5652945 ecr 0,nop,wscale 7], length 0
22:50:21.911808 00:01:e8:8b:56:47 > 84:2b:2b:5d:d1:9b, ethertype 802.1Q (0x8100), length 78: vlan 2980, p 2, ethertype IPv4, (tos 0x0
curl http://10.144.14.215:8085/Snh_KFlowReq?flow_idx=
<?xml-stylesheet type="text/xsl" href="/universal_parse.xsl"?>
<KFlowResp type="sandesh">
<flow_list type="list" identifier="1">
<list type="struct" size="2">
<KFlowInfo>
<index type="i32" identifier="1">176552</index>
<rflow type="i32" identifier="2">213412</rflow>
<sip type="string" identifier="3">172.18.47.145</sip>
Boot vms
kill rabbit
nova cobalt-migrate
fail
Search for '5a138705165b4b8d9a4b3fe374038736' below
Jun 27 00:45:31 10.35.0.13 [2014-06-27 00:45:31.306] 11112/MainThread savage.arp_forever/INFO: execute['arping', '-q', '-A', '-I', 'services', '-c', '1', '-U', '10.35.2.5']
Jun 27 00:45:31 10.35.0.15 [2014-06-27 00:45:31.316] 3710/MainThread savage.arp_forever/INFO: execute['arping', '-q', '-A', '-I', 'services', '-c', '1', '-U', '10.35.2.8']
Jun 27 00:45:32 10.35.0.2 [2014-06-27 00:45:32.846] 629/MainThread savage.Grandpa/INFO: State change: optimal -> optimal: Grandpa<state=optimal, leases=3, blessed=3| ready=3>
This file has been truncated, but you can view the full file.
cat /var/log/cmessages
Jun 27 00:05:21 10.144.14.194 dev: Reinit starting (VERSION=latest).
Jun 27 00:05:21 10.144.14.194 syslog-ng[567]: Syslog connection established; fd='93', server='AF_INET(10.144.14.194:514)', local='AF_INET(0.0.0.0:0)'
Jun 27 00:05:21 10.144.14.194 syslog-ng[567]: Configuration reload request received, reloading configuration;
Jun 27 00:05:22 10.144.14.194 [2014-06-27 00:05:22.608] 30529/MainThread savage.jobs_installer/DEBUG: Extracting job image /mnt/flash/tftpboot/jobs.img.
Jun 27 00:05:22 10.144.14.194 [2014-06-27 00:05:22.608] 30529/MainThread savage/INFO: execute['cpio', '-imd', '--no-absolute-filenames']
Jun 27 00:05:22 10.144.14.194 [2014-06-27 00:05:22.615] 30529/MainThread savage/WARNING: cmd cpio -imd --no-absolute-filenames returned with ignored stderr lines: ['1 block']
Jun 27 00:05:23 10.144.14.194 [2014-06-27 00:05:23.434] 30542/MainThread savage/INFO: guaranteed that /mnt/flash/conf exists (created=[], mode=None, owner=None, group=None)
Jun 27 00:05:23 10.144.14.194 [201