Skip to content

Instantly share code, notes, and snippets.

@yunong

yunong/gist:5631157

Created May 22, 2013 21:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save yunong/5631157 to your computer and use it in GitHub Desktop.
Save yunong/5631157 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
root@headnode (coal) ~]# sdc-amon /pub/580ca9b3-176e-44eb-9724-fff2e8b0cf92/alarms
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 7303
Date: Wed, 22 May 2013 21:41:29 GMT
Server: Amon Master/1.0.0
Request-Id: 5cdc8440-c328-11e2-b63f-4f6256f36ee4
Response-Time: 3
Connection: keep-alive
[
{
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"id": 1,
"probeGroup": "aed0a237-8e18-4cfc-aca1-9cac6db0c0ea",
"closed": false,
"suppressed": false,
"timeOpened": 1369249810988,
"timeClosed": null,
"timeLastEvent": 1369249810705,
"faults": [
{
"type": "probe",
"probe": "85315aed-d4e8-4279-968b-d749324d7c96",
"event": {
"v": 1,
"type": "probe",
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"probeUuid": "85315aed-d4e8-4279-968b-d749324d7c96",
"clear": false,
"data": {
"message": "Log \"/var/svc/log/smartdc-application-jobsupervisor:default.log\" matched (level=ERROR).",
"value": 1,
"details": {
"matches": [
{
"match": {
"name": "MarlinWorker",
"hostname": "bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"pid": 72120,
"component": "Worker-bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"level": 50,
"msg": "onRecord: dropping record for possibly cancelled job (initiating check) { bucket: 'marlin_taskinputs_v2',\n key: '6201993c-b6a9-47f7-9647-d06294089254',\n value: \n { taskInputId: '6201993c-b6a9-47f7-9647-d06294089254',\n jobId: '08d8e959-7dae-4184-9bc8-7b95ba7da4d2',\n taskId: '9ca4e5f2-f7fa-4339-8fcd-94e8f94737b0',\n domain: 'bf970c5b-bd82-4c1d-975d-9fecd9e5e478',\n mantaComputeId: '1.cn.coal.joyent.us',\n agentGeneration: '2013-05-22T18:54:55.840Z',\n input: '/poseidon/jobs/08d8e959-7dae-4184-9bc8-7b95ba7da4d2/stor/poseidon/stor/manatee_backups/1.moray.coal.joyent.us/2013/05/22/19/manta-2013-05-22-19-00-05.gz.0.cb04aa6e-c571-44d0-9929-b29041f9875b',\n p0input: '/poseidon/stor/manatee_backups/1.moray.coal.joyent.us/2013/05/22/19/manta-2013-05-22-19-00-05.gz',\n account: '580ca9b3-176e-44eb-9724-fff2e8b0cf92',\n objectid: 'efbb1484-ae7c-41c9-8f43-e689213e8b10',\n servers: [ [Object] ],\n timeDispatched: '2013-05-22T19:10:04.395Z',\n prevRecordType: 'taskoutput',\n prevRecordId: '1f0de59e-fbe6-4866-bf12-3bd6228a290c',\n timeRead: '2013-05-22T19:10:04.725Z',\n wantInputRemoved: 'true' },\n _id: 1,\n _etag: 'u35e1c68',\n _mtime: 1369249805032,\n _txn_snap: null,\n _count: 1 }",
"time": "2013-05-22T19:10:10.687Z",
"v": 0
}
}
]
}
},
"machine": "bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"uuid": "4b3abf4d-1e75-4d9a-8df5-de15cc5c212d",
"time": 1369249810705,
"agent": "bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"agentAlias": "jobsupervisor.coal.joyent.us-bf970c5b"
}
}
],
"maintFaults": [],
"numEvents": 1
},
{
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"id": 2,
"probeGroup": "aed0a237-8e18-4cfc-aca1-9cac6db0c0ea",
"closed": false,
"suppressed": false,
"timeOpened": 1369249810988,
"timeClosed": null,
"timeLastEvent": 1369255807268,
"faults": [
{
"type": "probe",
"probe": "85315aed-d4e8-4279-968b-d749324d7c96",
"event": {
"v": 1,
"type": "probe",
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"probeUuid": "85315aed-d4e8-4279-968b-d749324d7c96",
"clear": false,
"data": {
"message": "Log \"/var/svc/log/smartdc-application-jobsupervisor:default.log\" matched (level=ERROR).",
"value": 246,
"details": {
"matches": [
{
"match": {
"name": "MarlinWorker",
"hostname": "bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"pid": 72120,
"component": "Worker-bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"level": 50,
"msg": "onRecord: dropping record for possibly cancelled job (initiating check) { bucket: 'marlin_taskinputs_v2',\n key: '6201993c-b6a9-47f7-9647-d06294089254',\n value: \n { taskInputId: '6201993c-b6a9-47f7-9647-d06294089254',\n jobId: '08d8e959-7dae-4184-9bc8-7b95ba7da4d2',\n taskId: '9ca4e5f2-f7fa-4339-8fcd-94e8f94737b0',\n domain: 'bf970c5b-bd82-4c1d-975d-9fecd9e5e478',\n mantaComputeId: '1.cn.coal.joyent.us',\n agentGeneration: '2013-05-22T18:54:55.840Z',\n input: '/poseidon/jobs/08d8e959-7dae-4184-9bc8-7b95ba7da4d2/stor/poseidon/stor/manatee_backups/1.moray.coal.joyent.us/2013/05/22/19/manta-2013-05-22-19-00-05.gz.0.cb04aa6e-c571-44d0-9929-b29041f9875b',\n p0input: '/poseidon/stor/manatee_backups/1.moray.coal.joyent.us/2013/05/22/19/manta-2013-05-22-19-00-05.gz',\n account: '580ca9b3-176e-44eb-9724-fff2e8b0cf92',\n objectid: 'efbb1484-ae7c-41c9-8f43-e689213e8b10',\n servers: [ [Object] ],\n timeDispatched: '2013-05-22T19:10:04.395Z',\n prevRecordType: 'taskoutput',\n prevRecordId: '1f0de59e-fbe6-4866-bf12-3bd6228a290c',\n timeRead: '2013-05-22T19:10:04.725Z',\n wantInputRemoved: 'true' },\n _id: 1,\n _etag: 'u35e1c68',\n _mtime: 1369249805032,\n _txn_snap: null,\n _count: 1 }",
"time": "2013-05-22T20:00:00.605Z",
"v": 0
}
}
]
}
},
"machine": "bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"uuid": "43bd85f9-8545-488f-9b7d-71f1983c524c",
"time": 1369252800638,
"agent": "bf970c5b-bd82-4c1d-975d-9fecd9e5e478",
"agentAlias": "jobsupervisor.coal.joyent.us-bf970c5b"
}
}
],
"maintFaults": [],
"numEvents": 14032
},
{
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"id": 3,
"probeGroup": "805f285d-b89e-440a-827c-5b7deb2765b5",
"closed": false,
"suppressed": false,
"timeOpened": 1369250177523,
"timeClosed": null,
"timeLastEvent": 1369257378569,
"faults": [
{
"type": "probe",
"probe": "666f4786-13c6-4bba-ba40-2cdbec48daf5",
"event": {
"v": 1,
"type": "probe",
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"probeUuid": "666f4786-13c6-4bba-ba40-2cdbec48daf5",
"clear": false,
"data": {
"message": "Command failed (exit status: 127).",
"value": 6,
"details": {
"cmd": "for s in $(mls /poseidon/stor/manta_gc/mako); do echo \"$s \"; test $(mls /poseidon/stor/manta_gc/mako/$s | wc -l) -lt 15; if [[ $? -eq 0 ]]; then echo \"success\"; else echo \"fail\"; fi; done",
"exitStatus": 127,
"signal": null,
"stdout": "",
"stderr": "/bin/sh: line 1: mls: not found\n"
}
},
"machine": "25b202f8-571a-44c1-a108-3d33533b31b6",
"uuid": "d00a535c-b45e-4c29-9b3a-71bf53fa8a13",
"time": 1369257378569,
"agent": "25b202f8-571a-44c1-a108-3d33533b31b6",
"agentAlias": "ops.coal.joyent.us-25b202f8"
}
},
{
"type": "probe",
"probe": "bbd07414-2b4d-44b3-a9ad-84eb1812a6d5",
"event": {
"v": 1,
"type": "probe",
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"probeUuid": "bbd07414-2b4d-44b3-a9ad-84eb1812a6d5",
"clear": false,
"data": {
"message": "Command failed (exit status: 127).",
"value": 6,
"details": {
"cmd": "for s in $(mls /poseidon/stor/manta_gc/moray); do echo \"$s \"; test $(mls /poseidon/stor/manta_gc/moray/$s | wc -l) -lt 15; if [[ $? -eq 0 ]]; then echo \"success\"; else echo \"fail\"; fi; done",
"exitStatus": 127,
"signal": null,
"stdout": "",
"stderr": "/bin/sh: line 1: mls: not found\n"
}
},
"machine": "25b202f8-571a-44c1-a108-3d33533b31b6",
"uuid": "7df19385-1f79-48da-9533-211b5109348f",
"time": 1369257378568,
"agent": "25b202f8-571a-44c1-a108-3d33533b31b6",
"agentAlias": "ops.coal.joyent.us-25b202f8"
}
}
],
"maintFaults": [],
"numEvents": 12
},
{
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"id": 4,
"probeGroup": "989d30d3-9eed-4410-a1f0-06a49e0039c1",
"closed": false,
"suppressed": false,
"timeOpened": 1369251241662,
"timeClosed": null,
"timeLastEvent": 1369258742292,
"faults": [
{
"type": "probe",
"probe": "b6ec0014-ad5c-48b1-96ce-45607fb1d808",
"event": {
"v": 1,
"type": "probe",
"user": "580ca9b3-176e-44eb-9724-fff2e8b0cf92",
"probeUuid": "b6ec0014-ad5c-48b1-96ce-45607fb1d808",
"clear": false,
"data": {
"message": "Command failed (exit status: 1).",
"value": 6,
"details": {
"cmd": "test ! $(find /var/log/manta/upload -type f | wc -l) -gt 0",
"exitStatus": 1,
"signal": null,
"stdout": "",
"stderr": ""
}
},
"machine": "0983b591-01a9-442e-9378-64b2f8645b1a",
"uuid": "90c4da70-3bca-4c1c-b9ae-1494d955d745",
"time": 1369258742292,
"agent": "0983b591-01a9-442e-9378-64b2f8645b1a",
"agentAlias": "nameservice.coal.joyent.us-0983b591"
}
}
],
"maintFaults": [],
"numEvents": 12
}
]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment