gistfile1.txt

[root@headnode (moya) ~]# sdcadm update --all -y
Finding candidate update images for 23 services (binder, papi, mahi, dhcpd, redis, ufds, adminui, cloudapi, sdc, ca, sapi, manatee, moray, amon, vmapi, napi, manta, imgapi, workflow, cnapi, amonredis, fwapi, assets).
Note: There are no "manta" instances. Only the service configuration will be updated.

This update will make the following changes:
    download 23 images (1192 MiB):
        image d6f341e4-ef02-11e4-8be7-2f79005449b8 (redis@release-20150430-20150430T063105Z-g0d6e5b9)
        image c7565f0e-ef03-11e4-ab3b-d7ca277eb670 (sapi@release-20150430-20150430T063118Z-g983d6be)
        image 43b44950-ef02-11e4-bcce-53d3affcacaa (amonredis@release-20150430-20150430T062628Z-g4bbca77)
        image d95cb044-eeff-11e4-b61f-87a8efe4eebf (ufds@release-20150430-20150430T060525Z-g9d089af)
        image 1856e112-eeff-11e4-99a7-37890f4f7d1e (sdc-postgres@release-20150430-20150430T060201Z-g38dfc89)
        image bf2a00e2-ef03-11e4-83b6-2b7f1988e077 (fwapi@release-20150430-20150430T062725Z-ga737049)
        image 76b5eb88-ef02-11e4-9cc3-7735cff09b1c (dhcpd@release-20150430-20150430T062631Z-g7d7030d)
        image 845480a6-ef02-11e4-a313-135b7a82221b (manta-moray@release-20150430-20150430T062614Z-g088ac51)
        image 7d26e094-ef02-11e4-a747-e3dc6a6df75c (assets@release-20150430-20150430T062822Z-g6012d50)
        image 1cd6061a-ef03-11e4-a62e-4f6a4a05d87a (manta-nameservice@release-20150430-20150430T062610Z-gba090a8)
        image 6bcd384e-ef01-11e4-b9c6-97fc1d95c019 (amon@release-20150430-20150430T061233Z-gabe4be5)
        image f2e25ed0-ef02-11e4-8be8-dbda0c1e293f (papi@release-20150430-20150430T063023Z-g21bb506)
        image d059c0b4-ef03-11e4-b05d-a3b1aa01c658 (vmapi@release-20150430-20150430T063407Z-g566bc15)
        image cd81f10a-ef02-11e4-ad44-073e74598cf2 (manta-authcache@release-20150430-20150430T062622Z-g81b6a2c)
        image b835dfc6-eeff-11e4-93bd-dfcdc477e98e (napi@release-20150430-20150430T060543Z-ge995deb)
        image 8ec5fa50-ef03-11e4-ad1d-574898c90fc0 (adminui@release-20150430-20150430T062930Z-gab70cec)
        image 1022fbc0-eeff-11e4-96a8-8f6f53001ec1 (manta-deployment@release-20150430-20150430T060139Z-gb25b8f6)
        image 946b15e4-ef03-11e4-95d4-1f14e504fd32 (cloudapi@release-20150430-20150430T062908Z-g03025fd)
        image 4fb5ef34-ef01-11e4-b8a0-4febdf9dde6d (imgapi@release-20150430-20150430T061433Z-g1f22225)
        image 15f98956-ef04-11e4-bc14-3f9087c29341 (sdc@release-20150430-20150430T063322Z-gc40d216)
        image ea13d346-ef03-11e4-beea-0f3e305ceddb (workflow@release-20150430-20150430T063315Z-gc0446f8)
        image 971a26cc-ef03-11e4-82d7-570e6c4825f1 (ca@release-20150430-20150430T062704Z-gb775d2d)
        image abf8d576-ef02-11e4-a7a0-2f3d05ac0248 (cnapi@release-20150430-20150430T062701Z-g9a4118c)
    update "papi" service to image f2e25ed0-ef02-11e4-8be8-dbda0c1e293f (papi@release-20150430-20150430T063023Z-g21bb506)
    update "dhcpd" service to image 76b5eb88-ef02-11e4-9cc3-7735cff09b1c (dhcpd@release-20150430-20150430T062631Z-g7d7030d)
    update "redis" service to image d6f341e4-ef02-11e4-8be7-2f79005449b8 (redis@release-20150430-20150430T063105Z-g0d6e5b9)
    update "adminui" service to image 8ec5fa50-ef03-11e4-ad1d-574898c90fc0 (adminui@release-20150430-20150430T062930Z-gab70cec)
    update "cloudapi" service to image 946b15e4-ef03-11e4-95d4-1f14e504fd32 (cloudapi@release-20150430-20150430T062908Z-g03025fd)
    update "sdc" service to image 15f98956-ef04-11e4-bc14-3f9087c29341 (sdc@release-20150430-20150430T063322Z-gc40d216)
    update "ca" service to image 971a26cc-ef03-11e4-82d7-570e6c4825f1 (ca@release-20150430-20150430T062704Z-gb775d2d)
    update "amon" service to image 6bcd384e-ef01-11e4-b9c6-97fc1d95c019 (amon@release-20150430-20150430T061233Z-gabe4be5)
    update "vmapi" service to image d059c0b4-ef03-11e4-b05d-a3b1aa01c658 (vmapi@release-20150430-20150430T063407Z-g566bc15)
    update "napi" service to image b835dfc6-eeff-11e4-93bd-dfcdc477e98e (napi@release-20150430-20150430T060543Z-ge995deb)
    update "manta" service to image 1022fbc0-eeff-11e4-96a8-8f6f53001ec1 (manta-deployment@release-20150430-20150430T060139Z-gb25b8f6)
    update "workflow" service to image ea13d346-ef03-11e4-beea-0f3e305ceddb (workflow@release-20150430-20150430T063315Z-gc0446f8)
    update "cnapi" service to image abf8d576-ef02-11e4-a7a0-2f3d05ac0248 (cnapi@release-20150430-20150430T062701Z-g9a4118c)
    update "amonredis" service to image 43b44950-ef02-11e4-bcce-53d3affcacaa (amonredis@release-20150430-20150430T062628Z-g4bbca77)
    update "fwapi" service to image bf2a00e2-ef03-11e4-83b6-2b7f1988e077 (fwapi@release-20150430-20150430T062725Z-ga737049)
    update "assets" service to image 7d26e094-ef02-11e4-a747-e3dc6a6df75c (assets@release-20150430-20150430T062822Z-g6012d50)
    update "imgapi" service to image 4fb5ef34-ef01-11e4-b8a0-4febdf9dde6d (imgapi@release-20150430-20150430T061433Z-g1f22225)
    update "ufds" service to image d95cb044-eeff-11e4-b61f-87a8efe4eebf (ufds@release-20150430-20150430T060525Z-g9d089af)
    update "moray" service to image 845480a6-ef02-11e4-a313-135b7a82221b (manta-moray@release-20150430-20150430T062614Z-g088ac51)
    update "sapi" service to image c7565f0e-ef03-11e4-ab3b-d7ca277eb670 (sapi@release-20150430-20150430T063118Z-g983d6be)
    update "manatee" service to image 1856e112-eeff-11e4-99a7-37890f4f7d1e (sdc-postgres@release-20150430-20150430T060201Z-g38dfc89)
    update "binder" service to image 1cd6061a-ef03-11e4-a62e-4f6a4a05d87a (manta-nameservice@release-20150430-20150430T062610Z-gba090a8)
    update "mahi" service to image cd81f10a-ef02-11e4-ad44-073e74598cf2 (manta-authcache@release-20150430-20150430T062622Z-g81b6a2c)

Create work dir: /var/sdcadm/updates/20150508T044705Z
Downloading image d6f341e4-ef02-11e4-8be7-2f79005449b8 (redis@release-20150430-20150430T063105Z-g0d6e5b9)
Downloading image c7565f0e-ef03-11e4-ab3b-d7ca277eb670 (sapi@release-20150430-20150430T063118Z-g983d6be)
Downloading image 43b44950-ef02-11e4-bcce-53d3affcacaa (amonredis@release-20150430-20150430T062628Z-g4bbca77)
Downloading image d95cb044-eeff-11e4-b61f-87a8efe4eebf (ufds@release-20150430-20150430T060525Z-g9d089af)
Imported image 43b44950-ef02-11e4-bcce-53d3affcacaa (amonredis@release-20150430-20150430T062628Z-g4bbca77)
Downloading image 1856e112-eeff-11e4-99a7-37890f4f7d1e (sdc-postgres@release-20150430-20150430T060201Z-g38dfc89)
Imported image d6f341e4-ef02-11e4-8be7-2f79005449b8 (redis@release-20150430-20150430T063105Z-g0d6e5b9)
Downloading image bf2a00e2-ef03-11e4-83b6-2b7f1988e077 (fwapi@release-20150430-20150430T062725Z-ga737049)
Imported image c7565f0e-ef03-11e4-ab3b-d7ca277eb670 (sapi@release-20150430-20150430T063118Z-g983d6be)
Downloading image 76b5eb88-ef02-11e4-9cc3-7735cff09b1c (dhcpd@release-20150430-20150430T062631Z-g7d7030d)
Imported image d95cb044-eeff-11e4-b61f-87a8efe4eebf (ufds@release-20150430-20150430T060525Z-g9d089af)
Downloading image 845480a6-ef02-11e4-a313-135b7a82221b (manta-moray@release-20150430-20150430T062614Z-g088ac51)
Imported image 76b5eb88-ef02-11e4-9cc3-7735cff09b1c (dhcpd@release-20150430-20150430T062631Z-g7d7030d)
Downloading image 7d26e094-ef02-11e4-a747-e3dc6a6df75c (assets@release-20150430-20150430T062822Z-g6012d50)
Imported image bf2a00e2-ef03-11e4-83b6-2b7f1988e077 (fwapi@release-20150430-20150430T062725Z-ga737049)
Downloading image 1cd6061a-ef03-11e4-a62e-4f6a4a05d87a (manta-nameservice@release-20150430-20150430T062610Z-gba090a8)
Imported image 7d26e094-ef02-11e4-a747-e3dc6a6df75c (assets@release-20150430-20150430T062822Z-g6012d50)
Downloading image 6bcd384e-ef01-11e4-b9c6-97fc1d95c019 (amon@release-20150430-20150430T061233Z-gabe4be5)
Imported image 1856e112-eeff-11e4-99a7-37890f4f7d1e (sdc-postgres@release-20150430-20150430T060201Z-g38dfc89)
Downloading image f2e25ed0-ef02-11e4-8be8-dbda0c1e293f (papi@release-20150430-20150430T063023Z-g21bb506)
Imported image 845480a6-ef02-11e4-a313-135b7a82221b (manta-moray@release-20150430-20150430T062614Z-g088ac51)
Downloading image d059c0b4-ef03-11e4-b05d-a3b1aa01c658 (vmapi@release-20150430-20150430T063407Z-g566bc15)
Imported image d059c0b4-ef03-11e4-b05d-a3b1aa01c658 (vmapi@release-20150430-20150430T063407Z-g566bc15)
Downloading image cd81f10a-ef02-11e4-ad44-073e74598cf2 (manta-authcache@release-20150430-20150430T062622Z-g81b6a2c)
Imported image 6bcd384e-ef01-11e4-b9c6-97fc1d95c019 (amon@release-20150430-20150430T061233Z-gabe4be5)
Downloading image b835dfc6-eeff-11e4-93bd-dfcdc477e98e (napi@release-20150430-20150430T060543Z-ge995deb)
Imported image f2e25ed0-ef02-11e4-8be8-dbda0c1e293f (papi@release-20150430-20150430T063023Z-g21bb506)
Downloading image 8ec5fa50-ef03-11e4-ad1d-574898c90fc0 (adminui@release-20150430-20150430T062930Z-gab70cec)
Imported image b835dfc6-eeff-11e4-93bd-dfcdc477e98e (napi@release-20150430-20150430T060543Z-ge995deb)
Downloading image 1022fbc0-eeff-11e4-96a8-8f6f53001ec1 (manta-deployment@release-20150430-20150430T060139Z-gb25b8f6)
Imported image cd81f10a-ef02-11e4-ad44-073e74598cf2 (manta-authcache@release-20150430-20150430T062622Z-g81b6a2c)
Downloading image 946b15e4-ef03-11e4-95d4-1f14e504fd32 (cloudapi@release-20150430-20150430T062908Z-g03025fd)
Imported image 1cd6061a-ef03-11e4-a62e-4f6a4a05d87a (manta-nameservice@release-20150430-20150430T062610Z-gba090a8)
Downloading image 4fb5ef34-ef01-11e4-b8a0-4febdf9dde6d (imgapi@release-20150430-20150430T061433Z-g1f22225)
Imported image 1022fbc0-eeff-11e4-96a8-8f6f53001ec1 (manta-deployment@release-20150430-20150430T060139Z-gb25b8f6)
Downloading image 15f98956-ef04-11e4-bc14-3f9087c29341 (sdc@release-20150430-20150430T063322Z-gc40d216)
Imported image 8ec5fa50-ef03-11e4-ad1d-574898c90fc0 (adminui@release-20150430-20150430T062930Z-gab70cec)
Downloading image ea13d346-ef03-11e4-beea-0f3e305ceddb (workflow@release-20150430-20150430T063315Z-gc0446f8)
Imported image 4fb5ef34-ef01-11e4-b8a0-4febdf9dde6d (imgapi@release-20150430-20150430T061433Z-g1f22225)
Downloading image 971a26cc-ef03-11e4-82d7-570e6c4825f1 (ca@release-20150430-20150430T062704Z-gb775d2d)
Imported image 946b15e4-ef03-11e4-95d4-1f14e504fd32 (cloudapi@release-20150430-20150430T062908Z-g03025fd)
Downloading image abf8d576-ef02-11e4-a7a0-2f3d05ac0248 (cnapi@release-20150430-20150430T062701Z-g9a4118c)
Imported image ea13d346-ef03-11e4-beea-0f3e305ceddb (workflow@release-20150430-20150430T063315Z-gc0446f8)
Imported image abf8d576-ef02-11e4-a7a0-2f3d05ac0248 (cnapi@release-20150430-20150430T062701Z-g9a4118c)
Imported image 971a26cc-ef03-11e4-82d7-570e6c4825f1 (ca@release-20150430-20150430T062704Z-gb775d2d)
Imported image 15f98956-ef04-11e4-bc14-3f9087c29341 (sdc@release-20150430-20150430T063322Z-gc40d216)
Installing image f2e25ed0-ef02-11e4-8be8-dbda0c1e293f (papi@release-20150430-20150430T063023Z-g21bb506)
Reprovisioning papi VM d32cbbcf-ea7a-4a82-929e-2c1fdd8eb4ca
Wait (60s) for papi instance d32cbbcf-ea7a-4a82-929e-2c1fdd8eb4ca to come up
Installing image 76b5eb88-ef02-11e4-9cc3-7735cff09b1c (dhcpd@release-20150430-20150430T062631Z-g7d7030d)
Reprovisioning dhcpd VM bb4abd6d-cd2c-4bcc-bf08-50030840cb49
Wait (60s) for dhcpd instance bb4abd6d-cd2c-4bcc-bf08-50030840cb49 to come up
Installing image d6f341e4-ef02-11e4-8be7-2f79005449b8 (redis@release-20150430-20150430T063105Z-g0d6e5b9)
Reprovisioning redis VM 64a6d82b-14d4-4c83-8ca7-5894c551fece
Wait (60s) for redis instance 64a6d82b-14d4-4c83-8ca7-5894c551fece to come up
Installing image 8ec5fa50-ef03-11e4-ad1d-574898c90fc0 (adminui@release-20150430-20150430T062930Z-gab70cec)
Reprovisioning adminui VM a276909a-90d3-431d-af6c-00ddce29485b
Wait (60s) for adminui instance a276909a-90d3-431d-af6c-00ddce29485b to come up
Installing image 946b15e4-ef03-11e4-95d4-1f14e504fd32 (cloudapi@release-20150430-20150430T062908Z-g03025fd)
Reprovisioning cloudapi VM 7c00deca-472f-412e-8339-6c000dfafed5
Wait (60s) for cloudapi instance 7c00deca-472f-412e-8339-6c000dfafed5 to come up
Installing image 15f98956-ef04-11e4-bc14-3f9087c29341 (sdc@release-20150430-20150430T063322Z-gc40d216)
Reprovisioning sdc VM 9020fad3-2932-4499-b642-4d9746ad76a1
Wait (60s) for sdc instance 9020fad3-2932-4499-b642-4d9746ad76a1 to come up
Installing image 971a26cc-ef03-11e4-82d7-570e6c4825f1 (ca@release-20150430-20150430T062704Z-gb775d2d)
Reprovisioning ca VM d59b5c44-3256-4529-bac1-c737b14548e3
Wait (60s) for ca instance d59b5c44-3256-4529-bac1-c737b14548e3 to come up
Installing image 6bcd384e-ef01-11e4-b9c6-97fc1d95c019 (amon@release-20150430-20150430T061233Z-gabe4be5)
Reprovisioning amon VM 5e71c81b-118a-4c59-ae2d-836a9718717e
Wait (60s) for amon instance 5e71c81b-118a-4c59-ae2d-836a9718717e to come up
Installing image d059c0b4-ef03-11e4-b05d-a3b1aa01c658 (vmapi@release-20150430-20150430T063407Z-g566bc15)
Reprovisioning vmapi VM 981c16e4-9121-4585-9066-6bb4e79b33ef
Wait (60s) for vmapi instance 981c16e4-9121-4585-9066-6bb4e79b33ef to come up
Installing image b835dfc6-eeff-11e4-93bd-dfcdc477e98e (napi@release-20150430-20150430T060543Z-ge995deb)
Reprovisioning napi VM 87a4b58d-8fa4-4a1c-bdd1-b370a6c5952e
Wait (60s) for napi instance 87a4b58d-8fa4-4a1c-bdd1-b370a6c5952e to come up
Installing image ea13d346-ef03-11e4-beea-0f3e305ceddb (workflow@release-20150430-20150430T063315Z-gc0446f8)
Reprovisioning workflow VM 89f64f42-3878-426a-987b-19f142af6b53
Wait (60s) for workflow instance 89f64f42-3878-426a-987b-19f142af6b53 to come up
Installing image abf8d576-ef02-11e4-a7a0-2f3d05ac0248 (cnapi@release-20150430-20150430T062701Z-g9a4118c)
Reprovisioning cnapi VM a229c06c-c07e-44a3-be53-52a1085ed600
Wait (60s) for cnapi instance a229c06c-c07e-44a3-be53-52a1085ed600 to come up
Installing image 43b44950-ef02-11e4-bcce-53d3affcacaa (amonredis@release-20150430-20150430T062628Z-g4bbca77)
Reprovisioning amonredis VM 343715e7-8431-4247-9e93-d9efd5fe9673
Wait (60s) for amonredis instance 343715e7-8431-4247-9e93-d9efd5fe9673 to come up
Installing image bf2a00e2-ef03-11e4-83b6-2b7f1988e077 (fwapi@release-20150430-20150430T062725Z-ga737049)
Reprovisioning fwapi VM 36f8e4cb-c559-4699-aca0-02bf0d6ecdbc
Wait (60s) for fwapi instance 36f8e4cb-c559-4699-aca0-02bf0d6ecdbc to come up
Update "assets" VM b2675d44-da23-49a5-b36c-c507d1c8755e user-script
Installing image 7d26e094-ef02-11e4-a747-e3dc6a6df75c (assets@release-20150430-20150430T062822Z-g6012d50)
Reprovisioning assets VM b2675d44-da23-49a5-b36c-c507d1c8755e
Wait (60s) for assets instance b2675d44-da23-49a5-b36c-c507d1c8755e to come up
Installing image 4fb5ef34-ef01-11e4-b8a0-4febdf9dde6d (imgapi@release-20150430-20150430T061433Z-g1f22225)
Reprovisioning imgapi VM eb893574-9218-4444-91f4-21e36cf5b390
Wait (60s) for imgapi instance eb893574-9218-4444-91f4-21e36cf5b390 to come up
Disabling imgapi service
Running IMGAPI migration-008-new-storage-layout.js
Running IMGAPI migration-009-backfill-archive.js
Running IMGAPI migration-010-backfill-billing_tags.js
Running IMGAPI migration-011-backfill-published_at.js
Enabling imgapi service
Running vmadm lookup to get local manatee
Running manatee-adm to find primary manatee
Creating ufds buckets backup 20150508T054238532.sql
Copying backup file to HeadNode
Moving backup file to /var/sdcadm/ufds-backup
Installing image d95cb044-eeff-11e4-b61f-87a8efe4eebf (ufds@release-20150430-20150430T060525Z-g9d089af)
Reprovisioning ufds VM 9bf14425-3f1b-4a91-9828-7b46628e956f
Wait (60s) for ufds instance 9bf14425-3f1b-4a91-9828-7b46628e956f to come up
Installing image 845480a6-ef02-11e4-a313-135b7a82221b (manta-moray@release-20150430-20150430T062614Z-g088ac51)
Provisioning Temporary moray VM moray0tmp
Wait (sleep) for moray instance ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e to come up
Running vmadm lookup to get tmp instance UUID
Checking if tmp instace ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e services have errors
Waiting until ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e instance is in DNS
Update error: {"message":"error Getting VM info ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e:\n\targv: [\"/opt/smartdc/bin/sdc-oneachnode\",\"-n 564d6390-f5b1-d9b7-68ec-50905e7eb627 \",\"-j\",\"/usr/sbin/vmadm get ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e\"]\n\texit status: 10\n\tstdout:\n[]\n\tstderr:\nERROR: could not find all nodes\n\n  missing nodes:\n    564d6390-f5b1-d9b7-68ec-50905e7eb627","we_cause":{"killed":false,"code":10,"signal":null},"code":"InternalError","exitStatus":1}
sdcadm update: error: error Getting VM info ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e:
	argv: ["/opt/smartdc/bin/sdc-oneachnode","-n 564d6390-f5b1-d9b7-68ec-50905e7eb627 ","-j","/usr/sbin/vmadm get ffa1e3c0-f762-4e0b-be60-8ba3bb7e417e"]
	exit status: 10
	stdout:
[]
	stderr:
ERROR: could not find all nodes

  missing nodes:
    564d6390-f5b1-d9b7-68ec-50905e7eb627