gistfile1.txt

root@mesos1-prod-uswest2:~# grep 'prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850' /var/log/upstart/singularity.log
INFO  [2016-03-24 20:11:41,426] com.hubspot.singularity.mesos.SingularityMesosScheduler: Launching task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a slot on slave 12e39206-7831-4a78-9ed4-af58e7c3683a-S8 (mesos-slave12-prod-uswest2.otsql.opentable.com)
INFO  [2016-03-24 20:11:42,132] com.hubspot.singularity.mesos.SingularityMesosScheduler: 25 tasks ([prod-onboarding-service-teamcity.2016.03.21T19.49.33-1458850301251-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-abtest-shadow-stryker.2016.03.22T17.39.44-1458850301288-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-oti-connect-support-teamcity.2016.03.11T23.23.02-1458850301324-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gcwebdiagnostics-teamcity.2015.07.08T23.09.27-1458850301360-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-abtest-shadow-aux-cpennello.2016.03.23T22.33.46-1458850301393-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, stage-gc-microsite-profile-teamcity.2016.01.30T00.23.16-1458850301459-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, offers-editor-teamcity.2015.12.02T12.38.18-1458850301492-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, log-analysis-ap-jziegler.2016.03.16T13.02.50-1458850301525-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-umami-bbi-simulator-teamcity.2015.11.06T00.45.26-1458850301558-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-conversion-teamcity.2016.02.08T20.51.50-1458850301592-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, wishlist-data-teamcity.2016.03.07T18.57.16-1458850301630-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, stage-gc-microsite-login-teamcity.2016.03.21T09.31.26-1458850301663-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, proximity-prod-uswest2-.2016.03.15T21.32.31-1458850301699-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-photocenter-teamcity.2016.03.23T17.18.07-1458850301733-3-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, connect-alerts-shadow-nweichbrodt.2015.08.08T00.18.58-1458850301769-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gc-microsite-login-teamcity.2016.03.21T20.11.38-1458850301802-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-tagservice-stryker.2016.01.25T22.46.25-1458850301834-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, analytics-collector-pdelgado.2016.02.12T02.38.37-1458850301870-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-pay-microsite-posintegration-teamcity.2015.12.21T14.04.19-1458850301903-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, kafka-secor-logstash-pdelgado.2016.03.23T20.57.46-1458850301943-3-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gc-reservations-teamcity.2016.01.05T18.36.32-1458850301983-3-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, kafka-secor-analytics-pdelgado.2016.03.23T21.11.27-1458850302016-5-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-waitlist-service-teamcity.2016.03.21T19.41.04-1458850302060-4-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gcwebsite-teamcity.2016.03.21T20.50.51-1458850302095-4-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]) launched with status DRIVER_RUNNING
DEBUG [2016-03-24 20:11:45,097] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:11:50,767] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:11:56,434] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:02,102] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:07,766] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:11,831] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_LOST (Slave mesos-slave12-prod-uswest2.otsql.opentable.com removed: a new slave registered at the same address) at 1458850331206 
INFO  [2016-03-24 20:12:11,849] com.hubspot.singularity.mesos.SingularityLogSupport: Fetching slave data to find log directory for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a from uri http://mesos-slave12-prod-uswest2.otsql.opentable.com:5051/slave(1)/state.json
WARN  [2016-03-24 20:12:11,852] com.hubspot.singularity.mesos.SingularityLogSupport: Couldn't find matching executor for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
WARN  [2016-03-24 20:12:11,877] com.hubspot.singularity.smtp.MailTemplateHelpers: Couldn't retrieve stdout for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a because task (true) or directory (false) wasn't present
WARN  [2016-03-24 20:12:11,877] com.hubspot.singularity.smtp.MailTemplateHelpers: Couldn't retrieve stderr for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a because task (true) or directory (false) wasn't present
DEBUG [2016-03-24 20:12:11,894] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.063
DEBUG [2016-03-24 20:12:11,904] com.hubspot.singularity.smtp.SingularitySmtpSender: Queuing an email to [platform-alert@opentable.com, admin-decommission]/[] (subject: Task lost (prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a))
WARN  [2016-03-24 20:12:11,907] com.hubspot.singularity.smtp.SingularitySmtpSender: Unable to send message [to: [platform-alert@opentable.com, admin-decommission], subject: Task lost (prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a)]
INFO  [2016-03-24 20:12:12,992] com.hubspot.singularity.mesos.SingularityMesosScheduler: Launching task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a slot on slave 12e39206-7831-4a78-9ed4-af58e7c3683a-S9 (mesos-slave12-prod-uswest2.otsql.opentable.com)
INFO  [2016-03-24 20:12:13,713] com.hubspot.singularity.mesos.SingularityMesosScheduler: 26 tasks ([prod-restaurant-api-gateway-teamcity.2016.02.01T22.38.33-1458850332781-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-onboarding-service-teamcity.2016.03.21T19.49.33-1458850332820-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-abtest-shadow-stryker.2016.03.22T17.39.44-1458850332852-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-oti-connect-support-teamcity.2016.03.11T23.23.02-1458850332887-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gcwebdiagnostics-teamcity.2015.07.08T23.09.27-1458850332924-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-abtest-shadow-aux-cpennello.2016.03.23T22.33.46-1458850332959-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, stage-gc-microsite-profile-teamcity.2016.01.30T00.23.16-1458850333027-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, offers-editor-teamcity.2015.12.02T12.38.18-1458850333059-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, log-analysis-ap-jziegler.2016.03.16T13.02.50-1458850333092-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-umami-bbi-simulator-teamcity.2015.11.06T00.45.26-1458850333125-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-conversion-teamcity.2016.02.08T20.51.50-1458850333156-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, wishlist-data-teamcity.2016.03.07T18.57.16-1458850333190-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, stage-gc-microsite-login-teamcity.2016.03.21T09.31.26-1458850333249-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, proximity-prod-uswest2-.2016.03.15T21.32.31-1458850333292-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-photocenter-teamcity.2016.03.23T17.18.07-1458850333330-3-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, connect-alerts-shadow-nweichbrodt.2015.08.08T00.18.58-1458850333365-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gc-microsite-login-teamcity.2016.03.21T20.11.38-1458850333404-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-tagservice-stryker.2016.01.25T22.46.25-1458850333438-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, analytics-collector-pdelgado.2016.02.12T02.38.37-1458850333469-2-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-pay-microsite-posintegration-teamcity.2015.12.21T14.04.19-1458850333501-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, kafka-secor-logstash-pdelgado.2016.03.23T20.57.46-1458850333534-3-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gc-reservations-teamcity.2016.01.05T18.36.32-1458850333566-3-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, kafka-secor-analytics-pdelgado.2016.03.23T21.11.27-1458850333607-5-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-waitlist-service-teamcity.2016.03.21T19.41.04-1458850333641-4-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, prod-gcwebsite-teamcity.2016.03.21T20.50.51-1458850333676-4-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]) launched with status DRIVER_RUNNING
DEBUG [2016-03-24 20:12:14,038] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:19,674] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:25,351] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:30,956] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:36,532] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:42,127] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:46,750] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING () at 1458850366716 
INFO  [2016-03-24 20:12:46,766] com.hubspot.singularity.mesos.SingularityLogSupport: Fetching slave data to find log directory for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a from uri http://mesos-slave12-prod-uswest2.otsql.opentable.com:5051/slave(1)/state.json
DEBUG [2016-03-24 20:12:46,780] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.031
DEBUG [2016-03-24 20:12:46,795] com.hubspot.singularity.mesos.SingularityLogSupport: Found a directory /mnt/mesos-slave/slaves/12e39206-7831-4a78-9ed4-af58e7c3683a-S9/frameworks/Singularity/executors/prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a/runs/97f5316d-b9d9-4adc-bbfa-0dc05e74a9de for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:47,691] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: No healthcheck present for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:12:53,248] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: Found a failed healthcheck: SingularityTaskHealthcheckResult [statusCode=Optional.absent(), durationMillis=Optional.of(2), timestamp=1458850371758, responseBody=Optional.absent(), errorMessage=Optional.of(Healthcheck failed due to exception: Connection refused: mesos-slave12-prod-uswest2.otsql.opentable.com/10.30.75.164:31962 to http://mesos-slave12-prod-uswest2.otsql.opentable.com:31962/health), taskId=prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]
DEBUG [2016-03-24 20:12:58,778] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: Found a failed healthcheck: SingularityTaskHealthcheckResult [statusCode=Optional.absent(), durationMillis=Optional.of(2), timestamp=1458850376788, responseBody=Optional.absent(), errorMessage=Optional.of(Healthcheck failed due to exception: Connection refused: mesos-slave12-prod-uswest2.otsql.opentable.com/10.30.75.164:31962 to http://mesos-slave12-prod-uswest2.otsql.opentable.com:31962/health), taskId=prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]
DEBUG [2016-03-24 20:13:04,304] com.hubspot.singularity.scheduler.SingularityDeployHealthHelper: Found a failed healthcheck: SingularityTaskHealthcheckResult [statusCode=Optional.of(500), durationMillis=Optional.of(182), timestamp=1458850381801, responseBody=Optional.of({"discovery-client-check":{"healthy":true,"message":"Last update at 2016-03-24T20:12:57.197Z"},"discovery-network-transport-check":{"healthy":true,"message":"[Qq] at index 805277543"},"jvm":{"healthy":true},"server-started":{"healthy":false,"message":"Server not yet started."},"standalone-server":{"healthy":false,"message":"Server still starting up"}}), errorMessage=Optional.absent(), taskId=prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]
INFO  [2016-03-24 20:13:07,012] com.hubspot.singularity.scheduler.SingularityNewTaskChecker: Requested immediate task check for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:13:07,019] com.hubspot.singularity.scheduler.SingularityNewTaskChecker: Got task state HEALTHY for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.005
DEBUG [2016-03-24 20:13:20,377] com.hubspot.singularity.scheduler.SingularityScheduler: Missing 0 instances of request prod-abaggregator-shadow-aux (matching tasks: [prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]), pending request: SingularityPendingRequest [requestId=prod-abaggregator-shadow-aux, deployId=cpennello.2016.03.24T15.36.06, timestamp=1458850400368, pendingType=TASK_DONE, user=Optional.absent(), cmdLineArgsList=Optional.absent(), runId=Optional.absent(), skipHealthchecks=Optional.absent(), message=Optional.absent(), actionId=Optional.absent()]
DEBUG [2016-03-24 20:20:08,065] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458850807917 
DEBUG [2016-03-24 20:20:08,074] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.009
DEBUG [2016-03-24 20:30:08,095] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458851407920 
DEBUG [2016-03-24 20:30:08,102] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.007
DEBUG [2016-03-24 20:40:08,101] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458852007922 
DEBUG [2016-03-24 20:40:08,109] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 20:42:52,403] com.hubspot.singularity.scheduler.SingularityCleaner: Adding task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a to cleanup (bounce)
INFO  [2016-03-24 20:43:51,313] com.hubspot.singularity.mesos.SingularityLogSupport: Fetching slave data to find log directory for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a from uri http://mesos-slave12-prod-uswest2.otsql.opentable.com:5051/slave(1)/state.json
INFO  [2016-03-24 20:43:51,313] com.hubspot.singularity.mesos.SingularityLogSupport: Fetching slave data to find log directory for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a from uri http://mesos-slave12-prod-uswest2.otsql.opentable.com:5051/slave(1)/state.json
WARN  [2016-03-24 20:43:51,330] com.hubspot.singularity.mesos.SingularityLogSupport: Couldn't find matching executor for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
WARN  [2016-03-24 20:43:51,338] com.hubspot.singularity.mesos.SingularityLogSupport: Couldn't find matching executor for task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850301426-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a
DEBUG [2016-03-24 20:50:08,093] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458852607924 
DEBUG [2016-03-24 20:50:08,101] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 21:00:08,135] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458853207927 
DEBUG [2016-03-24 21:00:08,143] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 21:10:08,115] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458853807929 
DEBUG [2016-03-24 21:10:08,123] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 21:20:08,110] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458854407932 
DEBUG [2016-03-24 21:20:08,121] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.011
DEBUG [2016-03-24 21:30:08,122] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458855007934 
DEBUG [2016-03-24 21:30:08,130] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 21:40:08,107] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458855607937 
DEBUG [2016-03-24 21:40:08,115] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
INFO  [2016-03-24 21:43:14,427] com.hubspot.singularity.scheduler.SingularityExpiringUserActionPoller: Discarding cleanup for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a (SingularityTaskCleanup [user=Optional.absent(), cleanupType=BOUNCING, timestamp=1458852172402, taskId=prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a, message=Optional.absent(), actionId=Optional.of(5fcf3104-f242-4f16-b9f4-ecdaaf150da4)]) because of SingularityExpiringBounce [toString()=SingularityExpiringParent [requestId=prod-abaggregator-shadow-aux, user=Optional.absent(), startMillis=1458852169374, actionId=5fcf3104-f242-4f16-b9f4-ecdaaf150da4, expiringAPIRequestObject=SingularityBounceRequest [incremental=Optional.of(false), skipHealthchecks=Optional.of(false), toString()=SingularityExpiringRequestParent [durationMillis=Optional.absent(), actionId=Optional.absent(), message=Optional.absent()]]]]
DEBUG [2016-03-24 21:43:16,809] com.hubspot.singularity.scheduler.SingularityScheduler: Missing 0 instances of request prod-abaggregator-shadow-aux (matching tasks: [prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a]), pending request: SingularityPendingRequest [requestId=prod-abaggregator-shadow-aux, deployId=cpennello.2016.03.24T15.36.06, timestamp=1458855794432, pendingType=CANCEL_BOUNCE, user=Optional.absent(), cmdLineArgsList=Optional.absent(), runId=Optional.absent(), skipHealthchecks=Optional.absent(), message=Optional.of(Bounce expired after 60:00.000), actionId=Optional.of(5fcf3104-f242-4f16-b9f4-ecdaaf150da4)]
DEBUG [2016-03-24 21:50:08,143] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458856207939 
DEBUG [2016-03-24 21:50:08,151] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 22:00:08,146] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458856807941 
DEBUG [2016-03-24 22:00:08,154] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 22:10:08,176] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458857407944 
DEBUG [2016-03-24 22:10:08,188] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.012
DEBUG [2016-03-24 22:20:08,123] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458858007946 
DEBUG [2016-03-24 22:20:08,132] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.009
DEBUG [2016-03-24 22:30:08,139] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458858607948 
DEBUG [2016-03-24 22:30:08,147] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 22:40:08,142] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458859207951 
DEBUG [2016-03-24 22:40:08,151] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.009
DEBUG [2016-03-24 22:50:08,481] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458859807954 
DEBUG [2016-03-24 22:50:08,488] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.008
DEBUG [2016-03-24 23:00:08,321] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458860407957 
DEBUG [2016-03-24 23:00:08,328] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.007
DEBUG [2016-03-24 23:10:08,313] com.hubspot.singularity.mesos.SingularityMesosScheduler: Task prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a is now TASK_RUNNING (Reconciliation: Latest task state) at 1458861007962 
DEBUG [2016-03-24 23:10:08,320] com.hubspot.singularity.mesos.SingularityMesosSchedulerDelegator: Handled status update for prod-abaggregator-shadow-aux-cpennello.2016.03.24T15.36.06-1458850332992-1-mesos_slave12_prod_uswest2.otsql.opentable.com-us_west_2a in 00:00.007