gistfile1.txt

mesos-docker-executor: /lib64/libcurl.so.4: no version information available (required by /opt/mesosphere/packages/mesos--d3f2ef31fb91d0928f838d25ff7bc580aad7ccc3/lib/libmesos-0.24.1.so)
I1001 17:50:50.365727  2985 exec.cpp:133] Version: 0.24.1
I1001 17:50:50.367946  2989 exec.cpp:207] Executor registered on slave 20151001-152803-2080636938-5050-1237-S0
+ /work/bin/etcd-mesos-scheduler -alsologtostderr=true -framework-name=etcd -cluster-size=3 -master=zk://master.mesos:2181/mesos -zk-framework-persist=zk://master.mesos:2181/etcd -v=1 -auto-reseed=true -reseed-timeout=240 -sandbox-disk-limit=4096 -sandbox-cpu-limit=1 -sandbox-mem-limit=2048 -admin-port=10000 -driver-port=10001 -artifact-port=10002 -framework-weburi=http://etcd.marathon.mesos:10000/stats
I1001 17:50:53.471500       6 app.go:225] Found stored framework ID in Zookeeper, attempting to re-use: 20151001-152803-2080636938-5050-1237-0001
I1001 17:50:56.474654       6 scheduler.go:225] found failover_timeout = 168h0m0s
I1001 17:50:56.474747       6 scheduler.go:251] Initializing mesos scheduler driver
I1001 17:50:56.474818       6 scheduler.go:719] Starting the scheduler driver...
I1001 17:50:56.474879       6 http_transporter.go:396] listening on 10.0.1.11 port 10001
I1001 17:50:56.474918       6 scheduler.go:590] running instances: 0 desired: 3 offers: 0
I1001 17:50:56.474954       6 scheduler.go:598] PeriodicLaunchRequestor skipping due to Immutable scheduler state.
I1001 17:50:56.474977       6 scheduler.go:927] Admin HTTP interface Listening on port 10000
I1001 17:50:57.475148       6 scheduler.go:750] Mesos scheduler driver started with PID=scheduler(1)@10.0.1.11:10001
I1001 17:50:57.475197       6 scheduler.go:761] starting master detector *zoo.MasterDetector: &{client:<nil> leaderNode: bootstrapLock:{w:{state:0 sema:0} writerSem:0 readerSem:0 readerCount:0 readerWait:0} bootstrapFunc:0x6f5a10 ignoreInstalled:0 minDetectorCyclePeriod:1000000000 done:0xc20805c7e0 cancel:0x6f5a00}
I1001 17:50:57.475371       6 scheduler.go:923] Scheduler driver running.  Waiting to be stopped.
I1001 17:50:57.480296       6 scheduler.go:302] New master master@10.0.4.124:5050 detected
I1001 17:50:57.480325       6 scheduler.go:362] No credentials were provided. Attempting to register scheduler without authentication.
I1001 17:50:57.480377       6 scheduler.go:859] Reregistering with master: master@10.0.4.124:5050
I1001 17:50:57.480472       6 scheduler.go:821] will retry registration in 1.636417748s if necessary
I1001 17:50:57.483355       6 scheduler.go:462] Framework registered with ID=20151001-152803-2080636938-5050-1237-0001
I1001 17:50:57.483614       6 scheduler.go:171] Framework Registered with Master  &MasterInfo{Id:*20151001-152803-2080636938-5050-1237,Ip:*2080636938,Port:*5050,Pid:*master@10.0.4.124:5050,Hostname:*ip-10-0-4-124.us-west-2.compute.internal,Version:*0.24.1,XXX_unrecognized:[58 57 10 40 105 112 45 49 48 45 48 45 52 45 49 50 52 46 117 115 45 119 101 115 116 45 50 46 99 111 109 112 117 116 101 46 105 110 116 101 114 110 97 108 18 10 49 48 46 48 46 52 46 49 50 52 24 186 39],}
W1001 17:50:57.490302       6 scheduler.go:189] Framework ID is already persisted for this cluster.
I1001 17:50:57.491262       6 scheduler.go:516] Trying to sync with master.
I1001 17:50:57.491293       6 state.go:70] Trying to get master state from http://ip-10-0-4-124.us-west-2.compute.internal:5050/state.json
I1001 17:50:57.494457       6 scheduler.go:308] Status update: task etcd-1443720518 ip-10-0-1-12.us-west-2.compute.internal 1025 1026 1027  is in state  TASK_RUNNING
I1001 17:50:59.117120       6 scheduler.go:847] skipping registration request: stopped=false, connected=true, authenticated=true
I1001 17:51:00.500498       6 scheduler.go:534] Scheduler synchronized with master.
I1001 17:51:00.500524       6 scheduler.go:482] Scheduler transitioning to Mutable state.
I1001 17:51:03.085920       6 offercache.go:70] We already have enough offers cached.
W1001 17:51:04.087771       6 scheduler.go:621] Prune attempting to deconfigure unknown etcd instance: 
I1001 17:51:04.087799       6 membership.go:235] Attempting to remove task  from the etcd cluster configuration.
I1001 17:51:11.089150       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:51:11.089223       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 1 seconds and retrying.
I1001 17:51:19.090559       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:51:19.090603       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 2 seconds and retrying.
I1001 17:51:28.091972       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:51:28.092018       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 4 seconds and retrying.
I1001 17:51:39.093293       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:51:39.093340       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:51:46.475208       6 scheduler.go:590] running instances: 1 desired: 3 offers: 3
I1001 17:51:46.475891       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:51:49.502802       6 healthcheck.go:93] Could not query cluster: 501: All the given peers are not reachable (failed to propose on members [http://ip-10-0-1-8.us-west-2.compute.internal:1026] twice [last error: Get http://ip-10-0-1-8.us-west-2.compute.internal:1026/v2/keys/?quorum=false&recursive=false&sorted=false: dial tcp: i/o timeout]) [0]
I1001 17:51:54.094718       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:51:54.094765       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:52:02.096248       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:52:03.096595       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
E1001 17:52:03.096639       6 scheduler.go:738] Failed health check, rescheduling launch attempt for later: could not contact endpoint
I1001 17:52:03.096656       6 scheduler.go:760] Skipping launch attempt for now.
W1001 17:52:13.097663       6 scheduler.go:621] Prune attempting to deconfigure unknown etcd instance: 
I1001 17:52:13.097690       6 membership.go:235] Attempting to remove task  from the etcd cluster configuration.
I1001 17:52:20.099024       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:52:20.099069       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 1 seconds and retrying.
I1001 17:52:28.100362       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:52:28.100408       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 2 seconds and retrying.
I1001 17:52:36.475471       6 scheduler.go:590] running instances: 1 desired: 3 offers: 3
I1001 17:52:37.101748       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:52:37.101796       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 4 seconds and retrying.
I1001 17:52:39.503907       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:52:40.504257       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
I1001 17:52:48.103078       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:52:48.103122       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:53:03.104438       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:53:03.104510       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:53:11.105880       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:53:12.106226       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
E1001 17:53:12.106273       6 scheduler.go:738] Failed health check, rescheduling launch attempt for later: could not contact endpoint
I1001 17:53:12.106291       6 scheduler.go:760] Skipping launch attempt for now.
W1001 17:53:22.107297       6 scheduler.go:621] Prune attempting to deconfigure unknown etcd instance: 
I1001 17:53:22.107325       6 membership.go:235] Attempting to remove task  from the etcd cluster configuration.
I1001 17:53:26.475753       6 scheduler.go:590] running instances: 1 desired: 3 offers: 3
I1001 17:53:29.108471       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:53:29.108514       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 1 seconds and retrying.
I1001 17:53:31.506169       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:53:32.506512       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
I1001 17:53:37.109813       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:53:37.109861       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 2 seconds and retrying.
I1001 17:53:46.111188       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:53:46.111234       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 4 seconds and retrying.
I1001 17:53:57.112537       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:53:57.112581       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:54:12.113826       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:54:12.113869       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:54:16.476047       6 scheduler.go:590] running instances: 1 desired: 3 offers: 3
I1001 17:54:20.115196       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:54:21.115544       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
E1001 17:54:21.115588       6 scheduler.go:738] Failed health check, rescheduling launch attempt for later: could not contact endpoint
I1001 17:54:21.115605       6 scheduler.go:760] Skipping launch attempt for now.
I1001 17:54:22.507445       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:54:23.507812       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
W1001 17:54:31.116643       6 scheduler.go:621] Prune attempting to deconfigure unknown etcd instance: 
I1001 17:54:31.116671       6 membership.go:235] Attempting to remove task  from the etcd cluster configuration.
I1001 17:54:38.118020       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:54:38.118067       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 1 seconds and retrying.
I1001 17:54:46.119374       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:54:46.119452       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 2 seconds and retrying.
I1001 17:54:55.120812       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:54:55.120859       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 4 seconds and retrying.
I1001 17:55:06.122146       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:55:06.122194       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:55:06.476328       6 scheduler.go:590] running instances: 1 desired: 3 offers: 3
I1001 17:55:13.508700       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:55:14.509042       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
I1001 17:55:21.123549       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:55:21.123593       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:55:29.124942       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:55:30.125300       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
E1001 17:55:30.125342       6 scheduler.go:738] Failed health check, rescheduling launch attempt for later: could not contact endpoint
I1001 17:55:30.125356       6 scheduler.go:760] Skipping launch attempt for now.
W1001 17:55:40.126280       6 scheduler.go:621] Prune attempting to deconfigure unknown etcd instance: 
I1001 17:55:40.126308       6 membership.go:235] Attempting to remove task  from the etcd cluster configuration.
I1001 17:55:47.127589       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:55:47.127634       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 1 seconds and retrying.
I1001 17:55:55.128938       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:55:55.128983       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 2 seconds and retrying.
I1001 17:55:56.476607       6 scheduler.go:590] running instances: 1 desired: 3 offers: 3
I1001 17:56:04.130297       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:56:04.130345       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 4 seconds and retrying.
I1001 17:56:04.509888       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:56:05.510292       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
I1001 17:56:15.131681       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:56:15.131734       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:56:30.133027       6 membership.go:284] RemoveInstance response: {"message":"Internal Server Error"}
W1001 17:56:30.133074       6 membership.go:312] Failed to retrieve list of configured members.  Backing off for 8 seconds and retrying.
I1001 17:56:38.134438       6 healthcheck.go:66] Leader stats response:{"message":"not current leader"}
E1001 17:56:39.134785       6 healthcheck.go:87] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
E1001 17:56:39.134827       6 scheduler.go:720] Cluster has been livelocked for longer than 240 seconds!
W1001 17:56:39.134842       6 scheduler.go:722] Initiating reseed...
I1001 17:56:39.134850       6 scheduler.go:760] Skipping launch attempt for now.
E1001 17:56:40.135253       6 reseed.go:68] Could not establish connection with cluster using endpoints %+vhttp://ip-10-0-1-12.us-west-2.compute.internal:1026
E1001 17:56:40.135294       6 scheduler.go:958] Failed to retrieve any candidates for reseeding! No recovery possible!
I1001 17:56:40.135308       6 scheduler.go:988] Aborting framework [&FrameworkID{Value:*20151001-152803-2080636938-5050-1237-0001,XXX_unrecognized:[],}]
I1001 17:56:40.135339       6 scheduler.go:936] Stopping the scheduler driver
I1001 17:56:40.135350       6 messenger.go:278] stopping messenger..
I1001 17:56:40.135363       6 http_transporter.go:461] stopping HTTP transport
I1001 17:56:40.135412       6 messenger.go:383] exiting decodeLoop, transport shutting down
I1001 17:56:40.135492       6 messenger.go:278] stopping messenger..