fleetctl journal --lines 500 couchbase_sidekick@1.service

gistfile1.txt

-- Logs begin at Tue 2015-03-24 08:37:40 UTC, end at Wed 2015-04-08 19:05:45 UTC. --
Mar 24 08:53:10 mm-database-3 systemd[1]: Starting couchbase_sidekick...
Mar 24 08:53:10 mm-database-3 docker[1038]: Error response from daemon: No such container: couchbase-sidekick
Mar 24 08:53:10 mm-database-3 docker[1038]: time="2015-03-24T08:53:10Z" level="fatal" msg="Error: failed to kill one or more containers"
Mar 24 08:53:10 mm-database-3 docker[1057]: Error response from daemon: No such container: couchbase-sidekick
Mar 24 08:53:10 mm-database-3 docker[1057]: time="2015-03-24T08:53:10Z" level="fatal" msg="Error: failed to remove one or more containers"
Mar 24 08:53:10 mm-database-3 docker[1066]: Pulling repository tleyden5iwx/couchbase-cluster-go
Mar 24 08:53:11 mm-database-3 docker[1066]: 40e863e58f4f: Pulling image (latest) from tleyden5iwx/couchbase-cluster-go
Mar 24 08:53:11 mm-database-3 docker[1066]: 40e863e58f4f: Pulling image (latest) from tleyden5iwx/couchbase-cluster-go, endpoint: https://registry-1.docker.io/v1/
Mar 24 08:53:12 mm-database-3 docker[1066]: 40e863e58f4f: Pulling dependent layers
Mar 24 08:53:12 mm-database-3 docker[1066]: 511136ea3c5a: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: f0dde87450ec: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 76b658ecb564: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 4faa69f72743: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 2103b00b3fdf: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: c47afe6fa540: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 00e269017528: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 66c0e6a9593a: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 7530871874b9: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 3f412317c9fb: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: d894f2b0206b: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 574a610a8b7a: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 15679ede727a: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 52c3857e12cb: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 40e863e58f4f: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: 40e863e58f4f: Download complete
Mar 24 08:53:12 mm-database-3 docker[1066]: Status: Image is up to date for tleyden5iwx/couchbase-cluster-go:latest
Mar 24 08:53:12 mm-database-3 systemd[1]: Started couchbase_sidekick.
Mar 24 08:53:12 mm-database-3 bash[1170]: 2015/03/24 08:53:12 Update-Wrapper: updating to latest code
Mar 24 08:53:12 mm-database-3 bash[1170]: github.com/tleyden/couchbase-cluster-go (download)
Mar 24 08:53:14 mm-database-3 bash[1170]: github.com/coreos/fleet (download)
Mar 24 08:53:14 mm-database-3 bash[1170]: github.com/coreos/go-systemd (download)
Mar 24 08:53:15 mm-database-3 bash[1170]: github.com/tleyden/go-etcd (download)
Mar 24 08:53:16 mm-database-3 bash[1170]: github.com/docopt/docopt-go (download)
Mar 24 08:53:17 mm-database-3 bash[1170]: github.com/tleyden/couchbase-cluster-go
Mar 24 08:53:17 mm-database-3 bash[1170]: github.com/tleyden/couchbase-cluster-go/cmd/couchbase-cluster
Mar 24 08:53:18 mm-database-3 bash[1170]: github.com/tleyden/couchbase-cluster-go/cmd/couchbase-fleet
Mar 24 08:53:20 mm-database-3 bash[1170]: github.com/tleyden/couchbase-cluster-go/cmd/sync-gw-cluster
Mar 24 08:53:21 mm-database-3 bash[1170]: github.com/tleyden/couchbase-cluster-go/cmd/sync-gw-config
Mar 24 08:53:23 mm-database-3 bash[1170]: 2015/03/24 08:53:23 Connect to etcd on localhost
Mar 24 08:53:23 mm-database-3 bash[1170]: 2015/03/24 08:53:23 BecomeFirstClusterNode()
Mar 24 08:53:23 mm-database-3 bash[1170]: 2015/03/24 08:53:23 Key /couchbase.com/couchbase-node-state already exists
Mar 24 08:53:23 mm-database-3 bash[1170]: 2015/03/24 08:53:23 Got error Get http://10.131.173.220:8091/pools: dial tcp 10.131.173.220:8091: connection refused trying to fetch details.  Assume that the cluster is not up yet, sleeping and will retry
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 Version: 3.0.1-1444-rel-community
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 JoinExistingCluster() called
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 Calling FindLiveNode()
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 Couchbase node ip: 10.131.173.218
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 Verifying REST service at http://10.131.173.218:8091/ to be up
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 liveNodeIp: 10.131.173.218
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 JoinLiveNode() called with 10.131.173.218
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 GetClusterNodes() called with: 10.131.173.218
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 No cluster node found for 10.131.173.220.  Not retrying
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 WaitUntilInClusterAndHealthy() returned error: Unable to find node with hostname 10.131.173.220 in [map[mcdMemoryAllocated:798 recoveryType:none otpNode:ns_1@10.131.173.218 memoryFree:5.59140864e+08 memoryTotal:1.046544384e+09 otpCookie:rhlnzolcfpswjejf status:healthy clusterCompatibility:196608 ports:map[proxy:11211 direct:11210] uptime:250 couchApiBase:http://10.131.173.218:8092/ clusterMembership:active hostname:10.131.173.218:8091 os:x86_64-unknown-linux-gnu mcdMemoryReserved:798 interestingStats:map[curr_items:0 ep_bg_fetched:0 ops:0 cmd_get:0 couch_docs_actual_disk_size:1.2850336e+07 couch_docs_data_size:1.2576768e+07 get_hits:0 mem_used:3.4641056e+07 vb_replica_curr_items:0 couch_views_actual_disk_size:0 couch_views_data_size:0 curr_items_tot:0] thisNode:true version:3.0.1-1444-rel-community systemStats:map[swap_total:0 swap_used:0 mem_total:1.046544384e+09 mem_free:5.59140864e+08 cpu_utilization_rate:81]] map[interestingStats:map[couch_docs_data_size:1.2337176e+07 couch_views_actual_disk_size:0 get_hits:0 vb_replica_curr_items:0 cmd_get:0 couch_docs_actual_disk_size:1.1809269e+07 curr_items_tot:0 ep_bg_fetched:0 mem_used:3.3570928e+07 ops:0 couch_views_data_size:0 curr_items:0] os:x86_64-unknown-linux-gnu clusterMembership:active status:healthy hostname:10.131.173.219:8091 version:3.0.1-1444-rel-community memoryFree:5.7817088e+08 mcdMemoryReserved:798 mcdMemoryAllocated:798 otpNode:ns_1@10.131.173.219 clusterCompatibility:196608 systemStats:map[cpu_utilization_rate:88.11881188118812 swap_total:0 swap_used:0 mem_total:1.046544384e+09 mem_free:5.7817088e+08] memoryTotal:1.046544384e+09 couchApiBase:http://10.131.173.219:8092/ ports:map[proxy:11211 direct:11210] uptime:78 otpCookie:rhlnzolcfpswjejf recoveryType:none]].  Call AddNodeRetry()
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 AddNode()
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 AddNode posting to http://10.131.173.218:8091/controller/addNode with data: hostname=10.131.173.220&password=HIDDEN&user=HIDDEN
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 Using username/password pulled from etcd
Mar 24 08:53:33 mm-database-3 bash[1170]: 2015/03/24 08:53:33 POST to http://10.131.173.218:8091/controller/addNode
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 WaitUntilNoRebalanceRunning()
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 TriggerRebalance()
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 GetClusterNodes() called with: 10.131.173.218
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 OtpNodeList, otpNode: ns_1@10.131.173.218
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 OtpNodeList, otpNode: ns_1@10.131.173.219
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 OtpNodeList, otpNode: ns_1@10.131.173.220
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 TriggerRebalance otpNodeList: [ns_1@10.131.173.218 ns_1@10.131.173.219 ns_1@10.131.173.220]
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 TriggerRebalance encoded form value: knownNodes=ns_1%4010.131.173.218%2Cns_1%4010.131.173.219%2Cns_1%4010.131.173.220
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 Using username/password pulled from etcd
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 POST to http://10.131.173.218:8091/controller/rebalance
Mar 24 08:53:38 mm-database-3 bash[1170]: 2015/03/24 08:53:38 EventLoop()
Mar 25 06:19:10 mm-database-3 bash[1170]: 2015/03/25 06:18:58 Error updating /couchbase.com/couchbase-node-state dir in etc with new TTL. Ignoring error, but this could cause problems
Mar 25 06:19:10 mm-database-3 bash[1170]: 2015/03/25 06:19:02 Error publishing node state to etcd: 501: All the given peers are not reachable (Tried to connect to each peer twice and failed) [0]. Ignoring error, but other nodes won't be able to jointhis node until this issue is resolved.
Mar 25 06:19:10 mm-database-3 bash[1170]: 2015/03/25 06:19:10 Successfully node state to etcd: <nil>. The previous error seems to have fixed itself!
Mar 26 03:28:30 mm-database-3 systemd[1]: Stopping couchbase_sidekick...
Mar 26 03:28:31 mm-database-3 docker[30338]: couchbase-sidekick
Mar 26 03:28:31 mm-database-3 systemd[1]: couchbase_sidekick@1.service: main process exited, code=exited, status=2/INVALIDARGUMENT
Mar 26 03:28:31 mm-database-3 systemd[1]: Stopped couchbase_sidekick.
Mar 26 03:28:31 mm-database-3 systemd[1]: Unit couchbase_sidekick@1.service entered failed state.
Mar 26 03:28:31 mm-database-3 systemd[1]: couchbase_sidekick@1.service failed.
Mar 26 23:42:49 mm-database-3 systemd[1]: Stopped couchbase_sidekick@1.service.
Mar 26 23:43:04 mm-database-3 systemd[1]: Stopped couchbase_sidekick@1.service.
Mar 26 23:43:14 mm-database-3 systemd[1]: Starting couchbase_sidekick...
Mar 26 23:43:15 mm-database-3 docker[18772]: couchbase-sidekick
Mar 26 23:43:15 mm-database-3 docker[18777]: couchbase-sidekick
Mar 26 23:43:15 mm-database-3 docker[18786]: Pulling repository tleyden5iwx/couchbase-cluster-go
Mar 26 23:43:16 mm-database-3 docker[18786]: 40e863e58f4f: Pulling image (latest) from tleyden5iwx/couchbase-cluster-go
Mar 26 23:43:16 mm-database-3 docker[18786]: 40e863e58f4f: Pulling image (latest) from tleyden5iwx/couchbase-cluster-go, endpoint: https://registry-1.docker.io/v1/
Mar 26 23:43:17 mm-database-3 docker[18786]: 40e863e58f4f: Pulling dependent layers
Mar 26 23:43:17 mm-database-3 docker[18786]: 511136ea3c5a: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: f0dde87450ec: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 76b658ecb564: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 4faa69f72743: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 2103b00b3fdf: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: c47afe6fa540: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 00e269017528: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 66c0e6a9593a: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 7530871874b9: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 3f412317c9fb: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: d894f2b0206b: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 574a610a8b7a: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 15679ede727a: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 52c3857e12cb: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 40e863e58f4f: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: 40e863e58f4f: Download complete
Mar 26 23:43:17 mm-database-3 docker[18786]: Status: Image is up to date for tleyden5iwx/couchbase-cluster-go:latest
Mar 26 23:43:17 mm-database-3 systemd[1]: Started couchbase_sidekick.
Mar 26 23:43:17 mm-database-3 bash[18795]: 2015/03/26 23:43:17 Update-Wrapper: updating to latest code
Mar 26 23:43:17 mm-database-3 bash[18795]: github.com/tleyden/couchbase-cluster-go (download)
Mar 26 23:43:19 mm-database-3 bash[18795]: github.com/coreos/fleet (download)
Mar 26 23:43:20 mm-database-3 bash[18795]: github.com/coreos/go-systemd (download)
Mar 26 23:43:21 mm-database-3 bash[18795]: github.com/tleyden/go-etcd (download)
Mar 26 23:43:21 mm-database-3 bash[18795]: github.com/docopt/docopt-go (download)
Mar 26 23:43:22 mm-database-3 bash[18795]: github.com/tleyden/couchbase-cluster-go
Mar 26 23:43:23 mm-database-3 bash[18795]: github.com/tleyden/couchbase-cluster-go/cmd/couchbase-cluster
Mar 26 23:43:24 mm-database-3 bash[18795]: github.com/tleyden/couchbase-cluster-go/cmd/couchbase-fleet
Mar 26 23:43:25 mm-database-3 bash[18795]: github.com/tleyden/couchbase-cluster-go/cmd/sync-gw-cluster
Mar 26 23:43:25 mm-database-3 bash[18795]: github.com/tleyden/couchbase-cluster-go/cmd/sync-gw-config
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Connect to etcd on localhost
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 BecomeFirstClusterNode()
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Key /couchbase.com/couchbase-node-state already exists
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Version: 3.0.1-1444-rel-community
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 JoinExistingCluster() called
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Calling FindLiveNode()
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Couchbase node ip: 10.131.173.218
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Verifying REST service at http://10.131.173.218:8091/ to be up
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 liveNodeIp: 10.131.173.218
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 JoinLiveNode() called with 10.131.173.218
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 GetClusterNodes() called with: 10.131.173.218
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 No cluster node found for 10.131.173.220.  Not retrying
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 WaitUntilInClusterAndHealthy() returned error: Unable to find node with hostname 10.131.173.220 in [map[uptime:226445 memoryTotal:1.046544384e+09 mcdMemoryReserved:798 mcdMemoryAllocated:798 couchApiBase:http://10.131.173.218:8092/ clusterMembership:active status:healthy thisNode:true clusterCompatibility:196608 interestingStats:map[cmd_get:0 couch_docs_data_size:2.5276506e+07 couch_views_actual_disk_size:51992 curr_items:0 curr_items_tot:2 get_hits:0 vb_replica_curr_items:2 couch_docs_actual_disk_size:3.0681993e+07 couch_views_data_size:26095 ep_bg_fetched:0 mem_used:6.8847512e+07 ops:0] memoryFree:3.05942528e+08 recoveryType:none otpNode:ns_1@10.131.173.218 os:x86_64-unknown-linux-gnu systemStats:map[cpu_utilization_rate:8 swap_total:0 swap_used:0 mem_total:1.046544384e+09 mem_free:3.05942528e+08] otpCookie:rhlnzolcfpswjejf ports:map[proxy:11211 direct:11210] hostname:10.131.173.218:8091 version:3.0.1-1444-rel-community] map[ports:map[proxy:11211 direct:11210] interestingStats:map[vb_replica_curr_items:0 cmd_get:0 couch_docs_actual_disk_size:3.3712355e+07 couch_docs_data_size:2.8213338e+07 curr_items:2 ep_bg_fetched:0 ops:0 couch_views_actual_disk_size:52125 couch_views_data_size:26051 curr_items_tot:2 get_hits:0 mem_used:6.93542e+07] status:healthy clusterCompatibility:196608 os:x86_64-unknown-linux-gnu uptime:211540 mcdMemoryReserved:798 recoveryType:none hostname:10.131.173.219:8091 version:3.0.1-1444-rel-community systemStats:map[mem_free:3.7588992e+08 cpu_utilization_rate:6.060606060606061 swap_total:0 swap_used:0 mem_total:1.046544384e+09] memoryFree:3.7588992e+08 clusterMembership:active otpNode:ns_1@10.131.173.219 memoryTotal:1.046544384e+09 mcdMemoryAllocated:798 couchApiBase:http://10.131.173.219:8092/ otpCookie:rhlnzolcfpswjejf]].  Call AddNodeRetry()
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 AddNode()
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 AddNode posting to http://10.131.173.218:8091/controller/addNode with data: hostname=10.131.173.220&password=HIDDEN&user=HIDDEN
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 Using username/password pulled from etcd
Mar 26 23:43:26 mm-database-3 bash[18795]: 2015/03/26 23:43:26 POST to http://10.131.173.218:8091/controller/addNode
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 WaitUntilNoRebalanceRunning()
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 TriggerRebalance()
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 GetClusterNodes() called with: 10.131.173.218
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 OtpNodeList, otpNode: ns_1@10.131.173.218
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 OtpNodeList, otpNode: ns_1@10.131.173.219
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 OtpNodeList, otpNode: ns_1@10.131.173.220
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 TriggerRebalance otpNodeList: [ns_1@10.131.173.218 ns_1@10.131.173.219 ns_1@10.131.173.220]
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 TriggerRebalance encoded form value: knownNodes=ns_1%4010.131.173.218%2Cns_1%4010.131.173.219%2Cns_1%4010.131.173.220
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 Using username/password pulled from etcd
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 POST to http://10.131.173.218:8091/controller/rebalance
Mar 26 23:43:34 mm-database-3 bash[18795]: 2015/03/26 23:43:34 EventLoop()
Mar 27 00:06:27 mm-database-3 bash[18795]: 2015/03/27 00:06:27 Error updating /couchbase.com/couchbase-node-state dir in etc with new TTL. Ignoring error, but this could cause problems
Mar 27 00:10:30 mm-database-3 bash[18795]: 2015/03/27 00:10:30 Error updating /couchbase.com/couchbase-node-state dir in etc with new TTL. Ignoring error, but this could cause problems
Mar 27 00:10:30 mm-database-3 bash[18795]: 2015/03/27 00:10:30 Error publishing node state to etcd: 501: All the given peers are not reachable (Tried to connect to each peer twice and failed) [0]. Ignoring error, but other nodes won't be able to jointhis node until this issue is resolved.
Mar 27 00:10:35 mm-database-3 bash[18795]: 2015/03/27 00:10:35 Successfully node state to etcd: <nil>. The previous error seems to have fixed itself!
Mar 27 00:10:55 mm-database-3 bash[18795]: 2015/03/27 00:10:55 Error updating /couchbase.com/couchbase-node-state dir in etc with new TTL. Ignoring error, but this could cause problems
Mar 27 00:10:55 mm-database-3 bash[18795]: 2015/03/27 00:10:55 Error publishing node state to etcd: 501: All the given peers are not reachable (Tried to connect to each peer twice and failed) [0]. Ignoring error, but other nodes won't be able to jointhis node until this issue is resolved.
Mar 27 00:11:00 mm-database-3 bash[18795]: 2015/03/27 00:11:00 Successfully node state to etcd: <nil>. The previous error seems to have fixed itself!
Mar 29 03:08:30 mm-database-3 systemd[1]: Stopping couchbase_sidekick...
Mar 29 03:08:31 mm-database-3 docker[10450]: couchbase-sidekick
Mar 29 03:08:31 mm-database-3 systemd[1]: couchbase_sidekick@1.service: main process exited, code=exited, status=2/INVALIDARGUMENT
Mar 29 03:08:31 mm-database-3 systemd[1]: Stopped couchbase_sidekick.
Mar 29 03:08:31 mm-database-3 systemd[1]: Unit couchbase_sidekick@1.service entered failed state.
Mar 29 03:08:31 mm-database-3 systemd[1]: couchbase_sidekick@1.service failed.
Apr 06 05:07:09 mm-database-3 systemd[1]: Stopped couchbase_sidekick@1.service.
Apr 06 05:10:18 mm-database-3 systemd[1]: Starting couchbase_sidekick...
Apr 06 05:10:18 mm-database-3 docker[27200]: couchbase-sidekick
Apr 06 05:10:18 mm-database-3 docker[27212]: couchbase-sidekick
Apr 06 05:10:18 mm-database-3 docker[27226]: Pulling repository tleyden5iwx/couchbase-cluster-go
Apr 06 05:10:19 mm-database-3 docker[27226]: 40e863e58f4f: Pulling image (latest) from tleyden5iwx/couchbase-cluster-go
Apr 06 05:10:19 mm-database-3 docker[27226]: 40e863e58f4f: Pulling image (latest) from tleyden5iwx/couchbase-cluster-go, endpoint: https://registry-1.docker.io/v1/
Apr 06 05:10:20 mm-database-3 docker[27226]: 40e863e58f4f: Pulling dependent layers
Apr 06 05:10:20 mm-database-3 docker[27226]: 511136ea3c5a: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: f0dde87450ec: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 76b658ecb564: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 4faa69f72743: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 2103b00b3fdf: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: c47afe6fa540: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 00e269017528: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 66c0e6a9593a: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 7530871874b9: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 3f412317c9fb: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: d894f2b0206b: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 574a610a8b7a: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 15679ede727a: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 52c3857e12cb: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 40e863e58f4f: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: 40e863e58f4f: Download complete
Apr 06 05:10:20 mm-database-3 docker[27226]: Status: Image is up to date for tleyden5iwx/couchbase-cluster-go:latest
Apr 06 05:10:20 mm-database-3 systemd[1]: Started couchbase_sidekick.
Apr 06 05:10:20 mm-database-3 bash[27280]: 2015/04/06 05:10:20 Update-Wrapper: updating to latest code
Apr 06 05:10:21 mm-database-3 bash[27280]: github.com/tleyden/couchbase-cluster-go (download)
Apr 06 05:10:23 mm-database-3 bash[27280]: github.com/coreos/fleet (download)
Apr 06 05:10:26 mm-database-3 bash[27280]: github.com/coreos/go-systemd (download)
Apr 06 05:10:27 mm-database-3 bash[27280]: github.com/tleyden/go-etcd (download)
Apr 06 05:10:27 mm-database-3 bash[27280]: github.com/docopt/docopt-go (download)
Apr 06 05:10:29 mm-database-3 bash[27280]: github.com/coreos/fleet/unit
Apr 06 05:10:29 mm-database-3 bash[27280]: github.com/coreos/fleet/job
Apr 06 05:10:29 mm-database-3 bash[27280]: github.com/coreos/fleet/machine
Apr 06 05:10:29 mm-database-3 bash[27280]: github.com/coreos/fleet/schema
Apr 06 05:10:30 mm-database-3 bash[27280]: github.com/tleyden/couchbase-cluster-go
Apr 06 05:10:32 mm-database-3 bash[27280]: github.com/tleyden/couchbase-cluster-go/cmd/couchbase-cluster
Apr 06 05:10:35 mm-database-3 bash[27280]: github.com/tleyden/couchbase-cluster-go/cmd/couchbase-fleet
Apr 06 05:10:38 mm-database-3 bash[27280]: github.com/tleyden/couchbase-cluster-go/cmd/sync-gw-cluster
Apr 06 05:10:40 mm-database-3 bash[27280]: github.com/tleyden/couchbase-cluster-go/cmd/sync-gw-config
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Connect to etcd on localhost
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 BecomeFirstClusterNode()
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Created key: /couchbase.com/couchbase-node-state
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Version: 3.0.1-1444-rel-community
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 We became first cluster node, init cluster and bucket
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 ClusterInit()
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 IsClusterPasswordSet()
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 ClusterSetPassword()
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Using default username/password
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 POST to http://10.131.173.220:8091/settings/web
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Total RAM (MB) on machine: 998
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Attempting to set cluster ram to: 748 MB
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Using username/password pulled from etcd
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 POST to http://10.131.173.220:8091/pools/default
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 CreateDefaultBucket()
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 HasDefaultBucket()
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 Using username/password pulled from etcd
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 POST to http://10.131.173.220:8091/pools/default/buckets
Apr 06 05:10:41 mm-database-3 bash[27280]: 2015/04/06 05:10:41 EventLoop()
Apr 07 01:35:44 mm-database-3 bash[27280]: 2015/04/07 01:35:44 Error updating /couchbase.com/couchbase-node-state dir in etc with new TTL. Ignoring error, but this could cause problems
Apr 07 01:35:44 mm-database-3 systemd[1]: Stopping couchbase_sidekick...
Apr 07 01:35:47 mm-database-3 docker[12839]: couchbase-sidekick
Apr 07 01:35:47 mm-database-3 systemd[1]: couchbase_sidekick@1.service: main process exited, code=exited, status=2/INVALIDARGUMENT
Apr 07 01:35:47 mm-database-3 systemd[1]: Stopped couchbase_sidekick.
Apr 07 01:35:47 mm-database-3 systemd[1]: Unit couchbase_sidekick@1.service entered failed state.
Apr 07 01:35:47 mm-database-3 systemd[1]: couchbase_sidekick@1.service failed.