Skip to content

Instantly share code, notes, and snippets.

@akvadrako

akvadrako/gist:2617a080b267e854feffd5f9d79b9ba1

Created July 27, 2016 12:22
Show Gist options
  • Save akvadrako/2617a080b267e854feffd5f9d79b9ba1 to your computer and use it in GitHub Desktop.
Save akvadrako/2617a080b267e854feffd5f9d79b9ba1 to your computer and use it in GitHub Desktop.
Download ZIP
ecs docker startup issues
Raw
gistfile1.txt
[root@ip-10-11-12-73 ec2-user]# docker info
Containers: 1
Running: 0
Paused: 0
Stopped: 1
Images: 1
Server Version: 1.11.2
Storage Driver: devicemapper
Pool Name: docker-docker--pool
Pool Blocksize: 524.3 kB
Base Device Size: 10.74 GB
Backing Filesystem: ext4
Data file:
Metadata file:
Data Space Used: 337.6 MB
Data Space Total: 23.35 GB
Data Space Available: 23.02 GB
Metadata Space Used: 172 kB
Metadata Space Total: 25.17 MB
Metadata Space Available: 24.99 MB
Udev Sync Supported: true
Deferred Removal Enabled: true
Deferred Deletion Enabled: true
Deferred Deleted Device Count: 0
Library Version: 1.02.93-RHEL7 (2015-01-28)
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: null host bridge
Kernel Version: 4.4.14-24.50.amzn1.x86_64
Operating System: Amazon Linux AMI 2016.03
OSType: linux
Architecture: x86_64
CPUs: 1
Total Memory: 1.956 GiB
Name: ip-10-11-12-73
ID: 5INL:7IYL:AO4J:D5VT:FZCL:LVKZ:BXPI:4ONX:5S7P:EDUH:HKT7:5WQL
Docker Root Dir: /var/lib/docker
Debug mode (client): false
Debug mode (server): false
Registry: https://index.docker.io/v1/
[root@ip-10-11-12-73 ec2-user]# curl localhost:51678/v1/metadata
curl: (7) Failed to connect to localhost port 51678: Connection refused
[root@ip-10-11-12-73 ec2-user]# cat /var/log/ecs/ecs-init.log.2016-07-27-12
2016-07-27T12:13:24Z [INFO] pre-start
2016-07-27T12:13:27Z [INFO] start
2016-07-27T12:13:27Z [INFO] No existing agent container to remove.
2016-07-27T12:13:27Z [INFO] Starting Amazon EC2 Container Service Agent
2016-07-27T12:13:30Z [ERROR] could not start Agent: API error (500): rpc error: code = 2 desc = "oci runtime error: rootfs (\"/var/lib/docker/devicemapper/mnt/4cfe020c0f8658ca94528b36538a0a47098b915059a6d06932b0b166d07bfcf9/rootfs\") does not exist"
2016-07-27T12:13:30Z [INFO] post-stop
2016-07-27T12:13:30Z [INFO] Cleaning up the credentials endpoint setup for Amazon EC2 Container Service Agent
[root@ip-10-11-12-73 ec2-user]# cat /var/log/docker
\nWed Jul 27 12:13:09 UTC 2016\n
time="2016-07-27T12:13:09.767028951Z" level=info msg="New containerd process, pid: 2633\n"
time="2016-07-27T12:13:15.398023166Z" level=info msg="devmapper: Creating filesystem ext4 on device docker-202:1-263358-base"
time="2016-07-27T12:13:19.268203454Z" level=info msg="devmapper: Successfully created filesystem ext4 on device docker-202:1-263358-base"
time="2016-07-27T12:13:19.315013207Z" level=info msg="Graph migration to content-addressability took 0.00 seconds"
time="2016-07-27T12:13:21.498533895Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
time="2016-07-27T12:13:21.577816646Z" level=warning msg="Your kernel does not support cgroup blkio weight"
time="2016-07-27T12:13:21.577849144Z" level=warning msg="Your kernel does not support cgroup blkio weight_device"
time="2016-07-27T12:13:21.577945949Z" level=warning msg="mountpoint for pids not found"
time="2016-07-27T12:13:21.578591554Z" level=info msg="Loading containers: start."
time="2016-07-27T12:13:21.578631631Z" level=info msg="Loading containers: done."
time="2016-07-27T12:13:21.578641079Z" level=info msg="Daemon has completed initialization"
time="2016-07-27T12:13:21.578651441Z" level=info msg="Docker daemon" commit="b9f10c9/1.11.2" graphdriver=devicemapper version=1.11.2
\nWed Jul 27 12:13:21 UTC 2016\n
time="2016-07-27T12:13:21.756144777Z" level=info msg="previous instance of containerd still alive (2633)"
time="2016-07-27T12:13:21.831218628Z" level=info msg="Graph migration to content-addressability took 0.00 seconds"
time="2016-07-27T12:13:21.873861149Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
time="2016-07-27T12:13:21.899886244Z" level=warning msg="Your kernel does not support cgroup blkio weight"
time="2016-07-27T12:13:21.899915574Z" level=warning msg="Your kernel does not support cgroup blkio weight_device"
time="2016-07-27T12:13:21.900020462Z" level=warning msg="mountpoint for pids not found"
time="2016-07-27T12:13:21.900193945Z" level=info msg="Loading containers: start."
time="2016-07-27T12:13:21.900235707Z" level=info msg="Loading containers: done."
time="2016-07-27T12:13:21.900252054Z" level=info msg="Daemon has completed initialization"
time="2016-07-27T12:13:21.900262835Z" level=info msg="Docker daemon" commit="b9f10c9/1.11.2" graphdriver=devicemapper version=1.11.2
time="2016-07-27T12:13:21.920747346Z" level=info msg="API listen on /var/run/docker.sock"
time="2016-07-27T12:13:30Z" level=error msg="containerd: start container" error="oci runtime error: rootfs (\"/var/lib/docker/devicemapper/mnt/4cfe020c0f8658ca94528b36538a0a47098b915059a6d06932b0b166d07bfcf9/rootfs\") does not exist" id=dafda0116056566809dc8a764a695a1e5892c5252ec6c9e43039bb085a6b1550
time="2016-07-27T12:13:30.757100986Z" level=error msg="Handler for POST /v1.15/containers/dafda0116056566809dc8a764a695a1e5892c5252ec6c9e43039bb085a6b1550/start returned error: rpc error: code = 2 desc = \"oci runtime error: rootfs (\\\"/var/lib/docker/devicemapper/mnt/4cfe020c0f8658ca94528b36538a0a47098b915059a6d06932b0b166d07bfcf9/rootfs\\\") does not exist\""
[root@ip-10-11-12-73 ec2-user]# cat /etc/sysconfig/docker
# The max number of open files for the daemon itself, and all
# running containers. The default value of 1048576 mirrors the value
# used by the systemd service unit.
DAEMON_MAXFILES=1048576
# Additional startup options for the Docker daemon, for example:
# OPTIONS="--ip-forward=true --iptables=true"
# By default we limit the number of open files per container
OPTIONS="--default-ulimit nofile=1024:4096"
[root@ip-10-11-12-73 ec2-user]# cat /etc/sysconfig/docker-storage
DOCKER_STORAGE_OPTIONS="--storage-driver devicemapper --storage-opt dm.thinpooldev=/dev/mapper/docker-docker--pool --storage-opt dm.use_deferred_removal=true --storage-opt dm.use_deferred_deletion=true --storage-opt dm.fs=ext4"
[root@ip-10-11-12-73 ec2-user]# cat /var/log/cloud-init-output.log
Cloud-init v. 0.7.6 running 'init-local' at Wed, 27 Jul 2016 12:12:23 +0000. Up 40.95 seconds.
Cloud-init v. 0.7.6 running 'init' at Wed, 27 Jul 2016 12:12:24 +0000. Up 41.29 seconds.
ci-info: ++++++++++++++++++++++++Net device info++++++++++++++++++++++++
ci-info: Device Up Address Mask Hw-Address
ci-info: lo True 127.0.0.1 255.0.0.0 .
ci-info: eth0 True 10.11.12.73 255.255.255.192 0a:37:f2:39:fa:d5
ci-info: +++++++++++++++++++++++++++++++Route info+++++++++++++++++++++++++++++++
ci-info: Route Destination Gateway Genmask Interface Flags
ci-info: 0 0.0.0.0 10.11.12.65 0.0.0.0 eth0 UG
ci-info: 1 10.11.12.64 0.0.0.0 255.255.255.192 eth0 U
ci-info: 2 169.254.169.254 0.0.0.0 255.255.255.255 eth0 UH
INFO: Volume group backing root filesystem could not be determined
File descriptor 6 (/var/log/cloud-init.log) leaked on vgs invocation. Parent PID 2194: /bin/bash
Checking that no-one is using this disk right now ...
OK
sfdisk: /dev/xvdcz: unrecognized partition table type
sfdisk: No partitions found
Disk /dev/xvdcz: 2871 cylinders, 255 heads, 63 sectors/track
Old situation:
New situation:
Units: sectors of 512 bytes, counting from 0
Device Boot Start End #sectors Id System
/dev/xvdcz1 2048 46137343 46135296 8e Linux LVM
/dev/xvdcz2 0 - 0 0 Empty
/dev/xvdcz3 0 - 0 0 Empty
/dev/xvdcz4 0 - 0 0 Empty
Warning: partition 1 does not end at a cylinder boundary
Warning: no primary partition is marked bootable (active)
This does not matter for LILO, but the DOS MBR will not boot this disk.
Successfully wrote the new partition table
Re-reading the partition table ...
If you created or changed a DOS partition, /dev/foo7, say, then use dd(1)
to zero the first 512 bytes: dd if=/dev/zero of=/dev/foo7 bs=512 count=1
(See fdisk(8).)
File descriptor 6 (/var/log/cloud-init.log) leaked on pvcreate invocation. Parent PID 2194: /bin/bash
Physical volume "/dev/xvdcz1" successfully created
File descriptor 6 (/var/log/cloud-init.log) leaked on vgcreate invocation. Parent PID 2194: /bin/bash
Volume group "docker" successfully created
File descriptor 6 (/var/log/cloud-init.log) leaked on lvs invocation. Parent PID 2239: /bin/bash
File descriptor 6 (/var/log/cloud-init.log) leaked on vgs invocation. Parent PID 2194: /bin/bash
File descriptor 6 (/var/log/cloud-init.log) leaked on lvcreate invocation. Parent PID 2194: /bin/bash
Rounding up size to full physical extent 24.00 MiB
Logical volume "docker-poolmeta" created.
File descriptor 6 (/var/log/cloud-init.log) leaked on vgs invocation. Parent PID 2194: /bin/bash
File descriptor 6 (/var/log/cloud-init.log) leaked on vgs invocation. Parent PID 2262: /bin/bash
File descriptor 6 (/var/log/cloud-init.log) leaked on lvcreate invocation. Parent PID 2194: /bin/bash
Logical volume "docker-pool" created.
File descriptor 6 (/var/log/cloud-init.log) leaked on lvconvert invocation. Parent PID 2194: /bin/bash
WARNING: Converting logical volume docker/docker-pool and docker/docker-poolmeta to pool's data and metadata volumes.
THIS WILL DESTROY CONTENT OF LOGICAL VOLUME (filesystem etc.)
Converted docker/docker-pool to thin pool.
File descriptor 6 (/var/log/cloud-init.log) leaked on lvchange invocation. Parent PID 2194: /bin/bash
Logical volume "docker-pool" changed.
File descriptor 6 (/var/log/cloud-init.log) leaked on lvs invocation. Parent PID 2348: /bin/bash
File descriptor 6 (/var/log/cloud-init.log) leaked on lvm invocation. Parent PID 2389: /bin/bash
File descriptor 6 (/var/log/cloud-init.log) leaked on lvchange invocation. Parent PID 2194: /bin/bash
Logical volume "docker-pool" changed.
Generating public/private rsa key pair.
Your identification has been saved in /etc/ssh/ssh_host_rsa_key.
Your public key has been saved in /etc/ssh/ssh_host_rsa_key.pub.
The key fingerprint is:
3a:63:13:c2:e6:2d:75:29:ef:07:54:86:f6:26:30:de root@ip-10-11-12-73
The key's randomart image is:
+--[ RSA 2048]----+
| . |
| o o o |
| . = + |
| . . E.o |
| + +.So |
| o + *. |
| o B .. |
| o = . |
| .. |
+-----------------+
Generating public/private dsa key pair.
Your identification has been saved in /etc/ssh/ssh_host_dsa_key.
Your public key has been saved in /etc/ssh/ssh_host_dsa_key.pub.
The key fingerprint is:
9f:fb:60:5e:44:e5:95:a8:35:a5:b7:3a:ec:18:e6:86 root@ip-10-11-12-73
The key's randomart image is:
+--[ DSA 1024]----+
| +.o|
| *.o |
| +.o. |
| o . .|
| S . . |
| . o. . |
| *o.+ |
| Eo=+ . |
| ++.. |
+-----------------+
Generating public/private ecdsa key pair.
Your identification has been saved in /etc/ssh/ssh_host_ecdsa_key.
Your public key has been saved in /etc/ssh/ssh_host_ecdsa_key.pub.
The key fingerprint is:
ac:05:5c:c0:7a:85:f7:80:71:98:5a:bd:08:2b:c3:b3 root@ip-10-11-12-73
The key's randomart image is:
+--[ECDSA 256]---+
| .oOo |
| ..Bo= |
| . *o+ + |
| = + oo. . |
| = . S |
| E o |
| . |
| |
| |
+-----------------+
Cloud-init v. 0.7.6 running 'modules:config' at Wed, 27 Jul 2016 12:12:40 +0000. Up 57.53 seconds.
Loaded plugins: priorities, update-motd, upgrade-helper
Metadata Cache Created
Loaded plugins: priorities, update-motd, upgrade-helper
1 package(s) needed (+0 related) for security, out of 6 available
Resolving Dependencies
--> Running transaction check
---> Package libxml2.x86_64 0:2.9.1-6.2.50.amzn1 will be updated
---> Package libxml2.x86_64 0:2.9.1-6.3.49.amzn1 will be an update
--> Finished Dependency Resolution
Dependencies Resolved
================================================================================
Package Arch Version Repository Size
================================================================================
Updating:
libxml2 x86_64 2.9.1-6.3.49.amzn1 amzn-updates 723 k
Transaction Summary
================================================================================
Upgrade 1 Package
Total download size: 723 k
Downloading packages:
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Updating : libxml2-2.9.1-6.3.49.amzn1.x86_64 1/2
Cleanup : libxml2-2.9.1-6.2.50.amzn1.x86_64 2/2
Verifying : libxml2-2.9.1-6.3.49.amzn1.x86_64 1/2
Verifying : libxml2-2.9.1-6.2.50.amzn1.x86_64 2/2
Updated:
libxml2.x86_64 0:2.9.1-6.3.49.amzn1
Complete!
Cloud-init v. 0.7.6 running 'modules:final' at Wed, 27 Jul 2016 12:13:10 +0000. Up 86.81 seconds.
+ yum install -y nfs-utils
Loaded plugins: priorities, update-motd, upgrade-helper
Resolving Dependencies
--> Running transaction check
---> Package nfs-utils.x86_64 1:1.3.0-0.21.amzn1 will be installed
--> Processing Dependency: libtirpc >= 0.2.3-1 for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: libevent for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: rpcbind for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: libnfsidmap for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: quota for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: keyutils for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: libtirpc.so.1()(64bit) for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: libnfsidmap.so.0()(64bit) for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Processing Dependency: libevent-2.0.so.5()(64bit) for package: 1:nfs-utils-1.3.0-0.21.amzn1.x86_64
--> Running transaction check
---> Package keyutils.x86_64 0:1.5.8-3.12.amzn1 will be installed
---> Package libevent.x86_64 0:2.0.18-1.11.amzn1 will be installed
---> Package libnfsidmap.x86_64 0:0.25-11.10.amzn1 will be installed
---> Package libtirpc.x86_64 0:0.2.4-0.3.13.amzn1 will be installed
---> Package quota.x86_64 1:4.00-7.18.amzn1 will be installed
--> Processing Dependency: quota-nls = 1:4.00-7.18.amzn1 for package: 1:quota-4.00-7.18.amzn1.x86_64
--> Processing Dependency: tcp_wrappers for package: 1:quota-4.00-7.18.amzn1.x86_64
--> Processing Dependency: libnl.so.1()(64bit) for package: 1:quota-4.00-7.18.amzn1.x86_64
---> Package rpcbind.x86_64 0:0.2.0-11.8.amzn1 will be installed
--> Processing Dependency: libgssglue for package: rpcbind-0.2.0-11.8.amzn1.x86_64
--> Running transaction check
---> Package libgssglue.x86_64 0:0.1-11.7.amzn1 will be installed
---> Package libnl.x86_64 0:1.1.4-2.10.amzn1 will be installed
---> Package quota-nls.noarch 1:4.00-7.18.amzn1 will be installed
---> Package tcp_wrappers.x86_64 0:7.6-75.11.amzn1 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
================================================================================
Package Arch Version Repository Size
================================================================================
Installing:
nfs-utils x86_64 1:1.3.0-0.21.amzn1 amzn-main 446 k
Installing for dependencies:
keyutils x86_64 1.5.8-3.12.amzn1 amzn-main 54 k
libevent x86_64 2.0.18-1.11.amzn1 amzn-main 281 k
libgssglue x86_64 0.1-11.7.amzn1 amzn-main 23 k
libnfsidmap x86_64 0.25-11.10.amzn1 amzn-main 46 k
libnl x86_64 1.1.4-2.10.amzn1 amzn-main 142 k
libtirpc x86_64 0.2.4-0.3.13.amzn1 amzn-main 89 k
quota x86_64 1:4.00-7.18.amzn1 amzn-main 401 k
quota-nls noarch 1:4.00-7.18.amzn1 amzn-main 93 k
rpcbind x86_64 0.2.0-11.8.amzn1 amzn-main 56 k
tcp_wrappers x86_64 7.6-75.11.amzn1 amzn-main 90 k
Transaction Summary
================================================================================
Install 1 Package (+10 Dependent packages)
Total download size: 1.7 M
Installed size: 3.6 M
Downloading packages:
--------------------------------------------------------------------------------
Total 2.6 MB/s | 1.7 MB 00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : libtirpc-0.2.4-0.3.13.amzn1.x86_64 1/11
Installing : tcp_wrappers-7.6-75.11.amzn1.x86_64 2/11
Installing : libnfsidmap-0.25-11.10.amzn1.x86_64 3/11
Installing : libnl-1.1.4-2.10.amzn1.x86_64 4/11
Installing : keyutils-1.5.8-3.12.amzn1.x86_64 5/11
Installing : libevent-2.0.18-1.11.amzn1.x86_64 6/11
Installing : 1:quota-nls-4.00-7.18.amzn1.noarch 7/11
Installing : 1:quota-4.00-7.18.amzn1.x86_64 8/11
Installing : libgssglue-0.1-11.7.amzn1.x86_64 9/11
Installing : rpcbind-0.2.0-11.8.amzn1.x86_64 10/11
Installing : 1:nfs-utils-1.3.0-0.21.amzn1.x86_64 11/11
Verifying : 1:quota-4.00-7.18.amzn1.x86_64 1/11
Verifying : libgssglue-0.1-11.7.amzn1.x86_64 2/11
Verifying : 1:quota-nls-4.00-7.18.amzn1.noarch 3/11
Verifying : libevent-2.0.18-1.11.amzn1.x86_64 4/11
Verifying : keyutils-1.5.8-3.12.amzn1.x86_64 5/11
Verifying : libnl-1.1.4-2.10.amzn1.x86_64 6/11
Verifying : libtirpc-0.2.4-0.3.13.amzn1.x86_64 7/11
Verifying : rpcbind-0.2.0-11.8.amzn1.x86_64 8/11
Verifying : 1:nfs-utils-1.3.0-0.21.amzn1.x86_64 9/11
Verifying : libnfsidmap-0.25-11.10.amzn1.x86_64 10/11
Verifying : tcp_wrappers-7.6-75.11.amzn1.x86_64 11/11
Installed:
nfs-utils.x86_64 1:1.3.0-0.21.amzn1
Dependency Installed:
keyutils.x86_64 0:1.5.8-3.12.amzn1 libevent.x86_64 0:2.0.18-1.11.amzn1
libgssglue.x86_64 0:0.1-11.7.amzn1 libnfsidmap.x86_64 0:0.25-11.10.amzn1
libnl.x86_64 0:1.1.4-2.10.amzn1 libtirpc.x86_64 0:0.2.4-0.3.13.amzn1
quota.x86_64 1:4.00-7.18.amzn1 quota-nls.noarch 1:4.00-7.18.amzn1
rpcbind.x86_64 0:0.2.0-11.8.amzn1 tcp_wrappers.x86_64 0:7.6-75.11.amzn1
Complete!
+ service rpcbind start
Starting rpcbind: [ OK ]
+ service nfslock start
Starting NFS statd: [ OK ]
+ mkdir /nfs
+ mount -t nfs -o vers=3 ip-10-11-12-7.eu-west-1.compute.internal:/root/nfs_share /nfs
+ cat
+ cat
+ sleep 5
+ service docker stop
Stopping docker: [ OK ]
+ service docker start
Starting docker: .[ OK ]
+ stop ecs
stop: Unknown instance:
+ start ecs
ecs start/running, process 3038
Cloud-init v. 0.7.6 finished at Wed, 27 Jul 2016 12:13:28 +0000. Datasource DataSourceEc2. Up 104.73 seconds
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment