Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@tfoote

tfoote/12823 under low load Secret

Last active August 29, 2015 14:20
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tfoote/da7557aafd31a0fbe2f6 to your computer and use it in GitHub Desktop.
Save tfoote/da7557aafd31a0fbe2f6 to your computer and use it in GitHub Desktop.
Download ZIP
pull issue on a low load machine moby/moby#12823
Raw
12823 under low load
Process tree
```
root 464 0.0 0.0 19472 640 ? S Apr30 0:00 upstart-udev-bridge --daemon
root 474 0.0 0.0 51744 1864 ? Ss Apr30 0:00 /lib/systemd/systemd-udevd --daemon
root 645 0.0 0.0 15256 636 ? S Apr30 0:00 upstart-socket-bridge --daemon
root 686 0.0 0.0 10220 2920 ? Ss Apr30 0:00 dhclient -1 -v -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases eth0
root 867 0.0 1.1 653664 46580 ? Ssl Apr30 7:22 /usr/bin/docker -d -H unix:///var/run/docker.sock
message+ 880 0.0 0.0 39224 1220 ? Ss Apr30 0:13 dbus-daemon --system --fork
root 907 0.0 0.0 43824 2184 ? Ss Apr30 0:35 /lib/systemd/systemd-logind
syslog 919 0.0 0.3 260268 14216 ? Ssl Apr30 0:06 rsyslogd
root 931 0.0 0.0 15272 664 ? S Apr30 0:00 upstart-file-bridge --daemon
root 1029 0.0 0.0 14536 912 tty4 Ss+ Apr30 0:00 /sbin/getty -8 38400 tty4
root 1033 0.0 0.0 14536 920 tty5 Ss+ Apr30 0:00 /sbin/getty -8 38400 tty5
root 1037 0.0 0.0 14536 916 tty2 Ss+ Apr30 0:00 /sbin/getty -8 38400 tty2
root 1038 0.0 0.0 14536 920 tty3 Ss+ Apr30 0:00 /sbin/getty -8 38400 tty3
root 1040 0.0 0.0 14536 924 tty6 Ss+ Apr30 0:00 /sbin/getty -8 38400 tty6
root 1081 0.0 0.0 61364 3068 ? Ss Apr30 0:06 /usr/sbin/sshd -D
root 26555 0.1 0.1 105628 4300 ? Ss 17:41 0:00 \_ sshd: root@pts/0
root 26609 0.4 0.1 22016 4476 pts/0 Ss 17:41 0:00 \_ -bash
root 26625 0.0 0.0 17324 1376 pts/0 R+ 17:41 0:00 \_ ps auxf
root 1092 0.0 0.0 23652 936 ? Ss Apr30 0:00 cron
daemon 1093 0.0 0.0 19136 160 ? Ss Apr30 0:00 atd
root 1141 0.0 0.0 4364 628 ? Ss Apr30 0:00 acpid -c /etc/acpi/events -s /var/run/acpid.socket
root 1147 0.0 0.0 19184 764 ? Ss Apr30 0:21 /usr/sbin/irqbalance
jenkins+ 1173 0.0 0.0 18744 608 ? S Apr30 0:00 /usr/bin/daemon --name=jenkins-slave --inherit --output=/var/log/jenkins-slave/jenkins-slave.log --pidfile=/var/run/jenkins-slave/jenkins-slave.pid -- /usr/bin/java -jar /home/jenkins-slave/swarm-client-1.17-jar-with-dependencies.jar -mode exclusive -exe
jenkins+ 1175 0.1 3.8 2451884 156076 ? Sl Apr30 11:58 \_ /usr/bin/java -jar /home/jenkins-slave/swarm-client-1.17-jar-with-dependencies.jar -mode exclusive -executors 1 -username admin -password changeme -name building_repository -master http://master:8080 -labels building_repository -fsroot /home/jenkins-
jenkins+ 29862 0.0 0.0 4440 652 ? S 10:08 0:00 \_ /bin/sh -xe /tmp/hudson6692797644397207291.sh
jenkins+ 29866 0.1 0.2 33472 8912 ? S 10:08 0:34 \_ python3 -u /home/jenkins-slave/workspace/Jrel_sync-packages-to-testing_vivid_i386/ros_buildfarm/scripts/subprocess_reaper.py 29862 --cid-file /home/jenkins-slave/workspace/Jrel_sync-packages-to-testing_vivid_i386/docker_check_sync_criteria/do
jenkins+ 29869 0.0 0.1 26752 6496 ? S 10:08 0:00 \_ python3 -u /home/jenkins-slave/workspace/Jrel_sync-packages-to-testing_vivid_i386/ros_buildfarm/scripts/misc/docker_pull_baseimage.py
jenkins+ 29870 0.0 0.2 190096 11876 ? Sl 10:08 0:00 \_ docker pull ubuntu:trusty
newrelic 1217 0.0 0.0 16932 404 ? Ss Apr30 0:00 /usr/sbin/nrsysmond -c /etc/newrelic/nrsysmond.cfg -p /var/run/newrelic/nrsysmond.pid
newrelic 1218 0.0 0.1 251468 5372 ? Sl Apr30 3:26 \_ /usr/sbin/nrsysmond -c /etc/newrelic/nrsysmond.cfg -p /var/run/newrelic/nrsysmond.pid
root 1333 0.0 0.0 77744 2824 ? Ss Apr30 0:15 /usr/sbin/apache2 -k start
www-data 4196 0.0 0.0 77736 1648 ? S May03 0:00 \_ /usr/sbin/apache2 -k start
www-data 4218 0.0 0.2 1286096 8404 ? Sl May03 0:40 \_ /usr/sbin/apache2 -k start
www-data 4219 0.0 0.2 1286324 8588 ? Sl May03 0:40 \_ /usr/sbin/apache2 -k start
root 1416 0.0 0.0 14536 912 tty1 Ss+ Apr30 0:00 /sbin/getty -8 38400 tty1
root 1417 0.0 0.0 12784 852 ttyS0 Ss+ Apr30 0:00 /sbin/getty -8 38400 ttyS0
ntp 1585 0.0 0.0 31444 1988 ? Ss Apr30 0:19 /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 107:114
```
strace:
```
root@ip-172-31-0-76:~# date
Wed May 6 17:44:50 UTC 2015
root@ip-172-31-0-76:~# strace -p 29870 -f
Process 29870 attached with 6 threads
[pid 29872] futex(0xc20806e9f0, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29875] restart_syscall(<... resuming interrupted call ...> <unfinished ...>
[pid 29873] futex(0x123f9a0, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29871] futex(0x123fd98, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29874] futex(0xc20806f2f0, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29870] epoll_wait(4, <unfinished ...>
[pid 29875] <... restart_syscall resumed> ) = -1 ETIMEDOUT (Connection timed out)
[pid 29875] clock_gettime(CLOCK_MONOTONIC, {505902, 369683410}) = 0
[pid 29875] futex(0x123fd98, FUTEX_WAKE, 1) = 1
[pid 29871] <... futex resumed> ) = 0
[pid 29871] clock_gettime(CLOCK_MONOTONIC, {505902, 369750572}) = 0
[pid 29871] select(0, NULL, NULL, NULL, {0, 20} <unfinished ...>
[pid 29875] clock_gettime(CLOCK_REALTIME, {1430934323, 652610344}) = 0
[pid 29875] futex(0xc20806e9f0, FUTEX_WAKE, 1) = 1
[pid 29872] <... futex resumed> ) = 0
[pid 29872] clock_gettime(CLOCK_MONOTONIC, {505902, 369861497}) = 0
[pid 29872] clock_gettime(CLOCK_MONOTONIC, <unfinished ...>
[pid 29871] <... select resumed> ) = 0 (Timeout)
[pid 29871] futex(0x123fd98, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29875] futex(0x7f33e5ee6f60, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29872] <... clock_gettime resumed> {505902, 370478426}) = 0
[pid 29872] clock_gettime(CLOCK_REALTIME, {1430934323, 653312195}) = 0
[pid 29872] epoll_wait(4, {}, 128, 0) = 0
[pid 29872] futex(0x123fd98, FUTEX_WAKE, 1) = 1
[pid 29871] <... futex resumed> ) = 0
[pid 29871] clock_gettime(CLOCK_MONOTONIC, {505902, 370569237}) = 0
[pid 29871] select(0, NULL, NULL, NULL, {0, 20} <unfinished ...>
[pid 29872] futex(0x7f33e5ee6f60, FUTEX_WAKE, 1) = 1
[pid 29875] <... futex resumed> ) = 0
[pid 29875] futex(0xc20806f770, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29872] clock_gettime(CLOCK_MONOTONIC, {505902, 370661991}) = 0
[pid 29872] futex(0xc20806f770, FUTEX_WAKE, 1) = 1
[pid 29875] <... futex resumed> ) = 0
[pid 29875] epoll_wait(4, <unfinished ...>
[pid 29871] <... select resumed> ) = 0 (Timeout)
[pid 29871] clock_gettime(CLOCK_MONOTONIC, {505902, 370856908}) = 0
[pid 29871] select(0, NULL, NULL, NULL, {0, 20} <unfinished ...>
[pid 29875] <... epoll_wait resumed> {}, 128, 0) = 0
[pid 29875] futex(0xc20806f770, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29872] clock_gettime(CLOCK_MONOTONIC, {505902, 370928755}) = 0
[pid 29872] futex(0x7f33e5ee6f60, FUTEX_WAIT, 0, {60, 0} <unfinished ...>
[pid 29871] <... select resumed> ) = 0 (Timeout)
[pid 29871] futex(0x123fd98, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29872] <... futex resumed> ) = -1 ETIMEDOUT (Connection timed out)
[pid 29872] clock_gettime(CLOCK_MONOTONIC, {505962, 371274999}) = 0
[pid 29872] futex(0x123fd98, FUTEX_WAKE, 1) = 1
[pid 29871] <... futex resumed> ) = 0
[pid 29871] clock_gettime(CLOCK_MONOTONIC, {505962, 371370790}) = 0
[pid 29871] select(0, NULL, NULL, NULL, {0, 20} <unfinished ...>
[pid 29872] clock_gettime(CLOCK_REALTIME, {1430934383, 654248308}) = 0
[pid 29872] clock_gettime(CLOCK_MONOTONIC, {505962, 371460500}) = 0
[pid 29872] futex(0xc20806f770, FUTEX_WAKE, 1 <unfinished ...>
[pid 29871] <... select resumed> ) = 0 (Timeout)
[pid 29875] <... futex resumed> ) = 0
[pid 29872] <... futex resumed> ) = 1
[pid 29871] clock_gettime(CLOCK_MONOTONIC, <unfinished ...>
[pid 29875] epoll_wait(4, <unfinished ...>
[pid 29872] clock_gettime(CLOCK_MONOTONIC, <unfinished ...>
[pid 29871] <... clock_gettime resumed> {505962, 371651621}) = 0
[pid 29875] <... epoll_wait resumed> {}, 128, 0) = 0
[pid 29872] <... clock_gettime resumed> {505962, 371710289}) = 0
[pid 29875] futex(0xc20806f770, FUTEX_WAIT, 0, NULL <unfinished ...>
[pid 29871] select(0, NULL, NULL, NULL, {0, 20} <unfinished ...>
[pid 29872] futex(0x7f33e5ee6f60, FUTEX_WAIT, 0, {60, 0} <unfinished ...>
[pid 29871] <... select resumed> ) = 0 (Timeout)
[pid 29871] futex(0x123fd98, FUTEX_WAIT, 0, NULL^CProcess 29870 detached
Process 29871 detached
<detached ...>
Process 29872 detached
Process 29873 detached
Process 29874 detached
Process 29875 detached
root@ip-172-31-0-76:~# date
Wed May 6 17:46:33 UTC 2015
```
Docker info and version
```
root@ip-172-31-0-76:~# docker info
Containers: 1262
Images: 1254
Storage Driver: aufs
Root Dir: /var/lib/docker/aufs
Dirs: 3778
Execution Driver: native-0.2
Kernel Version: 3.13.0-49-generic
Operating System: Ubuntu 14.04.2 LTS
WARNING: No swap limit support
root@ip-172-31-0-76:~# docker version
Client version: 1.3.2
Client API version: 1.15
Go version (client): go1.3.3
Git commit (client): 39fa2fa
OS/Arch (client): linux/amd64
Server version: 1.3.2
Server API version: 1.15
Go version (server): go1.3.3
Git commit (server): 39fa2fa
```
```
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment