Debugging Docker daemon

gistfile1.txt

ubuntu@my-swarm-node-2:~$ curl -s --unix-socket /var/run/docker.sock http://./debug/pprof/goroutine\?debug\=2 > docker.sock-debug.txt
ubuntu@my-swarm-node-2:~$ head -200 docker.sock-debug.txt
goroutine 2276198 [running]:
runtime/pprof.writeGoroutineStacks({0x560b9f8fa920, 0xc00c0001c0})
        /usr/local/go/src/runtime/pprof/pprof.go:703 +0x70
runtime/pprof.writeGoroutine({0x560b9f8fa920?, 0xc00c0001c0?}, 0xc007a62000?)
        /usr/local/go/src/runtime/pprof/pprof.go:692 +0x2b
runtime/pprof.(*Profile).WriteTo(0x560b9f7162c0?, {0x560b9f8fa920?, 0xc00c0001c0?}, 0xc?)
        /usr/local/go/src/runtime/pprof/pprof.go:329 +0x14b
net/http/pprof.handler.ServeHTTP({0xc00072b9c1, 0x9}, {0x560b9f916940, 0xc00c0001c0}, 0x560b9f417940?)
        /usr/local/go/src/net/http/pprof/pprof.go:259 +0x4a5
github.com/docker/docker/api/server/router/debug.handlePprof({0x560b9f7162c0?, 0xc00b7f0db0?}, {0x560b9f916940, 0xc00c0001c0}, 0xc00e6535a0?, 0x560b9d2b6527?)
        /go/src/github.com/docker/docker/api/server/router/debug/debug_routes.go:10 +0x59
github.com/docker/docker/api/server/middleware.ExperimentalMiddleware.WrapHandler.func1({0x560b9f918180, 0xc00b7f0ed0}, {0x560b9f916940?, 0xc00c0001c0?}, 0x560b9f3ffa40?, 0xc00b6208e0?)
        /go/src/github.com/docker/docker/api/server/middleware/experimental.go:26 +0x15b
github.com/docker/docker/api/server/middleware.VersionMiddleware.WrapHandler.func1({0x560b9f918180, 0xc00b7f0ea0}, {0x560b9f916940, 0xc00c0001c0}, 0xc00e6537e8?, 0xc00e6537a0?)
        /go/src/github.com/docker/docker/api/server/middleware/version.go:62 +0x4d7
github.com/docker/docker/pkg/authorization.(*Middleware).WrapHandler.func1({0x560b9f918180, 0xc00b7f0ea0}, {0x560b9f916940?, 0xc00c0001c0?}, 0xc00d810f00, 0xc00b620890?)
        /go/src/github.com/docker/docker/pkg/authorization/middleware.go:59 +0x649
github.com/docker/docker/api/server.(*Server).makeHTTPHandler.func1({0x560b9f916940, 0xc00c0001c0}, 0xc00d810700)
        /go/src/github.com/docker/docker/api/server/server.go:53 +0x1ce
net/http.HandlerFunc.ServeHTTP(0xc00d810000?, {0x560b9f916940?, 0xc00c0001c0?}, 0xc00e6539e8?)
        /usr/local/go/src/net/http/server.go:2122 +0x2f
github.com/docker/docker/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc008a980c0, {0x560b9f916940, 0xc00c0001c0}, 0xc00ad5d200)
        /go/src/github.com/docker/docker/vendor/github.com/gorilla/mux/mux.go:210 +0x1cf
net/http.serverHandler.ServeHTTP({0xc00b7f0ab0?}, {0x560b9f916940, 0xc00c0001c0}, 0xc00ad5d200)
        /usr/local/go/src/net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc00b83fa70, {0x560b9f918180, 0xc007de6d20})
        /usr/local/go/src/net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:3089 +0x5ed

goroutine 1 [semacquire, 634 minutes, locked to thread]:
sync.runtime_Semacquire(0xc003f695e0?)
        /usr/local/go/src/runtime/sema.go:62 +0x27
sync.(*WaitGroup).Wait(0xc000830d00?)
        /usr/local/go/src/sync/waitgroup.go:116 +0x4b
main.(*DaemonCli).start(0xc000830d00, 0xc000f30540)
        /go/src/github.com/docker/docker/cmd/dockerd/daemon.go:304 +0x174b
main.runDaemon(...)
        /go/src/github.com/docker/docker/cmd/dockerd/docker_unix.go:14
main.newDaemonCommand.func1(0xc000f36000?, {0xc000255920?, 0x3?, 0x3?})
        /go/src/github.com/docker/docker/cmd/dockerd/docker.go:39 +0x94
github.com/docker/docker/vendor/github.com/spf13/cobra.(*Command).execute(0xc000f36000, {0xc00012c010, 0x3, 0x3})
        /go/src/github.com/docker/docker/vendor/github.com/spf13/cobra/command.go:916 +0x862
github.com/docker/docker/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc000f36000)
        /go/src/github.com/docker/docker/vendor/github.com/spf13/cobra/command.go:1044 +0x3bd
github.com/docker/docker/vendor/github.com/spf13/cobra.(*Command).Execute(...)
        /go/src/github.com/docker/docker/vendor/github.com/spf13/cobra/command.go:968
main.main()
        /go/src/github.com/docker/docker/cmd/dockerd/docker.go:109 +0x18c

goroutine 210 [sync.Cond.Wait, 634 minutes]:
sync.runtime_notifyListWait(0xc00159a450, 0x0)
        /usr/local/go/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0x0?)
        /usr/local/go/src/sync/cond.go:70 +0x8c
github.com/docker/docker/daemon/logger.(*messageRing).Dequeue(0xc00159a400)
        /go/src/github.com/docker/docker/daemon/logger/ring.go:189 +0x4b
github.com/docker/docker/daemon/logger.(*RingLogger).run(0xc000136500)
        /go/src/github.com/docker/docker/daemon/logger/ring.go:124 +0x85
created by github.com/docker/docker/daemon/logger.newRingLogger
        /go/src/github.com/docker/docker/daemon/logger/ring.go:45 +0x1f6

goroutine 3810 [IO wait]:
internal/poll.runtime_pollWait(0x7f358ab0f918, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc008d87bc0?, 0xc0041dc000?, 0x1)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc008d87bc0, {0xc0041dc000, 0x8000, 0x8000})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
        /usr/local/go/src/os/file_posix.go:31
os.(*File).Read(0xc0070de598, {0xc0041dc000?, 0x0?, 0x0?})
        /usr/local/go/src/os/file.go:118 +0x5e
github.com/docker/docker/vendor/github.com/containerd/fifo.(*fifo).Read(0xc0086238c0, {0xc0041dc000, 0x8000, 0x8000})
        /go/src/github.com/docker/docker/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x560b9f8f4660, 0xc008facc00}, {0x7f358044c0d8, 0xc0086238c0}, {0xc0041dc000, 0x8000, 0x8000})
        /usr/local/go/src/io/io.go:427 +0x1b2
io.CopyBuffer({0x560b9f8f4660?, 0xc008facc00?}, {0x7f358044c0d8?, 0xc0086238c0?}, {0xc0041dc000?, 0x560b9d2addbd?, 0xc00154cf50?})
        /usr/local/go/src/io/io.go:400 +0x3c
github.com/docker/docker/pkg/pools.Copy({0x560b9f8f4660, 0xc008facc00}, {0x7f358044c0d8, 0xc0086238c0})
        /go/src/github.com/docker/docker/pkg/pools/pools.go:81 +0x7d
github.com/docker/docker/container/stream.(*Config).CopyToPipe.func1.1()
        /go/src/github.com/docker/docker/container/stream/streams.go:123 +0x68
created by github.com/docker/docker/container/stream.(*Config).CopyToPipe.func1
        /go/src/github.com/docker/docker/container/stream/streams.go:122 +0xd8

goroutine 67 [select]:
github.com/docker/docker/vendor/go.opencensus.io/stats/view.(*worker).start(0xc000a88c00)
        /go/src/github.com/docker/docker/vendor/go.opencensus.io/stats/view/worker.go:276 +0xad
created by github.com/docker/docker/vendor/go.opencensus.io/stats/view.init.0
        /go/src/github.com/docker/docker/vendor/go.opencensus.io/stats/view/worker.go:34 +0x96

goroutine 87 [select, 634 minutes]:
io.(*pipe).read(0xc00012ad20, {0xc001003eec, 0xc114, 0x560ba0db1400?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0xc001003e99?, {0xc001003eec?, 0x0?, 0xc000279e18?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc000279f28)
        /usr/local/go/src/bufio/scan.go:214 +0x876
github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).writerScanner(0xc00070f080?, 0xc000014958, 0xc000de5050)
        /go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:86 +0x11f
created by github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).WriterLevel
        /go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:57 +0x3d1

goroutine 88 [select, 634 minutes]:
io.(*pipe).read(0xc00012ad80, {0xc001010618, 0xf9e8, 0xc00009ee00?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0xc00101059f?, {0xc001010618?, 0x6?, 0x100000000000008?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc00009ef28)
        /usr/local/go/src/bufio/scan.go:214 +0x876
github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).writerScanner(0x4?, 0xc000014968, 0xc000de5060)
        /go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:86 +0x11f
created by github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).WriterLevel
        /go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:57 +0x3d1

goroutine 89 [select, 634 minutes]:
io.(*pipe).read(0xc00012ade0, {0xc0004b2000, 0x10000, 0x560b9d2cbf65?})
        /usr/local/go/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0x7f35b36e8428?, {0xc0004b2000?, 0x560b9d2b647d?, 0x0?})
        /usr/local/go/src/io/pipe.go:136 +0x25
bufio.(*Scanner).Scan(0xc000275f28)
        /usr/local/go/src/bufio/scan.go:214 +0x876
github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).writerScanner(0x0?, 0xc000014978, 0xc000de5070)
        /go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:86 +0x11f
created by github.com/docker/docker/vendor/github.com/sirupsen/logrus.(*Entry).WriterLevel
        /go/src/github.com/docker/docker/vendor/github.com/sirupsen/logrus/writer.go:57 +0x3d1

goroutine 100 [syscall, 634 minutes]:
os/signal.signal_recv()
        /usr/local/go/src/runtime/sigqueue.go:152 +0x2f
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:23 +0x19
created by os/signal.Notify.func1.1
        /usr/local/go/src/os/signal/signal.go:151 +0x2a

goroutine 91 [chan receive, 634 minutes]:
github.com/docker/docker/cmd/dockerd/trap.Trap.func1()
        /go/src/github.com/docker/docker/cmd/dockerd/trap/trap.go:31 +0x5c
created by github.com/docker/docker/cmd/dockerd/trap.Trap
        /go/src/github.com/docker/docker/cmd/dockerd/trap/trap.go:29 +0xd9

goroutine 92 [chan receive, 634 minutes]:
main.(*DaemonCli).start.func2()
        /go/src/github.com/docker/docker/cmd/dockerd/daemon.go:198 +0x4b
created by main.(*DaemonCli).start
        /go/src/github.com/docker/docker/cmd/dockerd/daemon.go:192 +0x91b

goroutine 74 [IO wait, 34 minutes]:
internal/poll.runtime_pollWait(0x7f358ab0fbe8, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000570000?, 0xc000266000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000570000, {0xc000266000, 0x8000, 0x8000})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000570000, {0xc000266000?, 0x60100000000?, 0x8?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004ce040, {0xc000266000?, 0x560b9d3194d0?, 0x560b9d2e6367?})
        /usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00070efc0, {0xc000df6200, 0x9, 0x7f35b36dff18?})
        /usr/local/go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x560b9f8f34e0, 0xc00070efc0}, {0xc000df6200, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        /usr/local/go/src/io/io.go:351
github.com/docker/docker/vendor/golang.org/x/net/http2.readFrameHeader({0xc000df6200?, 0x9?, 0xc0000a1d5f?}, {0x560b9f8f34e0?, 0xc00070efc0?})
        /go/src/github.com/docker/docker/vendor/golang.org/x/net/http2/frame.go:237 +0x6e
github.com/docker/docker/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc000df61c0)
        /go/src/github.com/docker/docker/vendor/golang.org/x/net/http2/frame.go:498 +0x95
github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000ad4000)
        /go/src/github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1516 +0x45e
created by github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
        /go/src/github.com/docker/docker/vendor/google.golang.org/grpc/internal/transport/http2_client.go:369 +0x1b4f

goroutine 101 [chan receive, 634 minutes]:
github.com/docker/docker/daemon.(*Daemon).setupDumpStackTrap.func1()
        /go/src/github.com/docker/docker/daemon/debugtrap_unix.go:19 +0x58
created by github.com/docker/docker/daemon.(*Daemon).setupDumpStackTrap
        /go/src/github.com/docker/docker/daemon/debugtrap_unix.go:18 +0xd5

goroutine 115 [IO wait, 634 minutes]:
internal/poll.runtime_pollWait(0x7f358ab0fdc8, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000570c80?, 0xc000276d10?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000570c80)
        /usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd
net.(*netFD).accept(0xc000570c80)
        /usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*UnixListener).accept(0x560b9d3f8ae6?)
        /usr/local/go/src/net/unixsock_posix.go:172 +0x1c
net.(*UnixListener).Accept(0xc00057d3e0)
        /usr/local/go/src/net/unixsock.go:260 +0x3d
net/http.(*Server).Serve(0xc0004ca000, {0x560b9f916760, 0xc00057d3e0})
ubuntu@my-swarm-node-2:~$