Skip to content

Instantly share code, notes, and snippets.

@ngoclamtran
Created Nov 1, 2021
Embed
What would you like to do?
This file has been truncated, but you can view the full file.
goroutine 1832008 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc0003280e0})
/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc0003280e0}, 0x0)
/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc0003280e0}, 0xc)
/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc001aedcf1, 0x34934e0}, {0x2386300, 0xc0003280e0}, 0xc001aedce4)
/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc0003280e0}, 0xc001447b00)
/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc0003280e0}, 0x0)
/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc001aedcfb, {0x2386300, 0xc0003280e0}, 0xc001447b00)
/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc00129b740}, {0x2386300, 0xc0003280e0}, 0xc001447b00)
/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc001fbd0e0, {0x23922d0, 0xc0005296e0})
/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3033 +0x4e8
goroutine 1 [chan receive, 10 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc000792000)
/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc000143240)
/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc000579b00, 0xc000143200)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc000082680, {0x2392260, 0xc0000b0000}, {0xc0000ae000, 0x8, 0x8})
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
/temporal/cmd/server/main.go:50 +0x45
goroutine 26 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00003a300)
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92
goroutine 28 [syscall, 10 minutes]:
os/signal.signal_recv()
/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x2c
goroutine 4 [chan receive, 10 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 5 [chan receive, 10 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 1471878 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000140af0, {0x2392228, 0xc001f078c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 63 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda99830, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000878380, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000878380)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000878380)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000324f18)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000324f18)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000084d80)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790000)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 97 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000569b80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 41 [select]:
database/sql.(*DB).connectionOpener(0xc0005305b0, {0x2392228, 0xc0006b33c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 60 [select]:
database/sql.(*DB).connectionCleaner(0xc0005305b0, 0xc0000884b0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 69 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 11 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda99a00, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003ab80, 0x203000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003ab80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003ab80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0003245b8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0003245b8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc000328540, {0x23860f0, 0xc0003245b8})
/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc000328540)
/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
/temporal/common/pprof/pprof.go:72 +0x9c
goroutine 64 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000098f70, {0x2392228, 0xc0007a8880})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 39 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000884b0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1658034 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005cbb00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac210, {0x1f7c860, 0xc001bac0b0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8c800, 0xc001b2d5e8, 0xc001b2d5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8c800, {0x1f7c860, 0xc001bac0b0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002428f00}, {0x20768da, 0x8}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x1c75901, {0xc000be4e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x7f00048b45b8, {0xc000be4e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x20, 0xc001f097a0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0xc000f5fa00, {0xc001f83a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x0, 0xc001f09770, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x1e0cfc0, 0xc001f096e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x4167af, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002428f00}, {0x20768da, 0x203000}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x2e, 0xc001f096b0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0xc001216e50, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc002428f00}, {0x20768da, 0x0}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc002428f00}, 0xc001216f20, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc002428f00})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658097 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001dcdc20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e836b0, {0x1f7b7c0, 0xc000330780}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40400, 0xc000aca4d0, 0xc000aca4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40400, {0x1f7b7c0, 0xc000330780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x20, {0xc001bc5b40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x0, {0xc001bc5b40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0xc000c61270, 0xc0020ac090, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e64d40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x33, 0xc0020fde60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e64d40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e64d40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc000e64d40}, 0xc000a64586, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc002643980}, 0xc000e64c80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc002643980}, 0xc000e64c80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002643680, {0x2392298, 0xc002643980})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002643920, {0x2392298, 0xc002643980}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001dcd680, {0x23922d0, 0xc0020fd080}, 0x2a)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc000ba4240, {0x23922d0, 0xc0020fd080}, 0xc0013d1420, 0x1, 0x13d1430)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0020fcff0, 0xc000e643c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0020fcc90}, 0xc000e643c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020fcc90}, {0x1f35540, 0xc000e643c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0020fcc90}, {0x1f35540, 0xc000e643c0}, 0x237bc38, 0xc0024e52a8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcc90}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0}, 0xc000e17e00, 0xc000e64440)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0}, 0x40, 0xc000e64440)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0}, 0xc001bc5980, 0xc0024e52a8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0020fcb10}, 0xc0026433e0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001dcc6c0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001dcc6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 81 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0006bd810)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 48 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda99748, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000878800, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000878800)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000878800)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0003253c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0003253c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0000d6fc0, {0x23860f0, 0xc0003253c8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc00002a7d0)
/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc0006bd6d0}, 0x0)
/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
/temporal/service/history/fx.go:168 +0x9a
goroutine 927 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd8655f0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00149c180, 0xc000ff3ea0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00149c180, {0xc000ff3ea0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00149c180, {0xc000ff3ea0, 0xc000a33d70, 0xc001297f20})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aaf68, {0xc000ff3ea0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aaf68}, {0xc000ff3ea0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007e7b80, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 83 [select]:
database/sql.(*DB).connectionOpener(0xc0006de8f0, {0x2392228, 0xc000a195c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 72 [select]:
database/sql.(*DB).connectionCleaner(0xc0006de8f0, 0xc0006bd810)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 74 [sleep]:
time.Sleep(0x342770c00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 928 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007e7b80, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1471880 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0024ee3c0, {0x2392228, 0xc001f07900})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 80 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda993a8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000879500, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000879500)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000879500)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000325b18)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000325b18)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0007a4540, {0x23860f0, 0xc000325b18})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc0007f3fd0)
/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc0007deab0}, 0x32)
/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
/temporal/service/matching/fx.go:129 +0x9a
goroutine 89 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda992c0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872600, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000872600)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000872600)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000a248b8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000a248b8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc00053b8c0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790c00)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 101 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 1651782 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c53680)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e831e0, {0x1f7b560, 0xc001fbc640}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d900, 0xc000aaa440, 0xc000aaa410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d900, {0x1f7b560, 0xc001fbc640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x20, {0xc001bc4640, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0xc001de7080, {0xc001bc4640, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0xc0023f4400, 0xc001fd2510, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0017ed500}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x237ac70, 0xc001fd2450, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0017ed500}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0017ed500}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0017ed500}, 0xc001eab006, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc0025717a0}, 0xc0017ed480, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc0025717a0}, 0xc0017ed480, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0012e8b40, {0x2392298, 0xc0025717a0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0012e9320, {0x2392298, 0xc0025717a0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000bbec60, {0x23922d0, 0xc001fd2150}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fd2150}, 0xc001438db0, 0x1bd58c0, 0x1438dc0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001fd20f0, 0xc0017ed280)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001e9be30}, 0xc0017ed280)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e9be30}, {0x1f353a0, 0xc0017ed280})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001e9be30}, {0x1f353a0, 0xc0017ed280}, 0x237bc38, 0xc002004630)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9be30}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001e9bcb0, {0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280}, 0xc001e9aed0, 0xc0017ed2c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280}, 0x40, 0xc0017ed2c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280}, 0xc001bc45a0, 0xc002004630)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001e9bc50}, 0xc002571740, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c52900, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c52900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 92 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8
goroutine 93 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 94 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1678726 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000e88840, {0x2392298, 0xc001aea540}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000f70480, {0x23922d0, 0xc0022eecc0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002477050, {0x23922d0, 0xc0022eecc0}, 0xc00124d000, 0x1, 0x124d010)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0022eec60, 0xc001250600)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022ee9f0}, 0xc001250600)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022ee9f0}, {0x1f35540, 0xc001250600})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022ee9f0}, {0x1f35540, 0xc001250600}, 0x237bc38, 0xc0021ca930)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee9f0}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600}, 0xc001250000, 0xc001250680)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600}, 0x40, 0xc001250680)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600}, 0xc000ebb5c0, 0xc0021ca930)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0022ee7e0}, 0xc001aea4e0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000fe8240, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000fe8240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678773 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001dcd320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0160, {0x1f7b7c0, 0xc001f4d720}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078b900, 0xc000f49bf8, 0xc000f49bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078b900, {0x1f7b7c0, 0xc001f4d720})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x20, {0xc001c648e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0xc0020af600, {0xc001c648e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0xc00119e640, 0xc0022ff6e0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012cc240}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x237ac70, 0xc0022ff620, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012cc240}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012cc240}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012cc240}, 0xc0022013c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022fee70}, 0xc0012cc1c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022fee70}, 0xc0012cc1c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000f4a480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000f4a528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd817890, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0022fee70}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00119e7e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000f4a778, {0x235a640, 0xc000045ad0}, 0x612d623139342d30)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022fee70}, 0xc001283ec0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001726db8, 0x4c1cb4}, {0xc057f6daf65f8b90, 0x8c0912e4da}, {0xc001726db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0022fee70}, 0xc001283ec0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0xc001c647a0, 0xc00237ca68)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0x15, 0xc001283f00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0x2076951, 0xc001283f00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0x237bc38, 0xc001283f00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0022fec00}, {0x1f33320, 0xc001283ec0}, 0x1e5a940, 0xc001283f00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fec00}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0xc0022012c0, 0xc001283f00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0xc001c647a0, 0xc00237ca68)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0022feb10}, 0xc0020a6780, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbb9e0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbb9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1677974 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd9205e0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ca9900, 0xc000d26000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ca9900, {0xc000d26000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ca9900, {0xc000d26000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278eb88, {0xc000d26000, 0x0, 0x7effdd8ca7f8})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0c7e0, {0xc0006ba3c0, 0x9, 0xc00005e800})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0c7e0}, {0xc0006ba3c0, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba3c0, 0x9, 0xc0012e5650}, {0x23585c0, 0xc001c0c7e0})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba380)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000ab40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1657967 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019fd8c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e824d0, {0x1f7b560, 0xc001b154a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391b00, 0xc00198c440, 0xc00198c410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391b00, {0x1f7b560, 0xc001b154a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x20, {0xc00230b520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0xc001ffd4a0, {0xc00230b520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0xc0023f4400, 0xc0023c77a0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0024feb40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x237ac70, 0xc0023c76b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0024feb40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0024feb40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0024feb40}, 0xc0024a78f6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc001fdd500}, 0xc0024feac0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc001fdd500}, 0xc0024feac0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001f8f200, {0x2392298, 0xc001fdd500})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001f8f620, {0x2392298, 0xc001fdd500}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d93320, {0x23922d0, 0xc0023c6f90}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0023c6f90}, 0xc00139b080, 0x1bd58c0, 0x139b090)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0023c6f30, 0xc0024fe180)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0023c6c90}, 0xc0024fe180)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023c6c90}, {0x1f353a0, 0xc0024fe180})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0023c6c90}, {0x1f353a0, 0xc0024fe180}, 0x237bc38, 0xc00205cee8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6c90}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0023c6b10, {0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180}, 0x4167af, 0xc0024fe240)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180}, 0x40, 0xc0024fe240)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180}, 0xc00230b360, 0xc00205cee8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0023c6ab0}, 0xc001fdd4a0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019fc6c0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019fc6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 105 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000776410, {0x2392228, 0xc000021540})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 109 [sleep]:
time.Sleep(0x2cb417800)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 1658165 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda99490, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001301000, 0xc000c96000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001301000, {0xc000c96000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001301000, {0xc000c96000, 0xc001301018, 0xc000c96000})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c15bd0, {0xc000c96000, 0x448e71, 0x466a4e})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc0000beb40, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc0000beb40, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc0000beb40)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlConn).readResultSetHeaderPacket(0xc0023e5d10)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:537 +0x25
github.com/go-sql-driver/mysql.(*mysqlStmt).query(0xc0023e5d10, {0xc000e969b0, 0xc0012f9da0, 0xc000e9a650})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/statement.go:108 +0x125
github.com/go-sql-driver/mysql.(*mysqlStmt).QueryContext(0xc0023e5d10, {0x2392298, 0xc0012f9da0}, {0xc000e9a5b0, 0x94, 0x208f76a})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:565 +0x91
database/sql.ctxDriverStmtQuery({0x2392298, 0xc0012f9da0}, {0x2392458, 0xc0023e5d10}, {0xc000e9a5b0, 0x5, 0x5})
/usr/local/go/src/database/sql/ctxutil.go:82 +0x132
database/sql.rowsiFromStatement({0x2392298, 0xc0012f9da0}, {0x237efb0, 0xc0000beb40}, 0xc000e8d200, {0xc000dffdb0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:2794 +0x17e
database/sql.(*DB).queryDC(0x203001, {0x2392298, 0xc0012f9da0}, {0x0, 0x0}, 0xc0013974d0, 0xc00127dfc0, {0x208f76a, 0x94}, {0xc000dffdb0, ...})
/usr/local/go/src/database/sql/sql.go:1752 +0x3d7
database/sql.(*DB).query(0x1947548, {0x2392298, 0xc0012f9da0}, {0x208f76a, 0x94}, {0xc000dffdb0, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0012f9da0}, {0x208f76a, 0x94}, {0xc000dffdb0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0012f9da0}, {0x208f76a, 0xc00004c000}, {0xc000dffdb0, 0xc0019476c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0012f9da0}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0023e59e0}, {0x208f76a, 0xc000ee9400}, {0xc000dffdb0, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2d, {0x2392298, 0xc0012f9da0}, {0x1b5d8e0, 0xc0023e59e0}, {0x208f76a, 0xc0019477e0}, {0xc000dffdb0, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0012f9da0}, {0xd070a498, {0xc0013c8880, 0x3e, 0x3e}, 0x0, 0xc000e8cbe8, 0xc00245f358, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e8cbc0)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x2d)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0023985a0, 0x3f8ff900, 0x3f917fa0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc000716400, 0xc0012eeba0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001947c50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc001947d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001947de2, 0x4f8639)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001947d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc002398720)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002398720, {0x2392228, 0xc001171200})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658032 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000409560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e83290, {0x1f7b7c0, 0xc001fbce60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8cf00, 0xc0026ae4d0, 0xc0026ae4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8cf00, {0x1f7b7c0, 0xc001fbce60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x20, {0xc001bc4b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0xc0008529a0, {0xc001bc4b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0xc000c61270, 0xc0024ccea0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001366d80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x237ac70, 0xc0024ccde0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001366d80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001366d80}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc001366d80}, 0xc00204d296, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc001e8ccc0}, 0xc001366cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc001e8ccc0}, 0xc001366cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001e8ca20, {0x2392298, 0xc001e8ccc0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001e8cc60, {0x2392298, 0xc001e8ccc0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001923560, {0x23922d0, 0xc0024cc210}, 0x1e)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc00204d230, {0x23922d0, 0xc0024cc210}, 0xc00141b170, 0x1, 0x141b180)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0024cc1b0, 0xc001366400)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0024dbef0}, 0xc001366400)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0024dbef0}, {0x1f35540, 0xc001366400})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0024dbef0}, {0x1f35540, 0xc001366400}, 0x237bc38, 0xc002288588)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbef0}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0024da960, {0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400}, 0xc00245baa0, 0xc001366480)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400}, 0x40, 0xc001366480)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400}, 0xc001bc48c0, 0xc002288588)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0024dbc80}, 0xc001e8c780, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001922900, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001922900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1654705 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005d6480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116dad0, {0x1f7b7c0, 0xc001fbcfa0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8dd00, 0xc0026a9bf8, 0xc0026a9bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8dd00, {0x1f7b7c0, 0xc001fbcfa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x20, {0xc001bc5f20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0xc001470160, {0xc001bc5f20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0xc00119e640, 0xc0021ad950, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021aac40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x237ac70, 0xc0021ad890, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021aac40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021aac40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0021aac40}, 0xc001e163f0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0021ad1d0}, 0xc0021aabc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0021ad1d0}, 0xc0021aabc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0026aa480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0026aa528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd825670, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0021ad1d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021b4250})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0026aa778, {0x235a640, 0xc000045ad0}, 0x382d366530342d61)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0021ad1d0}, 0xc0021aa8c0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000b5cdb8, 0x4c1cb4}, {0xc057f6d8f59f3f24, 0x8a2b7c486f}, {0xc000b5cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0021ad1d0}, 0xc0021aa8c0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0xc001bc5ce0, 0xc0021ba5e8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0x15, 0xc0021aa900)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0x2076951, 0xc0021aa900)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0x237bc38, 0xc0021aa900)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021ad080}, {0x1f33320, 0xc0021aa8c0}, 0x1e5a940, 0xc0021aa900)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad080}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0x23c27b0, 0xc0021aa900)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bbdb60, {0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0xc001bc5ce0, 0xc0021ba5e8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0021ad050}, 0xc002362480, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c1eea0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c1eea0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1654718 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001af9680)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116def0, {0x1f7b7c0, 0xc001fbd7c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446b00, 0xc00196a4d0, 0xc00196a4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446b00, {0x1f7b7c0, 0xc001fbd7c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x20, {0xc00230a1e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0xc000138580, {0xc00230a1e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0xc0023f4400, 0xc0021c5ef0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021c3040}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x237ac70, 0xc0021c5e30, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021c3040}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021c3040}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc0021c3040}, 0xc0026f7536, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc0023637a0}, 0xc0021c2f80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc0023637a0}, 0xc0021c2f80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0026bca20, {0x2392298, 0xc0023637a0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0026bcc60, {0x2392298, 0xc0023637a0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00112e120, {0x23922d0, 0xc0021c5c20}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001e17020, {0x23922d0, 0xc0021c5c20}, 0xc0021c82d0, 0x1, 0x21c82e0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0021c5bc0, 0xc0021c2d40)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0021c5920}, 0xc0021c2d40)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021c5920}, {0x1f35540, 0xc0021c2d40})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0021c5920}, {0x1f35540, 0xc0021c2d40}, 0x237bc38, 0xc0021bbea8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c5920}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40}, 0xc0021c2780, 0xc0021c2dc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40}, 0x40, 0xc0021c2dc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40}, 0xc00230a100, 0xc0021bbea8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0021c57a0}, 0xc002363740, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001af86c0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001af86c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678784 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077f440)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae02c0, {0x1f7b7c0, 0xc001f4da40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078bc00, 0xc001befbf8, 0xc001befbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078bc00, {0x1f7b7c0, 0xc001f4da40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x20, {0xc001c64e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0xc001470000, {0xc001c64e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0xc00119e640, 0xc002445aa0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012cd3c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x237ac70, 0xc0024459e0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012cd3c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012cd3c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012cd3c0}, 0xc0022016c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc002445350}, 0xc0012cd340, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc002445350}, 0xc0012cd340, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001bf0480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bf0528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd817890, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002445350}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00119ecb0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bf0778, {0x235a640, 0xc000045ad0}, 0x612d376365342d34)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002445350}, 0xc0012cd040)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00194adb8, 0x4c1cb4}, {0xc057f6daf69ccde6, 0x8c09502730}, {0xc00194adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc002445350}, 0xc0012cd040)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0xc001c64ce0, 0xc00237d6f8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0x15, 0xc0012cd080)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0x2076951, 0xc0012cd080)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0x237bc38, 0xc0012cd080)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002445200}, {0x1f33320, 0xc0012cd040}, 0x1e5a940, 0xc0012cd080)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445200}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0x23c27b0, 0xc0012cd080)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0xc001c64ce0, 0xc00237d6f8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0024451d0}, 0xc0020a7380, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bc5200, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bc5200, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658051 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001e8c960, {0x2392228, 0xc001366c00})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 811 [select]:
go.temporal.io/server/service/history/shard.(*ControllerImpl).shardManagementPump(0xc000cd74a0)
/temporal/service/history/shard/controller_impl.go:312 +0x148
created by go.temporal.io/server/service/history/shard.(*ControllerImpl).Start
/temporal/service/history/shard/controller_impl.go:145 +0xaf
goroutine 1658141 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019758c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bacdc0, {0x1f7b560, 0xc001941680}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb9)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d800, 0xc001bd7d18, 0xc001bd7ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d800, {0x1f7b560, 0xc001941680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002127440}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x20, {0xc000eba2a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002127440}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002127440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0xc002084420, {0xc000eba2a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002127440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0xc00119e640, 0xc002127470, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002127440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e91fc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x237ac70, 0xc0021273b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e91fc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e91fc0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc000e91fc0}, 0xc002688f90, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc002126c90}, 0xc000e91f40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc002126c90}, 0xc000e91f40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001bd85a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bd8648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8c47b0, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc002126c90}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0023642e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bd8898, {0x235a640, 0xc000045ad0}, 0x612d643133342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002126c90}, 0xc002126ae0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001946db8, 0x4c1cb4}, {0xc057f6d9391f219f, 0x8a6a96f4ef}, {0xc001946db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc002126c90}, 0xc002126ae0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0xc000eba1e0, 0xc00209b218)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0x15, 0xc000e91d00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0x207690b, 0xc000e91d00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0x237bc38, 0xc000e91d00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002126b40}, {0x1f33180, 0xc002126ae0}, 0x1e5a940, 0xc000e91d00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126b40}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0x9, 0xc000e91d00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0xc002888768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002888b60, {0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0xc000eba1e0, 0xc00209b218)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc002126ab0}, 0xc002549e00, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001974a20, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001974a20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1586529 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920c38, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00132c880, 0xc0013f2000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00132c880, {0xc0013f2000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00132c880, {0xc0013f2000, 0x1060100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278f8f8, {0xc0013f2000, 0xc002847cf8, 0x44d2f4})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001ed2360, {0xc0006ba040, 0x9, 0x9772e782d4})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001ed2360}, {0xc0006ba040, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba040, 0x9, 0xc001331b30}, {0x23585c0, 0xc001ed2360})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba000)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001258600, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a41c0, {0x23d1088, 0xc001258600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 1657986 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023aafc0, {0x2392298, 0xc001fdd920}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001127200, {0x23922d0, 0xc001f123f0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001f123f0}, 0xc00139bb40, 0x1bd58c0, 0x139bb50)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001f121e0, 0xc0024fee00)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0023c7ef0}, 0xc0024fee00)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023c7ef0}, {0x1f353a0, 0xc0024fee00})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0023c7ef0}, {0x1f353a0, 0xc0024fee00}, 0x237bc38, 0xc00205d608)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7ef0}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0023c6b10, {0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00}, 0x4167af, 0xc0024fee40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00}, 0x40, 0xc0024fee40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00}, 0xc00230b640, 0xc00205d608)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0023c7cb0}, 0xc001fdd8c0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0019fd9e0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0019fd9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 140 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000c7a1a0, {0x2392228, 0xc000c76340})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 102 [select]:
database/sql.(*DB).connectionCleaner(0xc000c7a1a0, 0x0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 104 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdd95c748, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4da00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000a4da00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000a4da00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000bc10c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000bc10c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007c29a0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000354200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 148 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda990f0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4ce00, 0xc000c61040, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a4ce00, {0xc000c61040, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000a4ce00, {0xc000c61040, 0xc000c00f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14d08, {0xc000c61040, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14d08}, {0xc000c61040, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062a840, 0x1c)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 146 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c660, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4d480, 0xc000c60e40, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a4d480, {0xc000c60e40, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000a4d480, {0xc000c60e40, 0xc0025cae70, 0xc002114b40})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14d10, {0xc000c60e40, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14d10}, {0xc000c60e40, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062a580, 0xc8e090)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 147 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062a580, 0x790c00)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 149 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062a840, 0x1fe52a7)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 156 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 1677984 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001259980)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837
goroutine 350 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c630)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 164 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000893b0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 163 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c578, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4dd00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000a4dd00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000a4dd00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000bc13c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000bc13c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0007a41c0, {0x23860f0, 0xc000bc13c8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0xc000a6cfd0)
/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc00003a700}, 0x1c)
/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
/temporal/service/frontend/fx.go:282 +0x9a
goroutine 157 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 348 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 1658026 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ae37a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e83130, {0x1f7b560, 0xc001fbc780}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8c500, 0xc0017cbd18, 0xc0017cbce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8c500, {0x1f7b560, 0xc001fbc780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0024da510}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x20, {0xc001bc45a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0024da510}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024da510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0xc000139e40, {0xc001bc45a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024da510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0xc00119e640, 0xc0024da540, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024da510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0024cbc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x237ac70, 0xc0024da480, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0024cbc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0024cbc40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0024cbc40}, 0xc00204cde0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc00245bc50}, 0xc0024cbbc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc00245bc50}, 0xc0024cbbc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017cc5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017cc648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8c2280, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc00245bc50}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000c61170})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017cc898, {0x235a640, 0xc000045ad0}, 0x392d326638342d36)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00245bc50}, 0xc00245baa0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00172edb8, 0x4c1cb4}, {0xc057f6d936dace3f, 0x8a6852a189}, {0xc00172edb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc00245bc50}, 0xc00245baa0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0xc001bc4400, 0xc0019bb830)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0x15, 0xc0024cb980)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0x207690b, 0xc0024cb980)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0x237bc38, 0xc0024cb980)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00245bb00}, {0x1f33180, 0xc00245baa0}, 0x1e5a940, 0xc0024cb980)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bb00}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0x9, 0xc0024cb980)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0xc001bf5768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0xc001bc4400, 0xc0019bb830)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00245ba70}, 0xc001e8c120, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001ae2120, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001ae2120, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657951 [select]:
database/sql.(*DB).conn(0xc0006de8f0, {0x2392298, 0xc0021d2de0}, 0x1)
/usr/local/go/src/database/sql/sql.go:1312 +0x413
database/sql.(*DB).query(0xa3f548, {0x2392298, 0xc0021d2de0}, {0x208f76a, 0x94}, {0xc000e5e370, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1695 +0x5d
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0021d2de0}, {0x208f76a, 0x94}, {0xc000e5e370, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0021d2de0}, {0x208f76a, 0xc00004e800}, {0xc000e5e370, 0xc000a3f6c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0021d2de0}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0025f1500}, {0x208f76a, 0xc001aedc00}, {0xc000e5e370, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2b, {0x2392298, 0xc0021d2de0}, {0x1b5d8e0, 0xc0025f1500}, {0x208f76a, 0xc000a3f7e0}, {0xc000e5e370, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0021d2de0}, {0xdb6a293a, {0xc001db9980, 0x3c, 0x3c}, 0x0, 0xc000e3ffa8, 0xc00210ed58, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e3ff80)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x2b)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0023ec4e0, 0x3e8b0c20, 0x3e8c92c0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc00005e400, 0xc00133a000})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000a3fc50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc000a3fd10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc000a3fde2, 0xc00005e400)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc000a3fd60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc0023ec660)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0023ec660, {0x2392228, 0xc001318380})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 429 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 206 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95ba98, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000bc7a00, 0xc000041960, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000bc7a00, {0xc000041960, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000bc7a00, {0xc000041960, 0xc000ce1f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14e18, {0xc000041960, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14e18}, {0xc000041960, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062b340, 0x7e0fb0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 213 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 208 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c1d8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872500, 0xc000041a90, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000872500, {0xc000041a90, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000872500, {0xc000041a90, 0xc000ce2f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14e10, {0xc000041a90, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14e10}, {0xc000041a90, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062b600, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 351 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c6e0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 346 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 347 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 418 [select, 10 minutes]:
go.temporal.io/server/service/history/events.(*NotifierImpl).dequeueHistoryEventNotifications(0xc000830cd0)
/temporal/service/history/events/notifier.go:234 +0x105
created by go.temporal.io/server/service/history/events.(*NotifierImpl).Start
/temporal/service/history/events/notifier.go:253 +0x71
goroutine 417 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920240, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000d5b480, 0xc000f33000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000d5b480, {0xc000f33000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000d5b480, {0xc000f33000, 0xc000d5b480, 0xc0017af170})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c15098, {0xc000f33000, 0xc0017af1f8, 0x677dc5})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc000f719e0, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc000f719e0, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc000f719e0)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlConn).readResultSetHeaderPacket(0xc000f719e0)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:537 +0x25
github.com/go-sql-driver/mysql.(*mysqlConn).query(0xc000f719e0, {0x207fcf5, 0xc000ba2420}, {0x34c4b38, 0x1bd5ac0, 0x0})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:390 +0xb4
github.com/go-sql-driver/mysql.(*mysqlConn).QueryContext(0xc000f719e0, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x4e}, {0x34c4b38, 0x45e285, 0xc00072c600})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:512 +0xb2
database/sql.ctxDriverQuery({0x2392298, 0xc000ba2420}, {0x7effdd943e68, 0xc000f719e0}, {0x0, 0x0}, {0x207fcf5, 0x0}, {0x34c4b38, 0x0, ...})
/usr/local/go/src/database/sql/ctxutil.go:48 +0x17d
database/sql.(*DB).queryDC.func1()
/usr/local/go/src/database/sql/sql.go:1722 +0x175
database/sql.withLock({0x237b030, 0xc000f37710}, 0xc00072c6d0)
/usr/local/go/src/database/sql/sql.go:3396 +0x8c
database/sql.(*DB).queryDC(0x1, {0x2392298, 0xc000ba2420}, {0x0, 0x0}, 0xc000f37710, 0xc0024be030, {0x207fcf5, 0x4e}, {0x0, ...})
/usr/local/go/src/database/sql/sql.go:1717 +0x211
database/sql.(*DB).query(0x2, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x4e}, {0x0, 0x0, 0x0}, 0x0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x10000c000e96788, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x4e}, {0x0, 0x0, 0x0})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryRowxContext(0xc0006f6240, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x2392260}, {0x0, 0x1000003493960, 0xc000ba2420})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:171 +0x45
github.com/jmoiron/sqlx.GetContext({0x2392298, 0xc000ba2420}, {0x2380120, 0xc0006f6240}, {0x1b8e560, 0xc00245f638}, {0x207fcf5, 0x4f9101}, {0x0, 0x0, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:82 +0x6b
github.com/jmoiron/sqlx.(*DB).GetContext(0x0, {0x2392298, 0xc000ba2420}, {0x1b8e560, 0xc00245f638}, {0x207fcf5, 0x4f94ca}, {0x0, 0x0, 0x0})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:147 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromNamespaceMetadata(0xc000568a00, {0x2392298, 0xc000ba2420})
/temporal/common/persistence/sql/sqlplugin/mysql/namespace.go:216 +0x7d
go.temporal.io/server/common/persistence/sql.(*sqlMetadataManagerV2).GetMetadata(0xc00087bd40)
/temporal/common/persistence/sql/metadata.go:222 +0x8a
go.temporal.io/server/common/persistence.(*metadataManagerImpl).GetMetadata(0xc0006bfe90)
/temporal/common/persistence/metadata_manager.go:186 +0x22
go.temporal.io/server/common/persistence.(*metadataRateLimitedPersistenceClient).GetMetadata(0xc00087bd70)
/temporal/common/persistence/persistenceRateLimitedClients.go:584 +0x42
go.temporal.io/server/common/persistence.(*metadataPersistenceClient).GetMetadata(0xc00087bda0)
/temporal/common/persistence/persistenceMetricClients.go:822 +0x73
go.temporal.io/server/common/namespace.(*registry).refreshNamespaces(0xc00016ac00, {0xc0019dddb8, 0x438672})
/temporal/common/namespace/registry.go:382 +0x42
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016ac00, {0x2392228, 0xc000ea6dc0})
/temporal/common/namespace/registry.go:359 +0x1b3
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1165 [select, 10 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a1fd60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1477857 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0026bc960, {0x2392228, 0xc001c33f40})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654883 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005ca900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116cdc0, {0x1f7b560, 0xc001de39a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41c00, 0xc001720440, 0xc001720410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41c00, {0x1f7b560, 0xc001de39a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x20, {0xc0022b47a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0xc001f26580, {0xc0022b47a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0xc0023f4400, 0xc001b9b050, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0023a8100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x237ac70, 0xc001b9af90, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0023a8100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0023a8100}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0023a8100}, 0xc001db9686, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc002244f60}, 0xc0023a8080, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc002244f60}, 0xc0023a8080, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc000f96360, {0x2392298, 0xc002244f60})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000f965a0, {0x2392298, 0xc002244f60}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001923680, {0x23922d0, 0xc002255aa0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002255aa0}, 0xc002256730, 0x1bd58c0, 0x2256740)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc002255a40, 0xc00223bf40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022557a0}, 0xc00223bf40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022557a0}, {0x1f353a0, 0xc00223bf40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022557a0}, {0x1f353a0, 0xc00223bf40}, 0x237bc38, 0xc002241218)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022557a0}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002255620, {0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40}, 0xc02be2, 0xc00223bf80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40}, 0x40, 0xc00223bf80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40}, 0xc000a70ce0, 0xc002241218)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0022555c0}, 0xc002244f00, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001974240, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001974240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1480931 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000379650, {0x2392228, 0xc001421840})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1678720 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1e480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80b00, {0x1f7b7c0, 0xc001b15900}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446000, 0xc000d05bf8, 0xc000d05bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446000, {0x1f7b7c0, 0xc001b15900})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x20, {0xc000ebb120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0xc001f27760, {0xc000ebb120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0xc00119e640, 0xc0022b1ec0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012415c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x237ac70, 0xc0022b1dd0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012415c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012415c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012415c0}, 0xc00210a000, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022b13e0}, 0xc001241540, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022b13e0}, 0xc001241540, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000d06480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000d06528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd818110, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0022b13e0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001430540})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000d06778, {0x235a640, 0xc000045ad0}, 0x392d623966342d35)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022b13e0}, 0xc001241240)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00180edb8, 0x4c1cb4}, {0xc057f6daf59c7831, 0x8c084fd17f}, {0xc00180edb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0022b13e0}, 0xc001241240)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0xc000ebb020, 0xc0023fd488)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0x15, 0xc001241280)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0x2076951, 0xc001241280)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0x237bc38, 0xc001241280)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0022b11a0}, {0x1f33320, 0xc001241240}, 0x1e5a940, 0xc001241280)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b11a0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0x23c270d, 0xc001241280)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002848b60, {0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0xc000ebb020, 0xc0023fd488)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0022b1170}, 0xc001f51920, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000a56b40, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000a56b40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 207 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062b340, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 209 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062b600, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 217 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 215 [select]:
database/sql.(*DB).connectionCleaner(0xc000bfa1a0, 0x0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 166 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000bfa1a0, {0x2392228, 0xc0009f4d00})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 226 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 175 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdd920fd8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000879080, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000879080)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000879080)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0006b61b0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0006b61b0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007c31e0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000791200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 227 [sleep]:
time.Sleep(0x342770c00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 225 [chan receive, 10 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0xc000cd07d0)
/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc000c74c80}, 0x0)
/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
/temporal/service/worker/fx.go:78 +0x9a
goroutine 1658063 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001e8dd40, {0x2392228, 0xc000e16940})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 232 [sleep]:
time.Sleep(0x28fa6ae00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 1657949 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353b90, {0x2392228, 0xc001318340})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 218 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1661119 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001e18480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2580, {0x1f7c860, 0xc0007d24d0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40f00, 0xc0017415e8, 0xc0017415b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40f00, {0x1f7c860, 0xc0007d24d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0022df440}, {0x20768da, 0x8}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x1c75901, {0xc001c64ec0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x7f00048b4108, {0xc001c64ec0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x20, 0xc002113380, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0xc000f5fa00, {0xc002120480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x0, 0xc002113350, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x1e0cfc0, 0xc0021132c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0022df440}, {0x20768da, 0x203000}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x8, 0xc002113290, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc0022df440}, {0x20768da, 0x0}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0022df440}, 0xc001155720, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000a4c500, {0x2392298, 0xc0022df440})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1678861 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:627 +0x152
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 548 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2840)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 545 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 543 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1658164 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002398720, {0x2392228, 0xc001171200})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1154 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc00205f8b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 270 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920410, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000708b00, 0xc0009e08c0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000708b00, {0xc0009e08c0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000708b00, {0xc0009e08c0, 0xc000a33d70, 0xc0012f18e0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f380, {0xc0009e08c0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f380}, {0xc0009e08c0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007b4840, 0x1fef77d)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 271 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007b4840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1480908 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001418d70)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1661103 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001f8f620, {0x2392298, 0xc0022de660}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d93320, {0x23922d0, 0xc0020fec00}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0020fec00}, 0xc0020ecff0, 0x1bd58c0, 0x20ed000)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0020feba0, 0xc0020e8e00)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0020fe900}, 0xc0020e8e00)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020fe900}, {0x1f353a0, 0xc0020e8e00})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0020fe900}, {0x1f353a0, 0xc0020e8e00}, 0x237bc38, 0xc002102228)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe900}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0020fe780, {0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00}, 0xc001bbd5b0, 0xc0020e8e40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00}, 0x40, 0xc0020e8e40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00}, 0xc001c64a60, 0xc002102228)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0020fe720}, 0xc0022de600, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000be6d80, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000be6d80, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661215 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b39c20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d3ef0, {0x1f7b560, 0xc001f4d540}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41800, 0xc002329d18, 0xc002329ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41800, {0x1f7b560, 0xc001f4d540})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x20, {0xc001af6120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0xc00138ef20, {0xc001af6120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0xc00119e640, 0xc002314d20, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002301440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x237ac70, 0xc002314c60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002301440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002301440}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc002301440}, 0xc001f14570, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0023145a0}, 0xc0023013c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0023145a0}, 0xc0023013c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00232a5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00232a648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8dbcf0, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0023145a0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0023102e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00232a898, {0x235a640, 0xc000045ad0}, 0x382d363738342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0023145a0}, 0xc0023143f0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001948db8, 0x4c1cb4}, {0xc057f6d9783c7194, 0x8aa54f0edd}, {0xc001948db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0023145a0}, 0xc0023143f0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0xc001af6020, 0xc00230c528)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0x15, 0xc002301180)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0x207690b, 0xc002301180)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0x237bc38, 0xc002301180)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002314450}, {0x1f33180, 0xc0023143f0}, 0x1e5a940, 0xc002301180)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002314450}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0x9, 0xc002301180)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0xc002972768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002972b60, {0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0xc001af6020, 0xc00230c528)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0023143c0}, 0xc00257b440, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001b38c60, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001b38c60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 549 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d28f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1158 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc001f1c3a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1002 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc001209e80)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1159 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc001379560)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1000 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc001eb4f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1001 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc0021bc2b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1003 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc002457580)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1004 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 902 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 903 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2a50)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 265 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920328, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000708b80, 0xc000553a00, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000708b80, {0xc000553a00, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000708b80, {0xc000553a00, 0xc000814f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f390, {0xc000553a00, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f390}, {0xc000553a00, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00079a000, 0x747100)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 266 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00079a000, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1311610 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000c19d60)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1483790 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002633560, {0x2392228, 0xc000efb340})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1153 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc0021848b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 238 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 239 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1658117 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fdbe60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116c160, {0x1f7c860, 0xc00116c0b0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40700, 0xc0008075e8, 0xc0008075b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40700, {0x1f7c860, 0xc00116c0b0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00196f320}, {0x20768da, 0x8}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x1c75901, {0xc000f80b80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x7f00048b4108, {0xc000f80b80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x20, 0xc002224060, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0xc000f5fa00, {0xc000ed2800, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x0, 0xc002224030, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x1e0cfc0, 0xc001303f50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00196f320}, {0x20768da, 0x203000}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x8, 0xc001303f20, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0xc00002be50, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc00196f320}, {0x20768da, 0x0}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc00196f320}, 0x10, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000688700, {0x2392298, 0xc00196f320})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1152 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc00139bd40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 904 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2b00)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 900 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1651760 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0023ed6e0, {0x2392228, 0xc0014034c0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658082 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001af8d80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac840, {0x1f7c860, 0xc001bac790}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d000, 0xc000c0f5e8, 0xc000c0f5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d000, {0x1f7c860, 0xc001bac790})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002429e00}, {0x20768da, 0x8}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x1c75901, {0xc000be5d20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x7f00048b45b8, {0xc000be5d20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x20, 0xc0022a8d80, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0xc000f5fa00, {0xc0012e1500, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, 0xc0022a8d50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x1e0cfc0, 0xc0022a8cc0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002429e00}, {0x20768da, 0x203000}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x8, 0xc0022a8c90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc002429e00}, {0x20768da, 0x0}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc002429e00}, 0x60, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000688700, {0x2392298, 0xc002429e00})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1829721 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006c4150, {0x2392228, 0xc001970880})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1657963 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019fc5a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e82420, {0x1f7b560, 0xc001b14e60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391a00, 0xc001c49d18, 0xc001c49ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391a00, {0x1f7b560, 0xc001b14e60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x20, {0xc00230b240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0xc001f27e40, {0xc00230b240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0xc00119e640, 0xc0023c6480, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012dbcc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x237ac70, 0xc0023c63c0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012dbcc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012dbcc0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012dbcc0}, 0xc00204c030, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc00125fcb0}, 0xc0012dbc40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc00125fcb0}, 0xc0012dbc40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001c4a5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c4a648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd819d38, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc00125fcb0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0022fbe20})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c4a898, {0x235a640, 0xc000045ad0}, 0x622d313637342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00125fcb0}, 0xc00125fa10)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0017d2db8, 0x4c1cb4}, {0xc057f6d935dbf36c, 0x8a6753c6be}, {0xc0017d2db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc00125fcb0}, 0xc00125fa10)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0xc00230b0a0, 0xc00205c978)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0x15, 0xc0012dba00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0x207690b, 0xc0012dba00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0x237bc38, 0xc0012dba00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00125fa70}, {0x1f33180, 0xc00125fa10}, 0x1e5a940, 0xc0012dba00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fa70}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0x9, 0xc0012dba00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0xc001bf5768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0xc00230b0a0, 0xc00205c978)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00125f9e0}, 0xc001fdd080, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c52c60, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c52c60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 311 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd8656d8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000688f80, 0xc0006e9a30, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000688f80, {0xc0006e9a30, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000688f80, {0xc0006e9a30, 0xc000a27f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa158, {0xc0006e9a30, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa158}, {0xc0006e9a30, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007e6b00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 1829691 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024ee6c0, {0x2392298, 0xc00241bb60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000bcafc0, {0x23922d0, 0xc000c65410}, 0x14)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002468540, {0x23922d0, 0xc000c65410}, 0xc0019ea670, 0x1, 0x19ea680)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc000c653b0, 0xc000f1e880)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc000c65110}, 0xc000f1e880)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000c65110}, {0x1f35540, 0xc000f1e880})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc000c65110}, {0x1f35540, 0xc000f1e880}, 0x237bc38, 0xc0000a29d8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65110}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880}, 0xc000f1e200, 0xc000f1e900)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880}, 0x40, 0xc000f1e900)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880}, 0xc000f81360, 0xc0000a29d8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc000c64f90}, 0xc00241ba40, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000cb6fc0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000cb6fc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1477993 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001112840, {0x2392228, 0xc002465600})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 781 [select, 10 minutes]:
net/http.(*persistConn).writeLoop(0xc001348a20)
/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1748 +0x1e65
goroutine 1677980 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c3a8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000b7d900, 0xc000d1c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000b7d900, {0xc000d1c000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000b7d900, {0xc000d1c000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278ebb0, {0xc000d1c000, 0x0, 0x7effdd8ca7f8})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0cb40, {0xc0006ba820, 0x9, 0xc000716400})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0cb40}, {0xc0006ba820, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba820, 0x9, 0xc001f91e30}, {0x23585c0, 0xc001c0cb40})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba7e0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000037e00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1658163 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353d50, {0x2392228, 0xc0011711c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1483788 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c8620, {0x2392228, 0xc000efb300})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1657980 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0022b78b0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 288 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd865080, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000709380, 0xc0007040a0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000709380, {0xc0007040a0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000709380, {0xc0007040a0, 0xc000a2bf50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f488, {0xc0007040a0, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f488}, {0xc0007040a0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00070eb00, 0x1ff4eb7)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 286 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd864eb0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000709400, 0xc0009e1fd0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000709400, {0xc0009e1fd0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000709400, {0xc0009e1fd0, 0xc0025cae70, 0xc0021e1b80})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f490, {0xc0009e1fd0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f490}, {0xc0009e1fd0, 0x10, 0x10}, 0x10)