Skip to content

Instantly share code, notes, and snippets.

@ngoclamtran

ngoclamtran/node-2

Created November 1, 2021 04:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ngoclamtran/ab179b0811b9332dbe9b45ed9f405f77 to your computer and use it in GitHub Desktop.
Save ngoclamtran/ab179b0811b9332dbe9b45ed9f405f77 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
node-2
This file has been truncated, but you can view the full file.
goroutine 1832008 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc0003280e0})
/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc0003280e0}, 0x0)
/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc0003280e0}, 0xc)
/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc001aedcf1, 0x34934e0}, {0x2386300, 0xc0003280e0}, 0xc001aedce4)
/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc0003280e0}, 0xc001447b00)
/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc0003280e0}, 0x0)
/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc001aedcfb, {0x2386300, 0xc0003280e0}, 0xc001447b00)
/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc00129b740}, {0x2386300, 0xc0003280e0}, 0xc001447b00)
/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc001fbd0e0, {0x23922d0, 0xc0005296e0})
/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3033 +0x4e8
goroutine 1 [chan receive, 10 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc000792000)
/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc000143240)
/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc000579b00, 0xc000143200)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc000082680, {0x2392260, 0xc0000b0000}, {0xc0000ae000, 0x8, 0x8})
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
/temporal/cmd/server/main.go:50 +0x45
goroutine 26 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00003a300)
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92
goroutine 28 [syscall, 10 minutes]:
os/signal.signal_recv()
/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x2c
goroutine 4 [chan receive, 10 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 5 [chan receive, 10 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 1471878 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000140af0, {0x2392228, 0xc001f078c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 63 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda99830, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000878380, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000878380)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000878380)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000324f18)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000324f18)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000084d80)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790000)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 97 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000569b80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 41 [select]:
database/sql.(*DB).connectionOpener(0xc0005305b0, {0x2392228, 0xc0006b33c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 60 [select]:
database/sql.(*DB).connectionCleaner(0xc0005305b0, 0xc0000884b0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 69 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 11 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda99a00, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003ab80, 0x203000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003ab80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003ab80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0003245b8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0003245b8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc000328540, {0x23860f0, 0xc0003245b8})
/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc000328540)
/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
/temporal/common/pprof/pprof.go:72 +0x9c
goroutine 64 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000098f70, {0x2392228, 0xc0007a8880})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 39 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000884b0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1658034 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005cbb00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac210, {0x1f7c860, 0xc001bac0b0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8c800, 0xc001b2d5e8, 0xc001b2d5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8c800, {0x1f7c860, 0xc001bac0b0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002428f00}, {0x20768da, 0x8}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x1c75901, {0xc000be4e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x7f00048b45b8, {0xc000be4e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x20, 0xc001f097a0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0xc000f5fa00, {0xc001f83a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x0, 0xc001f09770, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x1e0cfc0, 0xc001f096e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x4167af, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002428f00}, {0x20768da, 0x203000}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0x2e, 0xc001f096b0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002428f00}, {0x20768da, 0x46}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, 0xc001216e50, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc002428f00}, {0x20768da, 0x0}, {0x1f33180, 0xc001f09650}, {0x1f7c860, 0xc001bac0b0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc002428f00}, 0xc001216f20, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc002428f00})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658097 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001dcdc20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e836b0, {0x1f7b7c0, 0xc000330780}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40400, 0xc000aca4d0, 0xc000aca4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40400, {0x1f7b7c0, 0xc000330780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x20, {0xc001bc5b40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x0, {0xc001bc5b40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0xc000c61270, 0xc0020ac090, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fdfb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e64d40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x33, 0xc0020fde60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e64d40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e64d40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000e64c80}, {0x1f7b7c0, 0xc000330780}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc000e64d40}, 0xc000a64586, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc002643980}, 0xc000e64c80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc002643980}, 0xc000e64c80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002643680, {0x2392298, 0xc002643980})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002643920, {0x2392298, 0xc002643980}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001dcd680, {0x23922d0, 0xc0020fd080}, 0x2a)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc000ba4240, {0x23922d0, 0xc0020fd080}, 0xc0013d1420, 0x1, 0x13d1430)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0020fcff0, 0xc000e643c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0020fcc90}, 0xc000e643c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020fcc90}, {0x1f35540, 0xc000e643c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0020fcc90}, {0x1f35540, 0xc000e643c0}, 0x237bc38, 0xc0024e52a8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcc90}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0}, 0xc000e17e00, 0xc000e64440)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb40}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0}, 0x40, 0xc000e64440)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fcb10}, {0x1f35540, 0xc000e643c0}, 0xc001bc5980, 0xc0024e52a8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0020fcb10}, 0xc0026433e0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001dcc6c0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001dcc6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 81 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0006bd810)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 48 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda99748, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000878800, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000878800)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000878800)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0003253c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0003253c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0000d6fc0, {0x23860f0, 0xc0003253c8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc00002a7d0)
/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc0006bd6d0}, 0x0)
/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
/temporal/service/history/fx.go:168 +0x9a
goroutine 927 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd8655f0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00149c180, 0xc000ff3ea0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00149c180, {0xc000ff3ea0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00149c180, {0xc000ff3ea0, 0xc000a33d70, 0xc001297f20})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aaf68, {0xc000ff3ea0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aaf68}, {0xc000ff3ea0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007e7b80, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 83 [select]:
database/sql.(*DB).connectionOpener(0xc0006de8f0, {0x2392228, 0xc000a195c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 72 [select]:
database/sql.(*DB).connectionCleaner(0xc0006de8f0, 0xc0006bd810)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 74 [sleep]:
time.Sleep(0x342770c00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 928 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007e7b80, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1471880 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0024ee3c0, {0x2392228, 0xc001f07900})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 80 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda993a8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000879500, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000879500)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000879500)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000325b18)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000325b18)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0007a4540, {0x23860f0, 0xc000325b18})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc0007f3fd0)
/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc0007deab0}, 0x32)
/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
/temporal/service/matching/fx.go:129 +0x9a
goroutine 89 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdda992c0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872600, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000872600)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000872600)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000a248b8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000a248b8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc00053b8c0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790c00)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 101 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 1651782 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c53680)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e831e0, {0x1f7b560, 0xc001fbc640}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d900, 0xc000aaa440, 0xc000aaa410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d900, {0x1f7b560, 0xc001fbc640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x20, {0xc001bc4640, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0xc001de7080, {0xc001bc4640, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0xc0023f4400, 0xc001fd2510, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd24e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0017ed500}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x237ac70, 0xc001fd2450, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0017ed500}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0017ed500}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0017ed480}, {0x1f7b560, 0xc001fbc640}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0017ed500}, 0xc001eab006, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc0025717a0}, 0xc0017ed480, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc0025717a0}, 0xc0017ed480, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0012e8b40, {0x2392298, 0xc0025717a0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0012e9320, {0x2392298, 0xc0025717a0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000bbec60, {0x23922d0, 0xc001fd2150}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fd2150}, 0xc001438db0, 0x1bd58c0, 0x1438dc0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001fd20f0, 0xc0017ed280)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001e9be30}, 0xc0017ed280)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e9be30}, {0x1f353a0, 0xc0017ed280})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001e9be30}, {0x1f353a0, 0xc0017ed280}, 0x237bc38, 0xc002004630)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9be30}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001e9bcb0, {0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280}, 0xc001e9aed0, 0xc0017ed2c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bce0}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280}, 0x40, 0xc0017ed2c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e9bc50}, {0x1f353a0, 0xc0017ed280}, 0xc001bc45a0, 0xc002004630)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001e9bc50}, 0xc002571740, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c52900, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c52900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 92 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8
goroutine 93 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 94 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1678726 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000e88840, {0x2392298, 0xc001aea540}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000f70480, {0x23922d0, 0xc0022eecc0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002477050, {0x23922d0, 0xc0022eecc0}, 0xc00124d000, 0x1, 0x124d010)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0022eec60, 0xc001250600)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022ee9f0}, 0xc001250600)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022ee9f0}, {0x1f35540, 0xc001250600})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022ee9f0}, {0x1f35540, 0xc001250600}, 0x237bc38, 0xc0021ca930)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee9f0}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600}, 0xc001250000, 0xc001250680)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee840}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600}, 0x40, 0xc001250680)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022ee7e0}, {0x1f35540, 0xc001250600}, 0xc000ebb5c0, 0xc0021ca930)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0022ee7e0}, 0xc001aea4e0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000fe8240, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000fe8240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678773 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001dcd320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0160, {0x1f7b7c0, 0xc001f4d720}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078b900, 0xc000f49bf8, 0xc000f49bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078b900, {0x1f7b7c0, 0xc001f4d720})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x20, {0xc001c648e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0xc0020af600, {0xc001c648e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0xc00119e640, 0xc0022ff6e0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022ff6b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012cc240}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x237ac70, 0xc0022ff620, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012cc240}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012cc240}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012cc1c0}, {0x1f7b7c0, 0xc001f4d720}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012cc240}, 0xc0022013c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022fee70}, 0xc0012cc1c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022fee70}, 0xc0012cc1c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000f4a480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000f4a528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd817890, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0022fee70}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00119e7e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000f4a778, {0x235a640, 0xc000045ad0}, 0x612d623139342d30)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022fee70}, 0xc001283ec0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001726db8, 0x4c1cb4}, {0xc057f6daf65f8b90, 0x8c0912e4da}, {0xc001726db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0022fee70}, 0xc001283ec0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0xc001c647a0, 0xc00237ca68)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0x15, 0xc001283f00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0x2076951, 0xc001283f00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0}, 0x237bc38, 0xc001283f00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fee70}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0022fec00}, {0x1f33320, 0xc001283ec0}, 0x1e5a940, 0xc001283f00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fec00}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0xc0022012c0, 0xc001283f00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022feb10}, {0x1f33320, 0xc001283ec0}, 0xc001c647a0, 0xc00237ca68)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0022feb10}, 0xc0020a6780, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbb9e0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbb9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1677974 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd9205e0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ca9900, 0xc000d26000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ca9900, {0xc000d26000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ca9900, {0xc000d26000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278eb88, {0xc000d26000, 0x0, 0x7effdd8ca7f8})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0c7e0, {0xc0006ba3c0, 0x9, 0xc00005e800})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0c7e0}, {0xc0006ba3c0, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba3c0, 0x9, 0xc0012e5650}, {0x23585c0, 0xc001c0c7e0})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba380)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000ab40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1657967 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019fd8c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e824d0, {0x1f7b560, 0xc001b154a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391b00, 0xc00198c440, 0xc00198c410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391b00, {0x1f7b560, 0xc001b154a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x20, {0xc00230b520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0xc001ffd4a0, {0xc00230b520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0xc0023f4400, 0xc0023c77a0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c7770}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0024feb40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x237ac70, 0xc0023c76b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0024feb40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0024feb40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0024feac0}, {0x1f7b560, 0xc001b154a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0024feb40}, 0xc0024a78f6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc001fdd500}, 0xc0024feac0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc001fdd500}, 0xc0024feac0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001f8f200, {0x2392298, 0xc001fdd500})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001f8f620, {0x2392298, 0xc001fdd500}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d93320, {0x23922d0, 0xc0023c6f90}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0023c6f90}, 0xc00139b080, 0x1bd58c0, 0x139b090)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0023c6f30, 0xc0024fe180)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0023c6c90}, 0xc0024fe180)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023c6c90}, {0x1f353a0, 0xc0024fe180})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0023c6c90}, {0x1f353a0, 0xc0024fe180}, 0x237bc38, 0xc00205cee8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6c90}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0023c6b10, {0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180}, 0x4167af, 0xc0024fe240)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6b40}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180}, 0x40, 0xc0024fe240)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023c6ab0}, {0x1f353a0, 0xc0024fe180}, 0xc00230b360, 0xc00205cee8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0023c6ab0}, 0xc001fdd4a0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019fc6c0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019fc6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 105 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000776410, {0x2392228, 0xc000021540})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 109 [sleep]:
time.Sleep(0x2cb417800)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 1658165 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda99490, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001301000, 0xc000c96000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001301000, {0xc000c96000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001301000, {0xc000c96000, 0xc001301018, 0xc000c96000})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c15bd0, {0xc000c96000, 0x448e71, 0x466a4e})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc0000beb40, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc0000beb40, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc0000beb40)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlConn).readResultSetHeaderPacket(0xc0023e5d10)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:537 +0x25
github.com/go-sql-driver/mysql.(*mysqlStmt).query(0xc0023e5d10, {0xc000e969b0, 0xc0012f9da0, 0xc000e9a650})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/statement.go:108 +0x125
github.com/go-sql-driver/mysql.(*mysqlStmt).QueryContext(0xc0023e5d10, {0x2392298, 0xc0012f9da0}, {0xc000e9a5b0, 0x94, 0x208f76a})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:565 +0x91
database/sql.ctxDriverStmtQuery({0x2392298, 0xc0012f9da0}, {0x2392458, 0xc0023e5d10}, {0xc000e9a5b0, 0x5, 0x5})
/usr/local/go/src/database/sql/ctxutil.go:82 +0x132
database/sql.rowsiFromStatement({0x2392298, 0xc0012f9da0}, {0x237efb0, 0xc0000beb40}, 0xc000e8d200, {0xc000dffdb0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:2794 +0x17e
database/sql.(*DB).queryDC(0x203001, {0x2392298, 0xc0012f9da0}, {0x0, 0x0}, 0xc0013974d0, 0xc00127dfc0, {0x208f76a, 0x94}, {0xc000dffdb0, ...})
/usr/local/go/src/database/sql/sql.go:1752 +0x3d7
database/sql.(*DB).query(0x1947548, {0x2392298, 0xc0012f9da0}, {0x208f76a, 0x94}, {0xc000dffdb0, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0012f9da0}, {0x208f76a, 0x94}, {0xc000dffdb0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0012f9da0}, {0x208f76a, 0xc00004c000}, {0xc000dffdb0, 0xc0019476c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0012f9da0}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0023e59e0}, {0x208f76a, 0xc000ee9400}, {0xc000dffdb0, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2d, {0x2392298, 0xc0012f9da0}, {0x1b5d8e0, 0xc0023e59e0}, {0x208f76a, 0xc0019477e0}, {0xc000dffdb0, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0012f9da0}, {0xd070a498, {0xc0013c8880, 0x3e, 0x3e}, 0x0, 0xc000e8cbe8, 0xc00245f358, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e8cbc0)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x2d)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0023985a0, 0x3f8ff900, 0x3f917fa0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc000716400, 0xc0012eeba0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001947c50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc001947d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001947de2, 0x4f8639)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001947d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc002398720)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002398720, {0x2392228, 0xc001171200})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658032 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000409560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e83290, {0x1f7b7c0, 0xc001fbce60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8cf00, 0xc0026ae4d0, 0xc0026ae4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8cf00, {0x1f7b7c0, 0xc001fbce60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x20, {0xc001bc4b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0xc0008529a0, {0xc001bc4b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0xc000c61270, 0xc0024ccea0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024cce70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001366d80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x237ac70, 0xc0024ccde0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001366d80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001366d80}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001366cc0}, {0x1f7b7c0, 0xc001fbce60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc001366d80}, 0xc00204d296, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc001e8ccc0}, 0xc001366cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc001e8ccc0}, 0xc001366cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001e8ca20, {0x2392298, 0xc001e8ccc0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001e8cc60, {0x2392298, 0xc001e8ccc0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001923560, {0x23922d0, 0xc0024cc210}, 0x1e)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc00204d230, {0x23922d0, 0xc0024cc210}, 0xc00141b170, 0x1, 0x141b180)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0024cc1b0, 0xc001366400)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0024dbef0}, 0xc001366400)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0024dbef0}, {0x1f35540, 0xc001366400})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0024dbef0}, {0x1f35540, 0xc001366400}, 0x237bc38, 0xc002288588)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbef0}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0024da960, {0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400}, 0xc00245baa0, 0xc001366480)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbcb0}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400}, 0x40, 0xc001366480)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0024dbc80}, {0x1f35540, 0xc001366400}, 0xc001bc48c0, 0xc002288588)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0024dbc80}, 0xc001e8c780, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001922900, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001922900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1654705 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005d6480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116dad0, {0x1f7b7c0, 0xc001fbcfa0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8dd00, 0xc0026a9bf8, 0xc0026a9bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8dd00, {0x1f7b7c0, 0xc001fbcfa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x20, {0xc001bc5f20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0xc001470160, {0xc001bc5f20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0xc00119e640, 0xc0021ad950, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ad920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021aac40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x237ac70, 0xc0021ad890, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021aac40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021aac40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0021aabc0}, {0x1f7b7c0, 0xc001fbcfa0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0021aac40}, 0xc001e163f0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0021ad1d0}, 0xc0021aabc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0021ad1d0}, 0xc0021aabc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0026aa480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0026aa528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd825670, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0021ad1d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021b4250})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0026aa778, {0x235a640, 0xc000045ad0}, 0x382d366530342d61)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0021ad1d0}, 0xc0021aa8c0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000b5cdb8, 0x4c1cb4}, {0xc057f6d8f59f3f24, 0x8a2b7c486f}, {0xc000b5cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0021ad1d0}, 0xc0021aa8c0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0xc001bc5ce0, 0xc0021ba5e8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0x15, 0xc0021aa900)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0x2076951, 0xc0021aa900)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0}, 0x237bc38, 0xc0021aa900)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad1d0}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021ad080}, {0x1f33320, 0xc0021aa8c0}, 0x1e5a940, 0xc0021aa900)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad080}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0x23c27b0, 0xc0021aa900)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bbdb60, {0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021ad050}, {0x1f33320, 0xc0021aa8c0}, 0xc001bc5ce0, 0xc0021ba5e8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0021ad050}, 0xc002362480, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c1eea0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c1eea0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1654718 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001af9680)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116def0, {0x1f7b7c0, 0xc001fbd7c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446b00, 0xc00196a4d0, 0xc00196a4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446b00, {0x1f7b7c0, 0xc001fbd7c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x20, {0xc00230a1e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0xc000138580, {0xc00230a1e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0xc0023f4400, 0xc0021c5ef0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021c5ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021c3040}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x237ac70, 0xc0021c5e30, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021c3040}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021c3040}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0021c2f80}, {0x1f7b7c0, 0xc001fbd7c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc0021c3040}, 0xc0026f7536, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc0023637a0}, 0xc0021c2f80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc0023637a0}, 0xc0021c2f80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0026bca20, {0x2392298, 0xc0023637a0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0026bcc60, {0x2392298, 0xc0023637a0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00112e120, {0x23922d0, 0xc0021c5c20}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001e17020, {0x23922d0, 0xc0021c5c20}, 0xc0021c82d0, 0x1, 0x21c82e0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0021c5bc0, 0xc0021c2d40)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0021c5920}, 0xc0021c2d40)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021c5920}, {0x1f35540, 0xc0021c2d40})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0021c5920}, {0x1f35540, 0xc0021c2d40}, 0x237bc38, 0xc0021bbea8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c5920}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40}, 0xc0021c2780, 0xc0021c2dc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57d0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40}, 0x40, 0xc0021c2dc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021c57a0}, {0x1f35540, 0xc0021c2d40}, 0xc00230a100, 0xc0021bbea8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0021c57a0}, 0xc002363740, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001af86c0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001af86c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678784 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077f440)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae02c0, {0x1f7b7c0, 0xc001f4da40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078bc00, 0xc001befbf8, 0xc001befbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078bc00, {0x1f7b7c0, 0xc001f4da40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x20, {0xc001c64e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0xc001470000, {0xc001c64e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0xc00119e640, 0xc002445aa0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002445a70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012cd3c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x237ac70, 0xc0024459e0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012cd3c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012cd3c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012cd340}, {0x1f7b7c0, 0xc001f4da40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012cd3c0}, 0xc0022016c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc002445350}, 0xc0012cd340, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc002445350}, 0xc0012cd340, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001bf0480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bf0528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd817890, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002445350}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00119ecb0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bf0778, {0x235a640, 0xc000045ad0}, 0x612d376365342d34)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002445350}, 0xc0012cd040)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00194adb8, 0x4c1cb4}, {0xc057f6daf69ccde6, 0x8c09502730}, {0xc00194adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc002445350}, 0xc0012cd040)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0xc001c64ce0, 0xc00237d6f8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0x15, 0xc0012cd080)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0x2076951, 0xc0012cd080)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040}, 0x237bc38, 0xc0012cd080)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445350}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002445200}, {0x1f33320, 0xc0012cd040}, 0x1e5a940, 0xc0012cd080)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002445200}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0x23c27b0, 0xc0012cd080)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0024451d0}, {0x1f33320, 0xc0012cd040}, 0xc001c64ce0, 0xc00237d6f8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0024451d0}, 0xc0020a7380, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bc5200, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bc5200, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658051 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001e8c960, {0x2392228, 0xc001366c00})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 811 [select]:
go.temporal.io/server/service/history/shard.(*ControllerImpl).shardManagementPump(0xc000cd74a0)
/temporal/service/history/shard/controller_impl.go:312 +0x148
created by go.temporal.io/server/service/history/shard.(*ControllerImpl).Start
/temporal/service/history/shard/controller_impl.go:145 +0xaf
goroutine 1658141 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019758c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bacdc0, {0x1f7b560, 0xc001941680}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb9)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d800, 0xc001bd7d18, 0xc001bd7ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d800, {0x1f7b560, 0xc001941680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002127440}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x20, {0xc000eba2a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002127440}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002127440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0xc002084420, {0xc000eba2a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002127440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0xc00119e640, 0xc002127470, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002127440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e91fc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x237ac70, 0xc0021273b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e91fc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e91fc0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e91f40}, {0x1f7b560, 0xc001941680}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc000e91fc0}, 0xc002688f90, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc002126c90}, 0xc000e91f40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc002126c90}, 0xc000e91f40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001bd85a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bd8648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8c47b0, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc002126c90}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0023642e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bd8898, {0x235a640, 0xc000045ad0}, 0x612d643133342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002126c90}, 0xc002126ae0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001946db8, 0x4c1cb4}, {0xc057f6d9391f219f, 0x8a6a96f4ef}, {0xc001946db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc002126c90}, 0xc002126ae0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0xc000eba1e0, 0xc00209b218)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0x15, 0xc000e91d00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0x207690b, 0xc000e91d00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0}, 0x237bc38, 0xc000e91d00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126c90}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002126b40}, {0x1f33180, 0xc002126ae0}, 0x1e5a940, 0xc000e91d00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126b40}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0x9, 0xc000e91d00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0xc002888768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002888b60, {0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002126ab0}, {0x1f33180, 0xc002126ae0}, 0xc000eba1e0, 0xc00209b218)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc002126ab0}, 0xc002549e00, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001974a20, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001974a20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1586529 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920c38, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00132c880, 0xc0013f2000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00132c880, {0xc0013f2000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00132c880, {0xc0013f2000, 0x1060100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278f8f8, {0xc0013f2000, 0xc002847cf8, 0x44d2f4})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001ed2360, {0xc0006ba040, 0x9, 0x9772e782d4})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001ed2360}, {0xc0006ba040, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba040, 0x9, 0xc001331b30}, {0x23585c0, 0xc001ed2360})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba000)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001258600, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a41c0, {0x23d1088, 0xc001258600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 1657986 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023aafc0, {0x2392298, 0xc001fdd920}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001127200, {0x23922d0, 0xc001f123f0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001f123f0}, 0xc00139bb40, 0x1bd58c0, 0x139bb50)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001f121e0, 0xc0024fee00)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0023c7ef0}, 0xc0024fee00)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023c7ef0}, {0x1f353a0, 0xc0024fee00})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0023c7ef0}, {0x1f353a0, 0xc0024fee00}, 0x237bc38, 0xc00205d608)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7ef0}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0023c6b10, {0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00}, 0x4167af, 0xc0024fee40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7d10}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00}, 0x40, 0xc0024fee40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023c7cb0}, {0x1f353a0, 0xc0024fee00}, 0xc00230b640, 0xc00205d608)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0023c7cb0}, 0xc001fdd8c0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0019fd9e0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0019fd9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 140 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000c7a1a0, {0x2392228, 0xc000c76340})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 102 [select]:
database/sql.(*DB).connectionCleaner(0xc000c7a1a0, 0x0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 104 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdd95c748, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4da00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000a4da00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000a4da00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000bc10c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000bc10c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007c29a0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000354200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 148 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda990f0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4ce00, 0xc000c61040, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a4ce00, {0xc000c61040, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000a4ce00, {0xc000c61040, 0xc000c00f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14d08, {0xc000c61040, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14d08}, {0xc000c61040, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062a840, 0x1c)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 146 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c660, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4d480, 0xc000c60e40, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a4d480, {0xc000c60e40, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000a4d480, {0xc000c60e40, 0xc0025cae70, 0xc002114b40})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14d10, {0xc000c60e40, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14d10}, {0xc000c60e40, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062a580, 0xc8e090)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 147 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062a580, 0x790c00)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 149 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062a840, 0x1fe52a7)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 156 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 1677984 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001259980)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837
goroutine 350 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c630)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 164 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000893b0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 163 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c578, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4dd00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000a4dd00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000a4dd00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000bc13c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000bc13c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0007a41c0, {0x23860f0, 0xc000bc13c8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0xc000a6cfd0)
/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc00003a700}, 0x1c)
/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
/temporal/service/frontend/fx.go:282 +0x9a
goroutine 157 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 348 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 1658026 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ae37a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e83130, {0x1f7b560, 0xc001fbc780}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8c500, 0xc0017cbd18, 0xc0017cbce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8c500, {0x1f7b560, 0xc001fbc780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0024da510}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x20, {0xc001bc45a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0024da510}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024da510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0xc000139e40, {0xc001bc45a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024da510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0xc00119e640, 0xc0024da540, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0024da510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0024cbc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x237ac70, 0xc0024da480, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0024cbc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0024cbc40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0024cbbc0}, {0x1f7b560, 0xc001fbc780}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0024cbc40}, 0xc00204cde0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc00245bc50}, 0xc0024cbbc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc00245bc50}, 0xc0024cbbc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017cc5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017cc648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8c2280, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc00245bc50}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000c61170})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017cc898, {0x235a640, 0xc000045ad0}, 0x392d326638342d36)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00245bc50}, 0xc00245baa0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00172edb8, 0x4c1cb4}, {0xc057f6d936dace3f, 0x8a6852a189}, {0xc00172edb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc00245bc50}, 0xc00245baa0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0xc001bc4400, 0xc0019bb830)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0x15, 0xc0024cb980)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0x207690b, 0xc0024cb980)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0}, 0x237bc38, 0xc0024cb980)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bc50}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00245bb00}, {0x1f33180, 0xc00245baa0}, 0x1e5a940, 0xc0024cb980)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245bb00}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0x9, 0xc0024cb980)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0xc001bf5768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00245ba70}, {0x1f33180, 0xc00245baa0}, 0xc001bc4400, 0xc0019bb830)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00245ba70}, 0xc001e8c120, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001ae2120, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001ae2120, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657951 [select]:
database/sql.(*DB).conn(0xc0006de8f0, {0x2392298, 0xc0021d2de0}, 0x1)
/usr/local/go/src/database/sql/sql.go:1312 +0x413
database/sql.(*DB).query(0xa3f548, {0x2392298, 0xc0021d2de0}, {0x208f76a, 0x94}, {0xc000e5e370, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1695 +0x5d
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0021d2de0}, {0x208f76a, 0x94}, {0xc000e5e370, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0021d2de0}, {0x208f76a, 0xc00004e800}, {0xc000e5e370, 0xc000a3f6c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0021d2de0}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0025f1500}, {0x208f76a, 0xc001aedc00}, {0xc000e5e370, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2b, {0x2392298, 0xc0021d2de0}, {0x1b5d8e0, 0xc0025f1500}, {0x208f76a, 0xc000a3f7e0}, {0xc000e5e370, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0021d2de0}, {0xdb6a293a, {0xc001db9980, 0x3c, 0x3c}, 0x0, 0xc000e3ffa8, 0xc00210ed58, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e3ff80)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x2b)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0023ec4e0, 0x3e8b0c20, 0x3e8c92c0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc00005e400, 0xc00133a000})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000a3fc50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc000a3fd10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc000a3fde2, 0xc00005e400)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc000a3fd60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc0023ec660)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0023ec660, {0x2392228, 0xc001318380})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 429 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 206 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95ba98, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000bc7a00, 0xc000041960, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000bc7a00, {0xc000041960, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000bc7a00, {0xc000041960, 0xc000ce1f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14e18, {0xc000041960, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14e18}, {0xc000041960, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062b340, 0x7e0fb0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 213 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 208 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c1d8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872500, 0xc000041a90, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000872500, {0xc000041a90, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000872500, {0xc000041a90, 0xc000ce2f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14e10, {0xc000041a90, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000c14e10}, {0xc000041a90, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00062b600, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 351 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c6e0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 346 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 347 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 418 [select, 10 minutes]:
go.temporal.io/server/service/history/events.(*NotifierImpl).dequeueHistoryEventNotifications(0xc000830cd0)
/temporal/service/history/events/notifier.go:234 +0x105
created by go.temporal.io/server/service/history/events.(*NotifierImpl).Start
/temporal/service/history/events/notifier.go:253 +0x71
goroutine 417 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920240, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000d5b480, 0xc000f33000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000d5b480, {0xc000f33000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000d5b480, {0xc000f33000, 0xc000d5b480, 0xc0017af170})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c15098, {0xc000f33000, 0xc0017af1f8, 0x677dc5})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc000f719e0, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc000f719e0, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc000f719e0)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlConn).readResultSetHeaderPacket(0xc000f719e0)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:537 +0x25
github.com/go-sql-driver/mysql.(*mysqlConn).query(0xc000f719e0, {0x207fcf5, 0xc000ba2420}, {0x34c4b38, 0x1bd5ac0, 0x0})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:390 +0xb4
github.com/go-sql-driver/mysql.(*mysqlConn).QueryContext(0xc000f719e0, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x4e}, {0x34c4b38, 0x45e285, 0xc00072c600})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:512 +0xb2
database/sql.ctxDriverQuery({0x2392298, 0xc000ba2420}, {0x7effdd943e68, 0xc000f719e0}, {0x0, 0x0}, {0x207fcf5, 0x0}, {0x34c4b38, 0x0, ...})
/usr/local/go/src/database/sql/ctxutil.go:48 +0x17d
database/sql.(*DB).queryDC.func1()
/usr/local/go/src/database/sql/sql.go:1722 +0x175
database/sql.withLock({0x237b030, 0xc000f37710}, 0xc00072c6d0)
/usr/local/go/src/database/sql/sql.go:3396 +0x8c
database/sql.(*DB).queryDC(0x1, {0x2392298, 0xc000ba2420}, {0x0, 0x0}, 0xc000f37710, 0xc0024be030, {0x207fcf5, 0x4e}, {0x0, ...})
/usr/local/go/src/database/sql/sql.go:1717 +0x211
database/sql.(*DB).query(0x2, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x4e}, {0x0, 0x0, 0x0}, 0x0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x10000c000e96788, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x4e}, {0x0, 0x0, 0x0})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryRowxContext(0xc0006f6240, {0x2392298, 0xc000ba2420}, {0x207fcf5, 0x2392260}, {0x0, 0x1000003493960, 0xc000ba2420})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:171 +0x45
github.com/jmoiron/sqlx.GetContext({0x2392298, 0xc000ba2420}, {0x2380120, 0xc0006f6240}, {0x1b8e560, 0xc00245f638}, {0x207fcf5, 0x4f9101}, {0x0, 0x0, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:82 +0x6b
github.com/jmoiron/sqlx.(*DB).GetContext(0x0, {0x2392298, 0xc000ba2420}, {0x1b8e560, 0xc00245f638}, {0x207fcf5, 0x4f94ca}, {0x0, 0x0, 0x0})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:147 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromNamespaceMetadata(0xc000568a00, {0x2392298, 0xc000ba2420})
/temporal/common/persistence/sql/sqlplugin/mysql/namespace.go:216 +0x7d
go.temporal.io/server/common/persistence/sql.(*sqlMetadataManagerV2).GetMetadata(0xc00087bd40)
/temporal/common/persistence/sql/metadata.go:222 +0x8a
go.temporal.io/server/common/persistence.(*metadataManagerImpl).GetMetadata(0xc0006bfe90)
/temporal/common/persistence/metadata_manager.go:186 +0x22
go.temporal.io/server/common/persistence.(*metadataRateLimitedPersistenceClient).GetMetadata(0xc00087bd70)
/temporal/common/persistence/persistenceRateLimitedClients.go:584 +0x42
go.temporal.io/server/common/persistence.(*metadataPersistenceClient).GetMetadata(0xc00087bda0)
/temporal/common/persistence/persistenceMetricClients.go:822 +0x73
go.temporal.io/server/common/namespace.(*registry).refreshNamespaces(0xc00016ac00, {0xc0019dddb8, 0x438672})
/temporal/common/namespace/registry.go:382 +0x42
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016ac00, {0x2392228, 0xc000ea6dc0})
/temporal/common/namespace/registry.go:359 +0x1b3
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1165 [select, 10 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a1fd60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1477857 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0026bc960, {0x2392228, 0xc001c33f40})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654883 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005ca900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116cdc0, {0x1f7b560, 0xc001de39a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41c00, 0xc001720440, 0xc001720410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41c00, {0x1f7b560, 0xc001de39a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x20, {0xc0022b47a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0xc001f26580, {0xc0022b47a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0xc0023f4400, 0xc001b9b050, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b9b020}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0023a8100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x237ac70, 0xc001b9af90, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0023a8100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0023a8100}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0023a8080}, {0x1f7b560, 0xc001de39a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0023a8100}, 0xc001db9686, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc002244f60}, 0xc0023a8080, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc002244f60}, 0xc0023a8080, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc000f96360, {0x2392298, 0xc002244f60})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000f965a0, {0x2392298, 0xc002244f60}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001923680, {0x23922d0, 0xc002255aa0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002255aa0}, 0xc002256730, 0x1bd58c0, 0x2256740)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc002255a40, 0xc00223bf40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022557a0}, 0xc00223bf40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022557a0}, {0x1f353a0, 0xc00223bf40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022557a0}, {0x1f353a0, 0xc00223bf40}, 0x237bc38, 0xc002241218)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022557a0}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002255620, {0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40}, 0xc02be2, 0xc00223bf80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002255650}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40}, 0x40, 0xc00223bf80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022555c0}, {0x1f353a0, 0xc00223bf40}, 0xc000a70ce0, 0xc002241218)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0022555c0}, 0xc002244f00, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001974240, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001974240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1480931 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000379650, {0x2392228, 0xc001421840})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1678720 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1e480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80b00, {0x1f7b7c0, 0xc001b15900}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446000, 0xc000d05bf8, 0xc000d05bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446000, {0x1f7b7c0, 0xc001b15900})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x20, {0xc000ebb120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0xc001f27760, {0xc000ebb120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0xc00119e640, 0xc0022b1ec0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b1e90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012415c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x237ac70, 0xc0022b1dd0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012415c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012415c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001241540}, {0x1f7b7c0, 0xc001b15900}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012415c0}, 0xc00210a000, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022b13e0}, 0xc001241540, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022b13e0}, 0xc001241540, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000d06480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000d06528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd818110, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0022b13e0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001430540})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000d06778, {0x235a640, 0xc000045ad0}, 0x392d623966342d35)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022b13e0}, 0xc001241240)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00180edb8, 0x4c1cb4}, {0xc057f6daf59c7831, 0x8c084fd17f}, {0xc00180edb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0022b13e0}, 0xc001241240)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0xc000ebb020, 0xc0023fd488)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0x15, 0xc001241280)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0x2076951, 0xc001241280)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240}, 0x237bc38, 0xc001241280)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b13e0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0022b11a0}, {0x1f33320, 0xc001241240}, 0x1e5a940, 0xc001241280)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b11a0}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0x23c270d, 0xc001241280)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002848b60, {0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022b1170}, {0x1f33320, 0xc001241240}, 0xc000ebb020, 0xc0023fd488)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0022b1170}, 0xc001f51920, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000a56b40, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000a56b40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 207 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062b340, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 209 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00062b600, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 217 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 215 [select]:
database/sql.(*DB).connectionCleaner(0xc000bfa1a0, 0x0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 166 [select, 10 minutes]:
database/sql.(*DB).connectionOpener(0xc000bfa1a0, {0x2392228, 0xc0009f4d00})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 226 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 175 [IO wait, 10 minutes]:
internal/poll.runtime_pollWait(0x7effdd920fd8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000879080, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000879080)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000879080)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0006b61b0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0006b61b0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007c31e0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000791200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 227 [sleep]:
time.Sleep(0x342770c00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 225 [chan receive, 10 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0xc000cd07d0)
/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc000c74c80}, 0x0)
/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
/temporal/service/worker/fx.go:78 +0x9a
goroutine 1658063 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001e8dd40, {0x2392228, 0xc000e16940})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 232 [sleep]:
time.Sleep(0x28fa6ae00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 1657949 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353b90, {0x2392228, 0xc001318340})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 218 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1661119 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001e18480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2580, {0x1f7c860, 0xc0007d24d0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40f00, 0xc0017415e8, 0xc0017415b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40f00, {0x1f7c860, 0xc0007d24d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0022df440}, {0x20768da, 0x8}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x1c75901, {0xc001c64ec0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x7f00048b4108, {0xc001c64ec0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x20, 0xc002113380, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0xc000f5fa00, {0xc002120480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x0, 0xc002113350, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x1e0cfc0, 0xc0021132c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0022df440}, {0x20768da, 0x203000}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x8, 0xc002113290, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0022df440}, {0x20768da, 0x46}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc0022df440}, {0x20768da, 0x0}, {0x1f33180, 0xc002113230}, {0x1f7c860, 0xc0007d24d0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0022df440}, 0xc001155720, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000a4c500, {0x2392298, 0xc0022df440})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1678861 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:627 +0x152
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 548 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2840)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 545 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 543 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1658164 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002398720, {0x2392228, 0xc001171200})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1154 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc00205f8b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 270 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920410, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000708b00, 0xc0009e08c0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000708b00, {0xc0009e08c0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000708b00, {0xc0009e08c0, 0xc000a33d70, 0xc0012f18e0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f380, {0xc0009e08c0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f380}, {0xc0009e08c0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007b4840, 0x1fef77d)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 271 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007b4840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1480908 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001418d70)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1661103 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001f8f620, {0x2392298, 0xc0022de660}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d93320, {0x23922d0, 0xc0020fec00}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0020fec00}, 0xc0020ecff0, 0x1bd58c0, 0x20ed000)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0020feba0, 0xc0020e8e00)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0020fe900}, 0xc0020e8e00)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020fe900}, {0x1f353a0, 0xc0020e8e00})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0020fe900}, {0x1f353a0, 0xc0020e8e00}, 0x237bc38, 0xc002102228)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe900}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0020fe780, {0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00}, 0xc001bbd5b0, 0xc0020e8e40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe7b0}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00}, 0x40, 0xc0020e8e40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fe720}, {0x1f353a0, 0xc0020e8e00}, 0xc001c64a60, 0xc002102228)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0020fe720}, 0xc0022de600, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000be6d80, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000be6d80, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661215 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b39c20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d3ef0, {0x1f7b560, 0xc001f4d540}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41800, 0xc002329d18, 0xc002329ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41800, {0x1f7b560, 0xc001f4d540})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x20, {0xc001af6120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0xc00138ef20, {0xc001af6120, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0xc00119e640, 0xc002314d20, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002314cf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002301440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x237ac70, 0xc002314c60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002301440}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002301440}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0023013c0}, {0x1f7b560, 0xc001f4d540}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc002301440}, 0xc001f14570, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0023145a0}, 0xc0023013c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0023145a0}, 0xc0023013c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00232a5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00232a648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8dbcf0, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0023145a0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0023102e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00232a898, {0x235a640, 0xc000045ad0}, 0x382d363738342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0023145a0}, 0xc0023143f0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001948db8, 0x4c1cb4}, {0xc057f6d9783c7194, 0x8aa54f0edd}, {0xc001948db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0023145a0}, 0xc0023143f0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0xc001af6020, 0xc00230c528)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0x15, 0xc002301180)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0x207690b, 0xc002301180)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0}, 0x237bc38, 0xc002301180)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023145a0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002314450}, {0x1f33180, 0xc0023143f0}, 0x1e5a940, 0xc002301180)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002314450}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0x9, 0xc002301180)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0xc002972768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002972b60, {0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023143c0}, {0x1f33180, 0xc0023143f0}, 0xc001af6020, 0xc00230c528)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0023143c0}, 0xc00257b440, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001b38c60, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001b38c60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 549 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d28f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1158 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc001f1c3a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1002 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc001209e80)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1159 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc001379560)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1000 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc001eb4f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1001 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc0021bc2b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1003 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc002457580)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1004 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 902 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 903 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2a50)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 265 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920328, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000708b80, 0xc000553a00, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000708b80, {0xc000553a00, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000708b80, {0xc000553a00, 0xc000814f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f390, {0xc000553a00, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f390}, {0xc000553a00, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00079a000, 0x747100)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 266 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00079a000, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1311610 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000c19d60)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1483790 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002633560, {0x2392228, 0xc000efb340})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1153 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc0021848b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 238 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 239 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1658117 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fdbe60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116c160, {0x1f7c860, 0xc00116c0b0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40700, 0xc0008075e8, 0xc0008075b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40700, {0x1f7c860, 0xc00116c0b0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00196f320}, {0x20768da, 0x8}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x1c75901, {0xc000f80b80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x7f00048b4108, {0xc000f80b80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x20, 0xc002224060, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0xc000f5fa00, {0xc000ed2800, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x0, 0xc002224030, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x1e0cfc0, 0xc001303f50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00196f320}, {0x20768da, 0x203000}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0x8, 0xc001303f20, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00196f320}, {0x20768da, 0x46}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, 0xc00002be50, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc00196f320}, {0x20768da, 0x0}, {0x1f33180, 0xc001303ec0}, {0x1f7c860, 0xc00116c0b0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc00196f320}, 0x10, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000688700, {0x2392298, 0xc00196f320})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1152 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc00139bd40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 904 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2b00)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 900 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1651760 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0023ed6e0, {0x2392228, 0xc0014034c0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658082 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001af8d80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac840, {0x1f7c860, 0xc001bac790}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d000, 0xc000c0f5e8, 0xc000c0f5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d000, {0x1f7c860, 0xc001bac790})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002429e00}, {0x20768da, 0x8}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x1c75901, {0xc000be5d20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x7f00048b45b8, {0xc000be5d20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x20, 0xc0022a8d80, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0xc000f5fa00, {0xc0012e1500, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, 0xc0022a8d50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x1e0cfc0, 0xc0022a8cc0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002429e00}, {0x20768da, 0x203000}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x8, 0xc0022a8c90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002429e00}, {0x20768da, 0x46}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc002429e00}, {0x20768da, 0x0}, {0x1f33180, 0xc0022a8c00}, {0x1f7c860, 0xc001bac790}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc002429e00}, 0x60, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000688700, {0x2392298, 0xc002429e00})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1829721 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006c4150, {0x2392228, 0xc001970880})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1657963 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019fc5a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e82420, {0x1f7b560, 0xc001b14e60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391a00, 0xc001c49d18, 0xc001c49ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391a00, {0x1f7b560, 0xc001b14e60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x20, {0xc00230b240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0xc001f27e40, {0xc00230b240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0xc00119e640, 0xc0023c6480, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023c6450}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012dbcc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x237ac70, 0xc0023c63c0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012dbcc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012dbcc0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0012dbc40}, {0x1f7b560, 0xc001b14e60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012dbcc0}, 0xc00204c030, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc00125fcb0}, 0xc0012dbc40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc00125fcb0}, 0xc0012dbc40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001c4a5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c4a648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd819d38, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc00125fcb0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0022fbe20})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c4a898, {0x235a640, 0xc000045ad0}, 0x622d313637342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00125fcb0}, 0xc00125fa10)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0017d2db8, 0x4c1cb4}, {0xc057f6d935dbf36c, 0x8a6753c6be}, {0xc0017d2db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc00125fcb0}, 0xc00125fa10)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0xc00230b0a0, 0xc00205c978)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0x15, 0xc0012dba00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0x207690b, 0xc0012dba00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10}, 0x237bc38, 0xc0012dba00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fcb0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00125fa70}, {0x1f33180, 0xc00125fa10}, 0x1e5a940, 0xc0012dba00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125fa70}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0x9, 0xc0012dba00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0xc001bf5768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00125f9e0}, {0x1f33180, 0xc00125fa10}, 0xc00230b0a0, 0xc00205c978)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00125f9e0}, 0xc001fdd080, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c52c60, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c52c60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 311 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd8656d8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000688f80, 0xc0006e9a30, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000688f80, {0xc0006e9a30, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000688f80, {0xc0006e9a30, 0xc000a27f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa158, {0xc0006e9a30, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa158}, {0xc0006e9a30, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007e6b00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 1829691 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024ee6c0, {0x2392298, 0xc00241bb60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000bcafc0, {0x23922d0, 0xc000c65410}, 0x14)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002468540, {0x23922d0, 0xc000c65410}, 0xc0019ea670, 0x1, 0x19ea680)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc000c653b0, 0xc000f1e880)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc000c65110}, 0xc000f1e880)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000c65110}, {0x1f35540, 0xc000f1e880})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc000c65110}, {0x1f35540, 0xc000f1e880}, 0x237bc38, 0xc0000a29d8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65110}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880}, 0xc000f1e200, 0xc000f1e900)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64fc0}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880}, 0x40, 0xc000f1e900)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c64f90}, {0x1f35540, 0xc000f1e880}, 0xc000f81360, 0xc0000a29d8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc000c64f90}, 0xc00241ba40, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000cb6fc0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000cb6fc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1477993 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001112840, {0x2392228, 0xc002465600})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 781 [select, 10 minutes]:
net/http.(*persistConn).writeLoop(0xc001348a20)
/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1748 +0x1e65
goroutine 1677980 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c3a8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000b7d900, 0xc000d1c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000b7d900, {0xc000d1c000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000b7d900, {0xc000d1c000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278ebb0, {0xc000d1c000, 0x0, 0x7effdd8ca7f8})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0cb40, {0xc0006ba820, 0x9, 0xc000716400})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0cb40}, {0xc0006ba820, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba820, 0x9, 0xc001f91e30}, {0x23585c0, 0xc001c0cb40})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba7e0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000037e00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1658163 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353d50, {0x2392228, 0xc0011711c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1483788 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c8620, {0x2392228, 0xc000efb300})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1657980 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0022b78b0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 288 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd865080, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000709380, 0xc0007040a0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000709380, {0xc0007040a0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000709380, {0xc0007040a0, 0xc000a2bf50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f488, {0xc0007040a0, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f488}, {0xc0007040a0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00070eb00, 0x1ff4eb7)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 286 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd864eb0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000709400, 0xc0009e1fd0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000709400, {0xc0009e1fd0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000709400, {0xc0009e1fd0, 0xc0025cae70, 0xc0021e1b80})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f490, {0xc0009e1fd0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f490}, {0xc0009e1fd0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007b4b00, 0xc8fe40)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 287 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007b4b00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 309 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd864dc8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000689400, 0xc0006e9970, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000689400, {0xc0006e9970, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000689400, {0xc0006e9970, 0xc0025cae70, 0xc001257a30})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa168, {0xc0006e9970, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa168}, {0xc0006e9970, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007e6840, 0x6ff270)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 310 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007e6840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 321 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00070eb00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 312 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007e6b00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 349 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c580)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 352 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c790)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1658162 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc00262b950)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1661128 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000be9d40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012fb8c0, {0x1f7b560, 0xc0006cc140}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6600, 0xc0017bbd18, 0xc0017bbce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6600, {0x1f7b560, 0xc0006cc140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00207fb30}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0x20, {0xc000ebbc20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00207fb30}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00207fb30}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0xc001b766e0, {0xc000ebbc20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00207fb30}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0xc00119e640, 0xc00207fb60, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00207fb30}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002073e80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0x33, 0xc00207faa0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002073e80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002073e80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc002073e00}, {0x1f7b560, 0xc0006cc140}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc002073e80}, 0xc00250ab40, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc00207f3e0}, 0xc002073e00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc00207f3e0}, 0xc002073e00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0017bc5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017bc648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd82b710, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc00207f3e0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021fe850})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017bc898, {0x235a640, 0xc000045ad0}, 0x612d656136342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00207f3e0}, 0xc00207f230)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001b82db8, 0x4c1cb4}, {0xc057f6d9760eb541, 0x8aa321528b}, {0xc001b82db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc00207f3e0}, 0xc00207f230)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230}, 0xc000ebbb40, 0xc0020770e0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230}, 0x15, 0xc002073bc0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230}, 0x207690b, 0xc002073bc0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230}, 0x237bc38, 0xc002073bc0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f3e0}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00207f290}, {0x1f33180, 0xc00207f230}, 0x1e5a940, 0xc002073bc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f290}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230}, 0x80, 0xc002073bc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230}, 0xc000850768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000850b60, {0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00207f200}, {0x1f33180, 0xc00207f230}, 0xc000ebbb40, 0xc0020770e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00207f200}, 0xc0023b2600, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000be8240, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000be8240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661126 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000be8120)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012fb810, {0x1f7c860, 0xc0012fb3f0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6500, 0xc0019535e8, 0xc0019535b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6500, {0x1f7c860, 0xc0012fb3f0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0023b23c0}, {0x20768da, 0x8}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x1c75901, {0xc000ebba40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x7f00048b45b8, {0xc000ebba40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x20, 0xc00207ee70, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0xc000f5fa00, {0xc0020739c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x0, 0xc00207ee40, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x1e0cfc0, 0xc00207edb0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0023b23c0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x8, 0xc00207ed80, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0023b23c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, 0x7, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc0023b23c0}, {0x20768da, 0x0}, {0x1f33180, 0xc00207ed20}, {0x1f7c860, 0xc0012fb3f0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0023b23c0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc0023b23c0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1661153 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001e19560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2630, {0x1f7b560, 0xc001f4c960}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41000, 0xc0017bfd18, 0xc0017bfce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41000, {0x1f7b560, 0xc001f4c960})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00213c060}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0x20, {0xc001c650a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00213c060}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00213c060}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0xc000cbbb80, {0xc001c650a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00213c060}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0xc00119e640, 0xc00213c090, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00213c060}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002120940}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0x237ac70, 0xc002113fb0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002120940}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002120940}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0021208c0}, {0x1f7b560, 0xc001f4c960}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc002120940}, 0xc000124c00, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0021138f0}, 0xc0021208c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0021138f0}, 0xc0021208c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017c05a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017c0648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8d3ee8, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0021138f0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002206b70})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017c0898, {0x235a640, 0xc000045ad0}, 0x622d366233342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0021138f0}, 0xc002113740)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001baedb8, 0x4c1cb4}, {0xc057f6d9768528af, 0x8aa397c604}, {0xc001baedb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0021138f0}, 0xc002113740)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740}, 0xc001c64fe0, 0xc0021034a0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740}, 0x15, 0xc002120680)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740}, 0x207690b, 0xc002120680)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740}, 0x237bc38, 0xc002120680)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021138f0}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021137a0}, {0x1f33180, 0xc002113740}, 0x1e5a940, 0xc002120680)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021137a0}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740}, 0x9, 0xc002120680)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740}, 0xc001d5e768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001d5eb60, {0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002113710}, {0x1f33180, 0xc002113740}, 0xc001c64fe0, 0xc0021034a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc002113710}, 0xc0022df680, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001e185a0, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001e185a0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658027 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023ab9e0, {0x2392298, 0xc001e8c3c0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001ae2900, {0x23922d0, 0xc0024db0e0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0024db0e0}, 0xc00141a910, 0x1bd58c0, 0x141a920)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0024db050, 0xc0024cbe40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0024dac00}, 0xc0024cbe40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0024dac00}, {0x1f353a0, 0xc0024cbe40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0024dac00}, {0x1f353a0, 0xc0024cbe40}, 0x237bc38, 0xc0019bbd70)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024dac00}, {0x1f353a0, 0xc0024cbe40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0024da960, {0x23922d0, 0xc0024da990}, {0x1f353a0, 0xc0024cbe40}, 0xc00245baa0, 0xc0024cbe80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024da990}, {0x1f353a0, 0xc0024cbe40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024da990}, {0x1f353a0, 0xc0024cbe40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024da990}, {0x1f353a0, 0xc0024cbe40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0024da900}, {0x1f353a0, 0xc0024cbe40}, 0x40, 0xc0024cbe80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024da900}, {0x1f353a0, 0xc0024cbe40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0024da900}, {0x1f353a0, 0xc0024cbe40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024da900}, {0x1f353a0, 0xc0024cbe40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0024da900}, {0x1f353a0, 0xc0024cbe40}, 0xc001bc4760, 0xc0019bbd70)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0024da900}, 0xc001e8c360, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001ae38c0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001ae38c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1829743 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006c41c0, {0x2392228, 0xc0023a93c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1008 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000088c80, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000bd8e40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 1829766 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001bfd200)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116d600, {0x1f7b560, 0xc000cd7540}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41f00, 0xc0017dc440, 0xc0017dc410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41f00, {0x1f7b560, 0xc000cd7540})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000683470}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0x20, {0xc0022b4de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000683470}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000683470}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0xc001b76f20, {0xc0022b4de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000683470}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0xc000c61270, 0xc0006834a0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000683470}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0019feac0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0x33, 0xc0006833e0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0019feac0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0019feac0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0019fea40}, {0x1f7b560, 0xc000cd7540}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0019feac0}, 0xc002748846, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc001f8a180}, 0xc0019fea40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc001f8a180}, 0xc0019fea40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0021e5ec0, {0x2392298, 0xc001f8a180})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001f8a120, {0x2392298, 0xc001f8a180}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001bfca20, {0x23922d0, 0xc000682630}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000682630}, 0xc000db2870, 0x1bd58c0, 0xdb2880)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0006825a0, 0xc0023a9f80)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0006821e0}, 0xc0023a9f80)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0006821e0}, {0x1f353a0, 0xc0023a9f80})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0006821e0}, {0x1f353a0, 0xc0023a9f80}, 0x237bc38, 0xc001e95800)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006821e0}, {0x1f353a0, 0xc0023a9f80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001d57f20, {0x23922d0, 0xc001d57f50}, {0x1f353a0, 0xc0023a9f80}, 0xc001d57050, 0xc0023a9fc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d57f50}, {0x1f353a0, 0xc0023a9f80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d57f50}, {0x1f353a0, 0xc0023a9f80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d57f50}, {0x1f353a0, 0xc0023a9f80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d57ec0}, {0x1f353a0, 0xc0023a9f80}, 0x40, 0xc0023a9fc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d57ec0}, {0x1f353a0, 0xc0023a9f80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d57ec0}, {0x1f353a0, 0xc0023a9f80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d57ec0}, {0x1f353a0, 0xc0023a9f80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d57ec0}, {0x1f353a0, 0xc0023a9f80}, 0xc0022b4cc0, 0xc001e95800)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001d57ec0}, 0xc0021e5c20, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000bbf8c0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000bbf8c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 496 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 495 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1651758 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003cf1f0, {0x2392228, 0xc001403480})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1829719 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000f71e60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116cd10, {0x1f7b560, 0xc001de34a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41a00, 0xc000eca440, 0xc000eca410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41a00, {0x1f7b560, 0xc001de34a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001dce7b0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0x20, {0xc0022b4580, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001dce7b0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001dce7b0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0xc001c56840, {0xc0022b4580, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001dce7b0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0xc000c61270, 0xc001dce7e0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001dce7b0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001970a00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0x237ac70, 0xc001dce720, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001970a00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001970a00}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001970980}, {0x1f7b560, 0xc001de34a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc001970a00}, 0xc0006fa4a6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc0024ef9e0}, 0xc001970980, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc0024ef9e0}, 0xc001970980, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0024ef740, {0x2392298, 0xc0024ef9e0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024ef980, {0x2392298, 0xc0024ef9e0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000f71560, {0x23922d0, 0xc000c3f800}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000c3f800}, 0xc001f0f430, 0x1bd58c0, 0x1f0f440)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc000c3f7a0, 0xc001970000)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc000c3f500}, 0xc001970000)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000c3f500}, {0x1f353a0, 0xc001970000})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc000c3f500}, {0x1f353a0, 0xc001970000}, 0x237bc38, 0xc000fe7728)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3f500}, {0x1f353a0, 0xc001970000})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc000c3f350, {0x23922d0, 0xc000c3f380}, {0x1f353a0, 0xc001970000}, 0xc0006a3ef0, 0xc001970040)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3f380}, {0x1f353a0, 0xc001970000})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c3f380}, {0x1f353a0, 0xc001970000}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3f380}, {0x1f353a0, 0xc001970000})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c3f2c0}, {0x1f353a0, 0xc001970000}, 0x40, 0xc001970040)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3f2c0}, {0x1f353a0, 0xc001970000})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c3f2c0}, {0x1f353a0, 0xc001970000}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3f2c0}, {0x1f353a0, 0xc001970000})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c3f2c0}, {0x1f353a0, 0xc001970000}, 0xc0022b4440, 0xc000fe7728)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc000c3f2c0}, 0xc0024ef4a0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000f70b40, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000f70b40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658101 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0026435c0, {0x2392228, 0xc000e64bc0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1829769 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0021e5e00, {0x2392228, 0xc0019fe980})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1678721 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001112540, {0x2392298, 0xc001f51e60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005bb8c0, {0x23922d0, 0xc00213cf60}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002476cf0, {0x23922d0, 0xc00213cf60}, 0xc00124c700, 0x1, 0x124c710)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc00213cf00, 0xc001241940)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc00213cc90}, 0xc001241940)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00213cc90}, {0x1f35540, 0xc001241940})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00213cc90}, {0x1f35540, 0xc001241940}, 0x237bc38, 0xc0023fdf20)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00213cc90}, {0x1f35540, 0xc001241940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00213cb40}, {0x1f35540, 0xc001241940}, 0xc001241240, 0xc0012419c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00213cb40}, {0x1f35540, 0xc001241940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00213cb40}, {0x1f35540, 0xc001241940}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00213cb40}, {0x1f35540, 0xc001241940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00213cb10}, {0x1f35540, 0xc001241940}, 0x40, 0xc0012419c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00213cb10}, {0x1f35540, 0xc001241940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00213cb10}, {0x1f35540, 0xc001241940}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00213cb10}, {0x1f35540, 0xc001241940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00213cb10}, {0x1f35540, 0xc001241940}, 0xc000ebb260, 0xc0023fdf20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00213cb10}, 0xc001f51e00, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c1ea20, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c1ea20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1311611 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000554ee0, {0x2392228, 0xc00212cfc0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1661132 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0023b2a20, {0x2392228, 0xc002190880})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1829767 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001237f40)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1678003 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920ef0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000b7da80, 0xc000dca000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000b7da80, {0xc000dca000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000b7da80, {0xc000dca000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000c14f20, {0xc000dca000, 0x0, 0x7effdd8d8350})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d58720, {0xc0003282e0, 0x9, 0xc00085a000})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001d58720}, {0xc0003282e0, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0003282e0, 0x9, 0xc00131c3f0}, {0x23585c0, 0xc001d58720})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003282a0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000cb4960)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1678681 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019fcd80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80840, {0x1f7b7c0, 0xc001b14d20}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7900, 0xc001035bf8, 0xc001035bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7900, {0x1f7b7c0, 0xc001b14d20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00206df20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0x20, {0xc000eba8e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00206df20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00206df20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0xc00138f760, {0xc000eba8e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00206df20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0xc00119e640, 0xc00206df50, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00206df20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012011c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0x237ac70, 0xc00206dd70, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012011c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012011c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001201140}, {0x1f7b7c0, 0xc001b14d20}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012011c0}, 0xc0021a79c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc00206d0b0}, 0xc001201140, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc00206d0b0}, 0xc001201140, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001036480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001036528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8db298, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00206d0b0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0025a1a30})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001036778, {0x235a640, 0xc000045ad0}, 0x382d353732342d34)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00206d0b0}, 0xc001200e40)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001820db8, 0x4c1cb4}, {0xc057f6daf512f4d1, 0x8c07c64e1b}, {0xc001820db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc00206d0b0}, 0xc001200e40)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40}, 0xc000eba7e0, 0xc0023c14e8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40}, 0x15, 0xc001200e80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40}, 0x2076951, 0xc001200e80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40}, 0x237bc38, 0xc001200e80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d0b0}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00206cf60}, {0x1f33320, 0xc001200e40}, 0x1e5a940, 0xc001200e80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206cf60}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40}, 0x1bd58c0, 0xc001200e80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002848b60, {0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00206cf30}, {0x1f33320, 0xc001200e40}, 0xc000eba7e0, 0xc0023c14e8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00206cf30}, 0xc000f97560, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0001290e0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0001290e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1829741 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021e5200, {0x2392298, 0xc0021e5260}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00086ca20, {0x23922d0, 0xc001b9bb60}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001b9bb60}, 0xc00201d850, 0x1bd58c0, 0x201d860)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001b9b950, 0xc0023a85c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001b9b6b0}, 0xc0023a85c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001b9b6b0}, {0x1f353a0, 0xc0023a85c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001b9b6b0}, {0x1f353a0, 0xc0023a85c0}, 0x237bc38, 0xc001e94b40)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b9b6b0}, {0x1f353a0, 0xc0023a85c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001b9b560}, {0x1f353a0, 0xc0023a85c0}, 0x203000, 0xc0023a8680)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b9b560}, {0x1f353a0, 0xc0023a85c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b9b560}, {0x1f353a0, 0xc0023a85c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b9b560}, {0x1f353a0, 0xc0023a85c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001b9b500}, {0x1f353a0, 0xc0023a85c0}, 0x40, 0xc0023a8680)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b9b500}, {0x1f353a0, 0xc0023a85c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001b9b500}, {0x1f353a0, 0xc0023a85c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b9b500}, {0x1f353a0, 0xc0023a85c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001b9b500}, {0x1f353a0, 0xc0023a85c0}, 0xc0022b48a0, 0xc001e94b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001b9b500}, 0xc0021e4ea0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0005cab40, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0005cab40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1651757 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001d68f00)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1829716 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000f70a20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116cc60, {0x1f7b560, 0xc001de3180}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8da00, 0xc001bcbd18, 0xc001bcbce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8da00, {0x1f7b560, 0xc001de3180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000c3ea20}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0x20, {0xc0022b4360, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000c3ea20}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c3ea20}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0xc001c56420, {0xc0022b4360, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c3ea20}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0xc00119e640, 0xc000c3ea50, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c3ea20}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000efbc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0x237ac70, 0xc000c3e990, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000efbc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000efbc40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000efbbc0}, {0x1f7b560, 0xc001de3180}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc000efbc40}, 0xc000124de0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc000c3e0f0}, 0xc000efbbc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc000c3e0f0}, 0xc000efbbc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001bcc5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bcc648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd83fe80, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc00229b3b0, {0x23922d0, 0xc000c3e0f0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001fa9800})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001bcc898, {0x235a640, 0xc000045ad0}, 0x622d623563342d35)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc000c3e0f0}, 0xc0006a3ef0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001730db8, 0x4c1cb4}, {0xc057f6e7368e8705, 0x9771e28a5c}, {0xc001730db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc000c3e0f0}, 0xc0006a3ef0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0}, 0xc0022b42a0, 0xc000fe7158)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0}, 0x15, 0xc000efb980)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0}, 0x207690b, 0xc000efb980)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0}, 0x237bc38, 0xc000efb980)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c3e0f0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0006a3f80}, {0x1f33180, 0xc0006a3ef0}, 0x1e5a940, 0xc000efb980)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a3f80}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0}, 0x9, 0xc000efb980)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0}, 0xc0011cf768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0011cfb60, {0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0006a3ec0}, {0x1f33180, 0xc0006a3ef0}, 0xc0022b42a0, 0xc000fe7158)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0006a3ec0}, 0xc0024ef0e0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0019c3680, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0019c3680, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678695 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005cb200)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012fbe40, {0x1f7b7c0, 0xc001f4d040}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078b600, 0xc000addbf8, 0xc000addbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078b600, {0x1f7b7c0, 0xc001f4d040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00207ee40}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0x20, {0xc001c64100, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00207ee40}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00207ee40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0xc001b778c0, {0xc001c64100, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00207ee40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0xc00119e640, 0xc00207ee70, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00207ee40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0011ab680}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0x237ac70, 0xc00207edb0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0011ab680}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0011ab680}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0011ab600}, {0x1f7b7c0, 0xc001f4d040}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0011ab680}, 0xc002200bc0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc00207e6c0}, 0xc0011ab600, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc00207e6c0}, 0xc0011ab600, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000ade480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000ade528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8861d0, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00207e6c0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002725600})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000ade778, {0x235a640, 0xc000045ad0}, 0x382d393263342d64)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00207e6c0}, 0xc0011ab300)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001982db8, 0x4c1cb4}, {0xc057f6daf522146f, 0x8c07d56db8}, {0xc001982db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc00207e6c0}, 0xc0011ab300)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300}, 0xc001c64020, 0xc0021ba0d8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300}, 0x15, 0xc0011ab340)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300}, 0x2076951, 0xc0011ab340)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300}, 0x237bc38, 0xc0011ab340)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e6c0}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00207e570}, {0x1f33320, 0xc0011ab300}, 0x1e5a940, 0xc0011ab340)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e570}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300}, 0x23c27b0, 0xc0011ab340)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00207e540}, {0x1f33320, 0xc0011ab300}, 0xc001c64020, 0xc0021ba0d8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00207e540}, 0xc000f9fec0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bcaa20, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bcaa20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 547 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2790)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 546 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d26e0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 544 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1657983 [runnable]:
internal/poll.runtime_pollWait(0x7effdd95c008, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0013a4800, 0xc001b24000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0013a4800, {0xc001b24000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0013a4800, {0xc001b24000, 0xc00172cfd8, 0x6625c9})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278f8e0, {0xc001b24000, 0xc00172d020, 0x6738a5})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc00008f680, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc00008f680, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc00008f680)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlStmt).readPrepareResultPacket(0xc0023e5c38)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:829 +0x28
github.com/go-sql-driver/mysql.(*mysqlConn).Prepare(0xc00008f680, {0x208f76a, 0xc0012f9aa0})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:167 +0x18f
github.com/go-sql-driver/mysql.(*mysqlConn).PrepareContext(0xc0012f9aa0, {0x2392298, 0xc0012f9aa0}, {0x208f76a, 0x94})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:540 +0x53
database/sql.ctxDriverPrepare({0x2392298, 0xc0012f9aa0}, {0x237efb0, 0xc00008f680}, {0x208f76a, 0x94})
/usr/local/go/src/database/sql/ctxutil.go:15 +0x11b
database/sql.(*DB).queryDC.func2()
/usr/local/go/src/database/sql/sql.go:1744 +0x4b
database/sql.withLock({0x237b030, 0xc001397a70}, 0xc00172d3d0)
/usr/local/go/src/database/sql/sql.go:3396 +0x8c
database/sql.(*DB).queryDC(0x203001, {0x2392298, 0xc0012f9aa0}, {0x0, 0x0}, 0xc001397a70, 0xc00127de70, {0x208f76a, 0x94}, {0xc000dff4a0, ...})
/usr/local/go/src/database/sql/sql.go:1743 +0x30f
database/sql.(*DB).query(0x172d548, {0x2392298, 0xc0012f9aa0}, {0x208f76a, 0x94}, {0xc000dff4a0, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0012f9aa0}, {0x208f76a, 0x94}, {0xc000dff4a0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0012f9aa0}, {0x208f76a, 0xc00004c000}, {0xc000dff4a0, 0xc00172d6c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0012f9aa0}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0023e57a0}, {0x208f76a, 0x0}, {0xc000dff4a0, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x6, {0x2392298, 0xc0012f9aa0}, {0x1b5d8e0, 0xc0023e57a0}, {0x208f76a, 0xc00172d7e0}, {0xc000dff4a0, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0012f9aa0}, {0x1135535c, {0xc000ee9350, 0x25, 0x25}, 0x0, 0xc000e8c7a8, 0xc00245f128, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e8c780)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x14)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc002570de0, 0x3ef8e920, 0x3efa6fc0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0x7effdd9581c8, 0x5})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00172dc50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc00172dd10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc00172dde2, 0xc000fdc9c0)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc00172dd60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc002571020)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002571020, {0x2392228, 0xc001319d00})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1661138 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003cf5e0, {0x2392228, 0xc0021915c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1829744 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0021e5080, {0x2392228, 0xc0023a9400})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658102 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001113bc0, {0x2392298, 0xc002643e60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00195a480, {0x23922d0, 0xc0020accf0}, 0x2a)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc000ba4420, {0x23922d0, 0xc0020accf0}, 0xc001f1c160, 0x1, 0x1f1c170)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0020acae0, 0xc000e65040)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0020ac840}, 0xc000e65040)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020ac840}, {0x1f35540, 0xc000e65040})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0020ac840}, {0x1f35540, 0xc000e65040}, 0x237bc38, 0xc0024e57e8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ac840}, {0x1f35540, 0xc000e65040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0020ac5d0}, {0x1f35540, 0xc000e65040}, 0x203000, 0xc000e650c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ac5d0}, {0x1f35540, 0xc000e65040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020ac5d0}, {0x1f35540, 0xc000e65040}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ac5d0}, {0x1f35540, 0xc000e65040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020ac5a0}, {0x1f35540, 0xc000e65040}, 0x40, 0xc000e650c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ac5a0}, {0x1f35540, 0xc000e65040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020ac5a0}, {0x1f35540, 0xc000e65040}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ac5a0}, {0x1f35540, 0xc000e65040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020ac5a0}, {0x1f35540, 0xc000e65040}, 0xc000f800e0, 0xc0024e57e8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0020ac5a0}, 0xc002643e00, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001dcde60, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001dcde60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1480911 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00243b680, {0x2392228, 0xc001420b80})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 884 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 677 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c8f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 676 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00062c840)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 675 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000540580)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 674 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005404d0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 673 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 883 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016afc0, {0x2392228, 0xc0012f7980})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1661133 [select]:
database/sql.(*DB).conn(0xc0006de8f0, {0x2392298, 0xc0012f9e60}, 0x1)
/usr/local/go/src/database/sql/sql.go:1312 +0x413
database/sql.(*DB).query(0xc001b61548, {0x2392298, 0xc0012f9e60}, {0x208f76a, 0x94}, {0xc000e96050, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1695 +0x5d
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0012f9e60}, {0x208f76a, 0x94}, {0xc000e96050, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0012f9e60}, {0x208f76a, 0xc00004c000}, {0xc000e96050, 0xc001b616c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0012f9e60}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0023e5a88}, {0x208f76a, 0xc000ee9400}, {0xc000e96050, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2b, {0x2392298, 0xc0012f9e60}, {0x1b5d8e0, 0xc0023e5a88}, {0x208f76a, 0xc001b617e0}, {0xc000e96050, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0012f9e60}, {0x9eb1eeec, {0xc0013c88c0, 0x3c, 0x3c}, 0x0, 0xc000e8cd28, 0xc00245f448, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e8cd00)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x2b)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0023b28a0, 0x3f717480, 0x3f72fb20, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc000716400, 0xc002185cc0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001b61c50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc001b61d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001b61de2, 0xc00111a4e0)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001b61d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc0023b2a20)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0023b2a20, {0x2392228, 0xc002190880})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 995 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a540f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 996 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc0021bcfb0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 997 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc0021bdba0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 998 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc001163d80)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 999 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001194d00, 0xc0011ae860)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001194d00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a54000)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1471877 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000d68af0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1215 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010f48f0, 0xc00124c900)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010f48f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 780 [select, 10 minutes]:
net/http.(*persistConn).readLoop(0xc001348a20)
/usr/local/go/src/net/http/transport.go:2207 +0xd8a
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1747 +0x1e05
goroutine 1213 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc0007c25a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a553b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a553b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 830 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 1480932 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0020ee240, {0x2392228, 0xc001421880})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 922 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920158, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00149c200, 0xc000ff3dd0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00149c200, {0xc000ff3dd0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00149c200, {0xc000ff3dd0, 0xc001320f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aaf70, {0xc000ff3dd0, 0x46b213, 0x617f641d})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aaf70}, {0xc000ff3dd0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007e78c0, 0x13578c0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 1160 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 919 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2c60)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1156 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc0013b3c20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1157 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc00139a3b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1658103 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077e480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e838c0, {0x1f7c860, 0xc001e83810}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40500, 0xc0017ad5e8, 0xc0017ad5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40500, {0x1f7c860, 0xc001e83810})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00196e240}, {0x20768da, 0x8}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x1c75901, {0xc000f80420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x7f00048b4108, {0xc000f80420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x20, 0xc0020ad680, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0xc000f5fa00, {0xc000e656c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x0, 0xc0020ad650, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x1e0cfc0, 0xc0020ad590, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00196e240}, {0x20768da, 0x203000}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x8, 0xc0020ad560, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00196e240}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, 0x1cab380, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc00196e240}, {0x20768da, 0x0}, {0x1f33180, 0xc0020ad500}, {0x1f7c860, 0xc001e83810}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc00196e240}, 0x10, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc00196e240})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1176 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd865420, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4cd00, 0xc00174a000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a4cd00, {0xc00174a000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000a4cd00, {0xc00174a000, 0x60100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e9b0, {0xc00174a000, 0x4383e0, 0xc00069a820})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000ba22a0, {0xc000328c80, 0x9, 0xc001744d18})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000ba22a0}, {0xc000328c80, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000328c80, 0x9, 0x18}, {0x23585c0, 0xc000ba22a0})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328c40)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0007310e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1678694 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bca900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012fbd90, {0x1f736c0, 0xc0011b2090}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078b300, 0xc001df35c0, 0xc001df3590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078b300, {0x1f736c0, 0xc0011b2090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000f9fc80}, {0x2076920, 0x8}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x1c75901, {0xc000a71f00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x7f00048b4108, {0xc000a71f00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x20, 0xc00207e150, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0xc000bfb110, {0xc0011ab080, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x0, 0xc00207e120, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x0, 0xc00207e090, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000f9fc80}, {0x2076920, 0x203000}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x203000, 0xc00207e060, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000f9fc80}, {0x2076920, 0x46}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002849e50, {0x2392298, 0xc000f9fc80}, {0x2076920, 0x200e02e40}, {0x1f33320, 0xc0011ab040}, {0x1f736c0, 0xc0011b2090}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc000f9fc80}, 0xc000354200, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc000f9fc80})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658090 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001fa0c00, {0x2392298, 0xc002548d80}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000fe86c0, {0x23922d0, 0xc0022b28d0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0022b28d0}, 0xc001dd6590, 0x1bd58c0, 0x1dd65a0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0022b2600, 0xc000e90800)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022b21e0}, 0xc000e90800)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022b21e0}, {0x1f353a0, 0xc000e90800})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022b21e0}, {0x1f353a0, 0xc000e90800}, 0x237bc38, 0xc00209a348)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b21e0}, {0x1f353a0, 0xc000e90800})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002211fb0}, {0x1f353a0, 0xc000e90800}, 0x203000, 0xc000e90840)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211fb0}, {0x1f353a0, 0xc000e90800})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002211fb0}, {0x1f353a0, 0xc000e90800}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211fb0}, {0x1f353a0, 0xc000e90800})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002211f50}, {0x1f353a0, 0xc000e90800}, 0x40, 0xc000e90840)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211f50}, {0x1f353a0, 0xc000e90800})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002211f50}, {0x1f353a0, 0xc000e90800}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211f50}, {0x1f353a0, 0xc000e90800})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002211f50}, {0x1f353a0, 0xc000e90800}, 0xc000084380, 0xc00209a348)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002211f50}, 0xc002548d20, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc000670900, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc000670900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1477840 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000140a10, {0x2392228, 0xc001c33f00})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1678790 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001193680)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0420, {0x1f7b7c0, 0xc001f4dc20}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078bf00, 0xc0017c3bf8, 0xc0017c3bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078bf00, {0x1f7b7c0, 0xc001f4dc20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00246d740}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0x20, {0xc001c654c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00246d740}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00246d740}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0xc001471e40, {0xc001c654c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00246d740}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0xc00119e640, 0xc00246d770, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00246d740}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012d41c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0x237ac70, 0xc00246d6b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012d41c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012d41c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012d4140}, {0x1f7b7c0, 0xc001f4dc20}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012d41c0}, 0xc002201940, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc00246d050}, 0xc0012d4140, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc00246d050}, 0xc0012d4140, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017c4480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017c4528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd960ae0, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00246d050}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00119f070})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017c4778, {0x235a640, 0xc000045ad0}, 0x392d313532342d62)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00246d050}, 0xc0012cde40)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001b28db8, 0x4c1cb4}, {0xc057f6daf6c25293, 0x8c0975abeb}, {0xc001b28db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc00246d050}, 0xc0012cde40)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40}, 0xc001c65320, 0xc0021ee168)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40}, 0x15, 0xc0012cde80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40}, 0x2076951, 0xc0012cde80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40}, 0x237bc38, 0xc0012cde80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246d050}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00246cf00}, {0x1f33320, 0xc0012cde40}, 0x1e5a940, 0xc0012cde80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246cf00}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40}, 0x23c27b0, 0xc0012cde80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00246ced0}, {0x1f33320, 0xc0012cde40}, 0xc001c65320, 0xc0021ee168)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00246ced0}, 0xc0020a7c20, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000fdb560, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000fdb560, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678791 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001c0c0c0, {0x2392298, 0xc0020a7f20}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001b7d320, {0x23922d0, 0xc00246df50}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002689d40, {0x23922d0, 0xc00246df50}, 0xc0012d3510, 0x1, 0x12d3520)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc00246def0, 0xc0012d43c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc00246dc20}, 0xc0012d43c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00246dc20}, {0x1f35540, 0xc0012d43c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00246dc20}, {0x1f35540, 0xc0012d43c0}, 0x237bc38, 0xc0021ee528)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246dc20}, {0x1f35540, 0xc0012d43c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00246dad0}, {0x1f35540, 0xc0012d43c0}, 0xc0012cde40, 0xc0012d4440)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246dad0}, {0x1f35540, 0xc0012d43c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00246dad0}, {0x1f35540, 0xc0012d43c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246dad0}, {0x1f35540, 0xc0012d43c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00246daa0}, {0x1f35540, 0xc0012d43c0}, 0x40, 0xc0012d4440)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246daa0}, {0x1f35540, 0xc0012d43c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00246daa0}, {0x1f35540, 0xc0012d43c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00246daa0}, {0x1f35540, 0xc0012d43c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00246daa0}, {0x1f35540, 0xc0012d43c0}, 0xc001c656c0, 0xc0021ee528)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00246daa0}, 0xc0020a7ec0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0011937a0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0011937a0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1681108 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c687e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00062cfd0, {0x1f7b7c0, 0xc001de2d20}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001447000, 0xc001b3dbf8, 0xc001b3dbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001447000, {0x1f7b7c0, 0xc001de2d20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0006a5110}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0x20, {0xc000f80480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0006a5110}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0006a5110}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0xc0013b6f20, {0xc000f80480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0006a5110}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0xc00119e640, 0xc0006a5140, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0006a5110}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fa5f00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0x237ac70, 0xc0006a5080, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fa5f00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fa5f00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001fa5e80}, {0x1f7b7c0, 0xc001de2d20}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc001fa5f00}, 0xc0022670c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0006a4930}, 0xc001fa5e80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0006a4930}, 0xc001fa5e80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001b3e480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001b3e528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd867990, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0006a4930}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0009e0200})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001b3e778, {0x235a640, 0xc000045ad0}, 0x612d656266342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0006a4930}, 0xc001fa5b80)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000490db8, 0x4c1cb4}, {0xc057f6db29d474aa, 0x8c382297f5}, {0xc000490db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0006a4930}, 0xc001fa5b80)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80}, 0xc000f80220, 0xc001783e48)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80}, 0x15, 0xc001fa5bc0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80}, 0x2076951, 0xc001fa5bc0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80}, 0x237bc38, 0xc001fa5bc0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4930}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0006a4780}, {0x1f33320, 0xc001fa5b80}, 0x1e5a940, 0xc001fa5bc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4780}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80}, 0x23c27b0, 0xc001fa5bc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000729b60, {0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0006a4750}, {0x1f33320, 0xc001fa5b80}, 0xc000f80220, 0xc001783e48)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0006a4750}, 0xc001dc1da0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00065cfc0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00065cfc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658121 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001fa0c00, {0x2392298, 0xc00196fbc0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000fe86c0, {0x23922d0, 0xc00200a3c0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00200a3c0}, 0xc001eb4520, 0x1bd58c0, 0x1eb4530)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc00200a1b0, 0xc000ed34c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc002225ef0}, 0xc000ed34c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002225ef0}, {0x1f353a0, 0xc000ed34c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002225ef0}, {0x1f353a0, 0xc000ed34c0}, 0x237bc38, 0xc001f173b0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002225ef0}, {0x1f353a0, 0xc000ed34c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002225140, {0x23922d0, 0xc002225da0}, {0x1f353a0, 0xc000ed34c0}, 0xc002224450, 0xc000ed3500)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002225da0}, {0x1f353a0, 0xc000ed34c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002225da0}, {0x1f353a0, 0xc000ed34c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002225da0}, {0x1f353a0, 0xc000ed34c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002225d40}, {0x1f353a0, 0xc000ed34c0}, 0x40, 0xc000ed3500)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002225d40}, {0x1f353a0, 0xc000ed34c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002225d40}, {0x1f353a0, 0xc000ed34c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002225d40}, {0x1f353a0, 0xc000ed34c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002225d40}, {0x1f353a0, 0xc000ed34c0}, 0xc000f81220, 0xc001f173b0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002225d40}, 0xc00196fb60, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0022ed0e0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0022ed0e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1483787 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000969770)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1658127 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001bfc7e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116c8f0, {0x1f7b7c0, 0xc0007c14a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40c00, 0xc0017dfbf8, 0xc0017dfbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40c00, {0x1f7b7c0, 0xc0007c14a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00214c840}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0x20, {0xc000f81ee0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00214c840}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00214c840}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0xc001bfadc0, {0xc000f81ee0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00214c840}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0xc00119e640, 0xc00214c870, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00214c840}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0010dc680}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0x237ac70, 0xc00214c7b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0010dc680}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0010dc680}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0010dc600}, {0x1f7b7c0, 0xc0007c14a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0010dc680}, 0xc002476420, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc00214c0c0}, 0xc0010dc600, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc00214c0c0}, 0xc0010dc600, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017e0480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017e0528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8175e8, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00214c0c0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000f6f7b0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017e0778, {0x235a640, 0xc000045ad0}, 0x622d333734342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00214c0c0}, 0xc0010dc300)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0019d8db8, 0x4c1cb4}, {0xc057f6d9383c1400, 0x8a69b3e749}, {0xc0019d8db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc00214c0c0}, 0xc0010dc300)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300}, 0xc000f81ac0, 0xc0023f6030)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300}, 0x15, 0xc0010dc340)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300}, 0x2076951, 0xc0010dc340)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300}, 0x237bc38, 0xc0010dc340)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214c0c0}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00200bf50}, {0x1f33320, 0xc0010dc300}, 0x1e5a940, 0xc0010dc340)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00200bf50}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300}, 0x23c27b0, 0xc0010dc340)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00200bf20}, {0x1f33320, 0xc0010dc300}, 0xc000f81ac0, 0xc0023f6030)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00200bf20}, 0xc00263e720, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0005c4b40, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0005c4b40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661184 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006678c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d31e0, {0x1f7c860, 0xc0007d3130}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41400, 0xc0004955e8, 0xc0004955b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41400, {0x1f7c860, 0xc0007d3130})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00257a720}, {0x20768da, 0x8}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x1c75901, {0xc001c65b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x7f00048b4108, {0xc001c65b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x20, 0xc0022efb60, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0xc000f5fa00, {0xc0022e3d00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x0, 0xc0022efb30, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x1e0cfc0, 0xc0022efaa0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00257a720}, {0x20768da, 0x203000}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x8, 0xc0022efa70, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00257a720}, {0x20768da, 0x46}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc00257a720}, {0x20768da, 0x0}, {0x1f33180, 0xc0022efa10}, {0x1f7c860, 0xc0007d3130}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc00257a720}, 0xc001154720, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000688700, {0x2392298, 0xc00257a720})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1829720 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001d69c20)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1648597 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000f96240, {0x2392228, 0xc00126f500})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658100 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0026435c0, {0x2392228, 0xc000e64bc0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654723 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00250dce0, {0x2392228, 0xc0021231c0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658161 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023988a0, {0x2392298, 0xc002398900}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00063cfc0, {0x23922d0, 0xc00133ce40}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00133ce40}, 0xc0012ee680, 0x1bd58c0, 0x12ee690)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc00133cb10, 0xc001170a00)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc00133c7e0}, 0xc001170a00)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00133c7e0}, {0x1f353a0, 0xc001170a00})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00133c7e0}, {0x1f353a0, 0xc001170a00}, 0x237bc38, 0xc00209bb48)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133c7e0}, {0x1f353a0, 0xc001170a00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002127800, {0x23922d0, 0xc00133c690}, {0x1f353a0, 0xc001170a00}, 0xc002126ae0, 0xc001170a40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133c690}, {0x1f353a0, 0xc001170a00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00133c690}, {0x1f353a0, 0xc001170a00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133c690}, {0x1f353a0, 0xc001170a00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00133c630}, {0x1f353a0, 0xc001170a00}, 0x40, 0xc001170a40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133c630}, {0x1f353a0, 0xc001170a00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00133c630}, {0x1f353a0, 0xc001170a00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133c630}, {0x1f353a0, 0xc001170a00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00133c630}, {0x1f353a0, 0xc001170a00}, 0xc000eba500, 0xc00209bb48)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00133c630}, 0xc002398540, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0003bcfc0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0003bcfc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1477858 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0026bc960, {0x2392228, 0xc001c33f40})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1678697 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001112540, {0x2392298, 0xc00243a720}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005bb8c0, {0x23922d0, 0xc00207fb90}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002688ae0, {0x23922d0, 0xc00207fb90}, 0xc0011af080, 0x1, 0x11af090)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc00207fad0, 0xc0011ab980)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc00207f830}, 0xc0011ab980)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00207f830}, {0x1f35540, 0xc0011ab980})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00207f830}, {0x1f35540, 0xc0011ab980}, 0x237bc38, 0xc0021ba798)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f830}, {0x1f35540, 0xc0011ab980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00207f6b0}, {0x1f35540, 0xc0011ab980}, 0xc0011ab300, 0xc0011aba00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f6b0}, {0x1f35540, 0xc0011ab980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00207f6b0}, {0x1f35540, 0xc0011ab980}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f6b0}, {0x1f35540, 0xc0011ab980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00207f680}, {0x1f35540, 0xc0011ab980}, 0x40, 0xc0011aba00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f680}, {0x1f35540, 0xc0011ab980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00207f680}, {0x1f35540, 0xc0011ab980}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00207f680}, {0x1f35540, 0xc0011ab980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00207f680}, {0x1f35540, 0xc0011ab980}, 0xc001c641e0, 0xc0021ba798)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00207f680}, 0xc00243a6c0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0005cb680, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0005cb680, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678714 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000e88840, {0x2392298, 0xc001f510e0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000f70480, {0x23922d0, 0xc0022b0030}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002476990, {0x23922d0, 0xc0022b0030}, 0xc001209a70, 0x1, 0x1209a80)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0020fff80, 0xc001240a80)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0020ffd10}, 0xc001240a80)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020ffd10}, {0x1f35540, 0xc001240a80})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0020ffd10}, {0x1f35540, 0xc001240a80}, 0x237bc38, 0xc0023fcdf8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ffd10}, {0x1f35540, 0xc001240a80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0020ffbc0}, {0x1f35540, 0xc001240a80}, 0xc001240240, 0xc001240b00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ffbc0}, {0x1f35540, 0xc001240a80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020ffbc0}, {0x1f35540, 0xc001240a80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ffbc0}, {0x1f35540, 0xc001240a80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020ffb90}, {0x1f35540, 0xc001240a80}, 0x40, 0xc001240b00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ffb90}, {0x1f35540, 0xc001240a80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020ffb90}, {0x1f35540, 0xc001240a80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ffb90}, {0x1f35540, 0xc001240a80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020ffb90}, {0x1f35540, 0xc001240a80}, 0xc000ebae40, 0xc0023fcdf8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0020ffb90}, 0xc001f51080, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c34480, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c34480, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658049 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002562500)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 760 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00078c780, {0x2392228, 0xc001141f40})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 761 [select]:
go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).purgeProcessor(0xc0000c80e0)
/temporal/common/persistence/namespaceReplicationQueue.go:404 +0xed
created by go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).Start
/temporal/common/persistence/namespaceReplicationQueue.go:123 +0x71
goroutine 762 [select, 10 minutes]:
go.temporal.io/server/service/frontend.(*VersionChecker).versionCheckLoop(0xc000089220)
/temporal/service/frontend/versionChecker.go:88 +0xe5
created by go.temporal.io/server/service/frontend.(*VersionChecker).Start.func1
/temporal/service/frontend/versionChecker.go:70 +0x5f
goroutine 923 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007e78c0, 0x13578c0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1658089 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0025484e0, {0x2392228, 0xc000e903c0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654875 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0000bed80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022ead10, {0x1f7b560, 0xc001de3040}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078aa00, 0xc001808440, 0xc001808410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078aa00, {0x1f7b560, 0xc001de3040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022543c0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0x20, {0xc000e81d60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022543c0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022543c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0xc0020ae580, {0xc000e81d60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022543c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0xc0023f4400, 0xc0022543f0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022543c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00223b700}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0x237ac70, 0xc002254330, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00223b700}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00223b700}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc00223b680}, {0x1f7b560, 0xc001de3040}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc00223b700}, 0xc002200986, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc0022447e0}, 0xc00223b680, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc0022447e0}, 0xc00223b680, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002244540, {0x2392298, 0xc0022447e0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002244780, {0x2392298, 0xc0022447e0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0000be240, {0x23922d0, 0xc002243710}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002243710}, 0xc002237410, 0x1bd58c0, 0x2237420)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0022436b0, 0xc00223ad80)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc002243410}, 0xc00223ad80)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002243410}, {0x1f353a0, 0xc00223ad80})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002243410}, {0x1f353a0, 0xc00223ad80}, 0x237bc38, 0xc002240870)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002243410}, {0x1f353a0, 0xc00223ad80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002243290, {0x23922d0, 0xc0022432c0}, {0x1f353a0, 0xc00223ad80}, 0xc002242420, 0xc00223adc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022432c0}, {0x1f353a0, 0xc00223ad80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022432c0}, {0x1f353a0, 0xc00223ad80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022432c0}, {0x1f353a0, 0xc00223ad80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002243230}, {0x1f353a0, 0xc00223ad80}, 0x40, 0xc00223adc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002243230}, {0x1f353a0, 0xc00223ad80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002243230}, {0x1f353a0, 0xc00223ad80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002243230}, {0x1f353a0, 0xc00223ad80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002243230}, {0x1f353a0, 0xc00223ad80}, 0xc000e819a0, 0xc002240870)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002243230}, 0xc0022442a0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc00066c480, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc00066c480, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1155 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001198e00, 0xc0021e12a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc001198e00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a541e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 905 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007d2bb0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 901 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 1106 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000089180, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000bd9620)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef
goroutine 1107 [select, 10 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006efe00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837
goroutine 1108 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd865250, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001199800, 0xc000ba8000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001199800, {0xc000ba8000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001199800, {0xc000ba8000, 0x1060100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e590, {0xc000ba8000, 0xc0011d1cf8, 0x44d2f4})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000bd8f60, {0xc000328ba0, 0x9, 0x9772e91b0c})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000bd8f60}, {0xc000328ba0, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000328ba0, 0x9, 0xc0000891d0}, {0x23585c0, 0xc000bd8f60})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328b60)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006efe00, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a4540, {0x23d1088, 0xc0006efe00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 776 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 775 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016ad80, {0x2392228, 0xc0014c2780})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1678871 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00066d200)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0630, {0x1f7b7c0, 0xc001fbc140}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8c700, 0xc000ad9bf8, 0xc000ad9bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8c700, {0x1f7b7c0, 0xc001fbc140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00131efc0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0x20, {0xc001c65a80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00131efc0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00131efc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0xc0019bd760, {0xc001c65a80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00131efc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0xc00119e640, 0xc00131eff0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00131efc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001316b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0x33, 0xc00131ef30, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001316b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001316b00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001316a80}, {0x1f7b7c0, 0xc001fbc140}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc001316b00}, 0xc00131a0c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc00131e8d0}, 0xc001316a80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc00131e8d0}, 0xc001316a80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000ada480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000ada528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8cbe08, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00131e8d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001ed6300})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000ada778, {0x235a640, 0xc000045ad0}, 0x392d346365342d63)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00131e8d0}, 0xc001316780)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00181cdb8, 0x4c1cb4}, {0xc057f6daf82fb1ab, 0x8c0ae30af7}, {0xc00181cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc00131e8d0}, 0xc001316780)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780}, 0xc001c659a0, 0xc0021efdd0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780}, 0x15, 0xc0013167c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780}, 0x2076951, 0xc0013167c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780}, 0x237bc38, 0xc0013167c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e8d0}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00131e780}, {0x1f33320, 0xc001316780}, 0x1e5a940, 0xc0013167c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e780}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780}, 0x23c27b0, 0xc0013167c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00131e750}, {0x1f33320, 0xc001316780}, 0xc001c659a0, 0xc0021efdd0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00131e750}, 0xc00220b800, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0022ed440, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0022ed440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658059 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c34c60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e833f0, {0x1f7b7c0, 0xc001fbd2c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xbf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40000, 0xc0017c7bf8, 0xc0017c7bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40000, {0x1f7b7c0, 0xc001fbd2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f79830}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0x20, {0xc001bc5040, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f79830}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f79830}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0xc000ea9600, {0xc001bc5040, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f79830}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0xc00119e640, 0xc001f79860, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f79830}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001367dc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0x237ac70, 0xc001f797a0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001367dc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001367dc0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001367d40}, {0x1f7b7c0, 0xc001fbd2c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc001367dc0}, 0xc0003efb20, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc001f790e0}, 0xc001367d40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc001f790e0}, 0xc001367d40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017c8480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017c8528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd81b890, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001f790e0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000c61a20})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017c8778, {0x235a640, 0xc000045ad0}, 0x612d636663342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc001f790e0}, 0xc001367a40)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000774db8, 0x4c1cb4}, {0xc057f6d9371c5c84, 0x8a68942fcf}, {0xc000774db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc001f790e0}, 0xc001367a40)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40}, 0xc001bc4f60, 0xc002289458)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40}, 0x15, 0xc001367a80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40}, 0x2076951, 0xc001367a80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40}, 0x237bc38, 0xc001367a80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f790e0}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f78f90}, {0x1f33320, 0xc001367a40}, 0x1e5a940, 0xc001367a80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f78f90}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40}, 0x23c27b0, 0xc001367a80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f78f60}, {0x1f33320, 0xc001367a40}, 0xc001bc4f60, 0xc002289458)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc001f78f60}, 0xc001e8d920, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbf680, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbf680, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1677985 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd9204f8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ca9c00, 0xc000e66000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ca9c00, {0xc000e66000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ca9c00, {0xc000e66000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278ebb8, {0xc000e66000, 0xff90, 0xc000cd74a0})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0cc00, {0xc0006ba900, 0x9, 0xc000716400})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0cc00}, {0xc0006ba900, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba900, 0x9, 0xc001f91e00}, {0x23585c0, 0xc001c0cc00})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba8c0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001259980, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a41c0, {0x23d1088, 0xc001259980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 1658053 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020ee3c0, {0x2392298, 0xc001e8d200}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019c3c20, {0x23922d0, 0xc0024cddd0}, 0x1e)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc00204d530, {0x23922d0, 0xc0024cddd0}, 0xc001f86160, 0x1, 0x1f86170)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0024cda40, 0xc001367040)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0024cd770}, 0xc001367040)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0024cd770}, {0x1f35540, 0xc001367040})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0024cd770}, {0x1f35540, 0xc001367040}, 0x237bc38, 0xc002288b40)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024cd770}, {0x1f35540, 0xc001367040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0024cd5f0}, {0x1f35540, 0xc001367040}, 0x203000, 0xc0013670c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024cd5f0}, {0x1f35540, 0xc001367040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0024cd5f0}, {0x1f35540, 0xc001367040}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024cd5f0}, {0x1f35540, 0xc001367040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0024cd590}, {0x1f35540, 0xc001367040}, 0x40, 0xc0013670c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024cd590}, {0x1f35540, 0xc001367040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0024cd590}, {0x1f35540, 0xc001367040}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0024cd590}, {0x1f35540, 0xc001367040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0024cd590}, {0x1f35540, 0xc001367040}, 0xc001bc4d40, 0xc002288b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0024cd590}, 0xc001e8d1a0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0004099e0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc0004099e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657947 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bc8240)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022eb600, {0x1f7b560, 0xc001de3680}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8c500, 0xc001daa440, 0xc001daa410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8c500, {0x1f7b560, 0xc001de3680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001fd2270}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0x20, {0xc0007c3760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001fd2270}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd2270}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0xc0020afb80, {0xc0007c3760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd2270}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0xc0023f4400, 0xc001fd22a0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd2270}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013184c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0x237ac70, 0xc001fd21b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013184c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013184c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001318440}, {0x1f7b560, 0xc001de3680}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0013184c0}, 0xc001c20d56, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc0023ecea0}, 0xc001318440, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc0023ecea0}, 0xc001318440, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0023ecae0, {0x2392298, 0xc0023ecea0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023ece40, {0x2392298, 0xc0023ecea0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00112fb00, {0x23922d0, 0xc00240f200}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00240f200}, 0xc001451000, 0x1bd58c0, 0x1451010)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc00240f1a0, 0xc0013c7b80)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc00240ef00}, 0xc0013c7b80)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00240ef00}, {0x1f353a0, 0xc0013c7b80})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00240ef00}, {0x1f353a0, 0xc0013c7b80}, 0x237bc38, 0xc0020053e0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00240ef00}, {0x1f353a0, 0xc0013c7b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00240ed80, {0x23922d0, 0xc00240edb0}, {0x1f353a0, 0xc0013c7b80}, 0xc0025819e0, 0xc0013c7bc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00240edb0}, {0x1f353a0, 0xc0013c7b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00240edb0}, {0x1f353a0, 0xc0013c7b80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00240edb0}, {0x1f353a0, 0xc0013c7b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00240ed20}, {0x1f353a0, 0xc0013c7b80}, 0x40, 0xc0013c7bc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00240ed20}, {0x1f353a0, 0xc0013c7b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00240ed20}, {0x1f353a0, 0xc0013c7b80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00240ed20}, {0x1f353a0, 0xc0013c7b80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00240ed20}, {0x1f353a0, 0xc0013c7b80}, 0xc0007c35e0, 0xc0020053e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00240ed20}, 0xc0023ec480, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc00112eea0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc00112eea0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1218 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010f48f0, 0xc0021c9cf0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010f48f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1216 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010f48f0, 0xc0012090c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010f48f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1217 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010f48f0, 0xc0014ebda0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010f48f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1658099 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003d88c0, {0x2392228, 0xc000e64b80})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1681107 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00065cd80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00062cf20, {0x1f736c0, 0xc0020e2c60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446c00, 0xc00076f5c0, 0xc00076f590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446c00, {0x1f736c0, 0xc0020e2c60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001dc1b00}, {0x2076920, 0x8}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x1c75901, {0xc00053bf20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x7f00048b45b8, {0xc00053bf20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x20, 0xc0006a42d0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0xc000bfb110, {0xc001fa5980, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x0, 0xc0006a42a0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x0, 0xc0006a4210, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001dc1b00}, {0x2076920, 0x203000}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x203000, 0xc0006a41e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001dc1b00}, {0x2076920, 0x46}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000729e50, {0x2392298, 0xc001dc1b00}, {0x2076920, 0x201e366d0}, {0x1f33320, 0xc001fa5940}, {0x1f736c0, 0xc0020e2c60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc001dc1b00}, 0x1fe4e76, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc001dc1b00})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 994 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc000f80600)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a540f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a540f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1480910 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00243b680, {0x2392228, 0xc001420b80})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654709 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c34900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116dc30, {0x1f736c0, 0xc0021a8bd0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8df00, 0xc000c095c0, 0xc000c09590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8df00, {0x1f736c0, 0xc0021a8bd0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002362b40}, {0x2076920, 0x8}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x1c75901, {0xc0000840a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x7f00048b45b8, {0xc0000840a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x20, 0xc0021becf0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0xc000bfb110, {0xc0021ab740, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x0, 0xc0021becc0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x1e0cfc0, 0xc0021bec30, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x3d6034e60fa, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002362b40}, {0x2076920, 0x203000}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x203000, 0xc0021bec00, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002362b40}, {0x2076920, 0x46}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0011b6650, {0x2392298, 0xc002362b40}, {0x2076920, 0x1011458b8}, {0x1f33320, 0xc0021ab700}, {0x1f736c0, 0xc0021a8bd0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc002362b40}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc002362b40})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1090 [select, 10 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000c25f40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1654784 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fdad80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80370, {0x1f7c860, 0xc001b802c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001447300, 0xc0017ab5e8, 0xc0017ab5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001447300, {0x1f7c860, 0xc001b802c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x8}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x1c75901, {0xc00230aa60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x7f00048b45b8, {0xc00230aa60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x20, 0xc0021eb140, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0xc000f5fa00, {0xc0021dde00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x0, 0xc0021eb110, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x1e0cfc0, 0xc0021eb080, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x7effdd877b00, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x8, 0xc0021eb050, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0021cd6e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc0021cd6e0}, {0x20768da, 0x0}, {0x1f33180, 0xc0021eaff0}, {0x1f7c860, 0xc001b802c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0021cd6e0}, 0x60, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc0021cd6e0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1678717 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a56a20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80a50, {0x1f736c0, 0xc0012078c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7f00, 0xc00173d5c0, 0xc00173d590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7f00, {0x1f736c0, 0xc0012078c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001f51680}, {0x2076920, 0x8}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x1c75901, {0xc000ebaf00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x7f00048b45b8, {0xc000ebaf00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x20, 0xc0022b0b10, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0xc000bfb110, {0xc001241000, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x0, 0xc0022b09f0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x0, 0xc0022b0900, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001f51680}, {0x2076920, 0x203000}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x203000, 0xc0022b08a0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001f51680}, {0x2076920, 0x46}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002848e50, {0x2392298, 0xc001f51680}, {0x2076920, 0x200000001}, {0x1f33320, 0xc001240fc0}, {0x1f736c0, 0xc0012078c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc001f51680}, 0x2, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc001f51680})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1007 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd865168, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872380, 0xc000b92000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000872380, {0xc000b92000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000872380, {0xc000b92000, 0x60100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e588, {0xc000b92000, 0x4383e0, 0xc000a37380})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000bd8a80, {0xc000328ac0, 0x9, 0xc00083bd18})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000bd8a80}, {0xc000328ac0, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000328ac0, 0x9, 0x18}, {0x23585c0, 0xc000bd8a80})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328a80)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0007303c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1654786 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001192900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022ea790, {0x1f7b560, 0xc001941ae0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8de00, 0xc0009b5d18, 0xc0009b5ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8de00, {0x1f7b560, 0xc001941ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002218840}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0x20, {0xc000e80b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002218840}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002218840}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0xc001b90580, {0xc000e80b60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002218840}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0xc00119e640, 0xc002218870, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002218840}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00221a200}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0x237ac70, 0xc0022187b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00221a200}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00221a200}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc00221a180}, {0x1f7b560, 0xc001941ae0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc00221a200}, 0xc00265a0f0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022180f0}, 0xc00221a180, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022180f0}, 0xc00221a180, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0009b65a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0009b6648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd822c38, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0022180f0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002206660})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0009b6898, {0x235a640, 0xc000045ad0}, 0x382d356266342d64)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022180f0}, 0xc00220ff20)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000a3cdb8, 0x4c1cb4}, {0xc057f6d8f6e1222a, 0x8a2cbe2b72}, {0xc000a3cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0022180f0}, 0xc00220ff20)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20}, 0xc000e809a0, 0xc002212a38)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20}, 0x15, 0xc002203f40)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20}, 0x207690b, 0xc002203f40)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20}, 0x237bc38, 0xc002203f40)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022180f0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00220ff80}, {0x1f33180, 0xc00220ff20}, 0x1e5a940, 0xc002203f40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00220ff80}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20}, 0x7effdd813118, 0xc002203f40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20}, 0xc001d61768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001d61b60, {0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00220fef0}, {0x1f33180, 0xc00220ff20}, 0xc000e809a0, 0xc002212a38)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00220fef0}, 0xc0022e6300, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00077e6c0, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00077e6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678679 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000128ea0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80790, {0x1f736c0, 0xc001206360}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7700, 0xc000c0d5c0, 0xc000c0d590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7700, {0x1f736c0, 0xc001206360})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000f97320}, {0x2076920, 0x8}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x1c75901, {0xc000eba6e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x7f00048b45b8, {0xc000eba6e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x20, 0xc00206cae0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0xc000bfb110, {0xc001200c40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x0, 0xc00206cab0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x0, 0xc00206ca20, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000f97320}, {0x2076920, 0x203000}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x203000, 0xc00206c9f0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000f97320}, {0x2076920, 0x46}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001976e50, {0x2392298, 0xc000f97320}, {0x2076920, 0x200000001}, {0x1f33320, 0xc001200c00}, {0x1f736c0, 0xc001206360}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc000f97320}, 0x2, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc000f97320})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1219 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a552c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1148 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000a1edc0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1149 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000379960, {0x2392228, 0xc00069f800})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1150 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0011123c0, {0x2392228, 0xc00069f840})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1151 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0011123c0, {0x2392228, 0xc00069f840})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1651759 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0023ed6e0, {0x2392228, 0xc0014034c0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1657955 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005d0b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80e70, {0x1f7c860, 0xc001b80c60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391700, 0xc0017235e8, 0xc0017235b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391700, {0x1f7c860, 0xc001b80c60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x8}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x1c75901, {0xc00230ab60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x7f00048b4108, {0xc00230ab60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x20, 0xc001e9b5f0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0xc000f5fa00, {0xc0012daa80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x0, 0xc001e9b5c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x1e0cfc0, 0xc001e9b530, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x203000}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x8, 0xc001e9b500, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001fdc5a0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, 0x144aacf, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc001fdc5a0}, {0x20768da, 0x0}, {0x1f33180, 0xc001e9b4a0}, {0x1f7c860, 0xc001b80c60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc001fdc5a0}, 0xc000a6fee0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc001fdc5a0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1177 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bd7360, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000ba2300)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 1214 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a553b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1179 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bd75e0, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000ba2d20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef
goroutine 1180 [select, 10 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00081b500)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837
goroutine 1181 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920898, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000a4df00, 0xc001762000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a4df00, {0xc001762000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000a4df00, {0xc001762000, 0x1060100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e9b8, {0xc001762000, 0xc001743cf8, 0x44d2f4})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000ba23c0, {0xc000328d60, 0x9, 0x9772effbd7})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000ba23c0}, {0xc000328d60, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000328d60, 0x9, 0xc000bd7630}, {0x23585c0, 0xc000ba23c0})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328d20)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00081b500, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a4540, {0x23d1088, 0xc00081b500})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 1678752 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0019ab560, {0x2392298, 0xc0020a6480}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019c30e0, {0x23922d0, 0xc0022fe3f0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002689230, {0x23922d0, 0xc0022fe3f0}, 0xc0012af040, 0x1, 0x12af050)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0022fe390, 0xc001283940)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022fe060}, 0xc001283940)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022fe060}, {0x1f35540, 0xc001283940})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022fe060}, {0x1f35540, 0xc001283940}, 0x237bc38, 0xc00237c630)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022fe060}, {0x1f35540, 0xc001283940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc002315ef0}, {0x1f35540, 0xc001283940}, 0xc0012832c0, 0xc0012839c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315ef0}, {0x1f35540, 0xc001283940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002315ef0}, {0x1f35540, 0xc001283940}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315ef0}, {0x1f35540, 0xc001283940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002315ec0}, {0x1f35540, 0xc001283940}, 0x40, 0xc0012839c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315ec0}, {0x1f35540, 0xc001283940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002315ec0}, {0x1f35540, 0xc001283940}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315ec0}, {0x1f35540, 0xc001283940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002315ec0}, {0x1f35540, 0xc001283940}, 0xc001c646a0, 0xc00237c630)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002315ec0}, 0xc0020a6420, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000bba240, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000bba240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657987 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00195b9e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e82790, {0x1f7c860, 0xc001e82630}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391c00, 0xc00172b5e8, 0xc00172b5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391c00, {0x1f7c860, 0xc001e82630})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001fdda40}, {0x20768da, 0x8}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x1c75901, {0xc00230b720, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x7f00048b4108, {0xc00230b720, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x20, 0xc001f129c0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0xc000f5fa00, {0xc0024ff180, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x0, 0xc001f12990, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x1e0cfc0, 0xc001f12870, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001fdda40}, {0x20768da, 0x203000}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x8, 0xc001f12840, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001fdda40}, {0x20768da, 0x46}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, 0x7, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc001fdda40}, {0x20768da, 0x0}, {0x1f33180, 0xc001f127e0}, {0x1f7c860, 0xc001e82630}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc001fdda40}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc001fdda40})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1657961 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c52b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e822c0, {0x1f7c860, 0xc001b81970}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001391900, 0xc0017a35e8, 0xc0017a35b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001391900, {0x1f7c860, 0xc001b81970})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001fdce40}, {0x20768da, 0x8}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x1c75901, {0xc00230af20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x7f00048b4108, {0xc00230af20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x20, 0xc00125f650, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0xc000f5fa00, {0xc0012db800, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x0, 0xc00125f620, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x1e0cfc0, 0xc00125f590, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001fdce40}, {0x20768da, 0x203000}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x8, 0xc00125f560, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001fdce40}, {0x20768da, 0x46}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, 0x144aacf, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc001fdce40}, {0x20768da, 0x0}, {0x1f33180, 0xc00125f500}, {0x1f7c860, 0xc001b81970}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc001fdce40}, 0xc000a6fee0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc001fdce40})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1661118 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002571200, {0x2392298, 0xc0022df320}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000ead320, {0x23922d0, 0xc002112e40}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002112e40}, 0xc002114420, 0x1bd58c0, 0x2114430)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc002112c30, 0xc0021200c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc002112990}, 0xc0021200c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002112990}, {0x1f353a0, 0xc0021200c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002112990}, {0x1f353a0, 0xc0021200c0}, 0x237bc38, 0xc002102f18)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002112990}, {0x1f353a0, 0xc0021200c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x2392298, {0x23922d0, 0xc002112840}, {0x1f353a0, 0xc0021200c0}, 0x2392298, 0xc002120100)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002112840}, {0x1f353a0, 0xc0021200c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002112840}, {0x1f353a0, 0xc0021200c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002112840}, {0x1f353a0, 0xc0021200c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021127e0}, {0x1f353a0, 0xc0021200c0}, 0x40, 0xc002120100)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021127e0}, {0x1f353a0, 0xc0021200c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021127e0}, {0x1f353a0, 0xc0021200c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021127e0}, {0x1f353a0, 0xc0021200c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021127e0}, {0x1f353a0, 0xc0021200c0}, 0xc001c64dc0, 0xc002102f18)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0021127e0}, 0xc0022df2c0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001b4a900, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001b4a900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1654877 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003533b0, {0x2392228, 0xc00223b500})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1658050 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003d8700, {0x2392228, 0xc001366bc0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1264 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000a4c500, 0xc0013b22d0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000a4c500)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1831997 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd920980, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000dc9e80, 0xc00129b751, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000dc9e80, {0xc00129b751, 0x1, 0x1})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000dc9e80, {0xc00129b751, 0x7, 0xc0011bbef0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00221cbd8, {0xc00129b751, 0x1, 0x100000001})
/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc00129b740)
/usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
/usr/local/go/src/net/http/server.go:668 +0xcf
goroutine 1829725 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002571200, {0x2392298, 0xc0024eff80}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000ead320, {0x23922d0, 0xc001dcf410}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001dcf410}, 0xc00201c390, 0x1bd58c0, 0x201c3a0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001dcf200, 0xc001970e00)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001dcef60}, 0xc001970e00)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001dcef60}, {0x1f353a0, 0xc001970e00})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001dcef60}, {0x1f353a0, 0xc001970e00}, 0x237bc38, 0xc000fe7d88)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dcef60}, {0x1f353a0, 0xc001970e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001dcee10}, {0x1f353a0, 0xc001970e00}, 0x203000, 0xc001970e40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dcee10}, {0x1f353a0, 0xc001970e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001dcee10}, {0x1f353a0, 0xc001970e00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dcee10}, {0x1f353a0, 0xc001970e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001dcedb0}, {0x1f353a0, 0xc001970e00}, 0x40, 0xc001970e40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dcedb0}, {0x1f353a0, 0xc001970e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001dcedb0}, {0x1f353a0, 0xc001970e00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dcedb0}, {0x1f353a0, 0xc001970e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001dcedb0}, {0x1f353a0, 0xc001970e00}, 0xc0022b4680, 0xc000fe7d88)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001dcedb0}, 0xc0024eff20, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc000bca240, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc000bca240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1311612 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000ba3f20, {0x2392228, 0xc00212d000})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654770 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005bafc0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b800b0, {0x1f736c0, 0xc0021d8b40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5f)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446f00, 0xc0017a75c0, 0xc0017a7590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446f00, {0x1f736c0, 0xc0021d8b40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0021cc900}, {0x2076920, 0x8}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x1c75901, {0xc00230a540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x7f00048b45b8, {0xc00230a540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x20, 0xc0021de750, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0xc000bfb110, {0xc0021dc900, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x0, 0xc0021de720, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x1e0cfc0, 0xc0021de690, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0xc0d3f1fdc7, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0021cc900}, {0x2076920, 0x203000}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x203000, 0xc0021de660, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0021cc900}, {0x2076920, 0x46}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0011b9e50, {0x2392298, 0xc0021cc900}, {0x2076920, 0x100000000}, {0x1f33320, 0xc0021dc8c0}, {0x1f736c0, 0xc0021d8b40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc0021cc900}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010f48f0, {0x2392298, 0xc0021cc900})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1678723 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001af87e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80bb0, {0x1f736c0, 0xc001207e60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446200, 0xc0017a95c0, 0xc0017a9590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446200, {0x1f736c0, 0xc001207e60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001f51f80}, {0x2076920, 0x8}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x1c75901, {0xc000ebb2e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x7f00048b45b8, {0xc000ebb2e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x20, 0xc00213d350, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0xc000bfb110, {0xc001241dc0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x0, 0xc00213d320, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x0, 0xc00213d290, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001f51f80}, {0x2076920, 0x203000}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x203000, 0xc00213d260, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001f51f80}, {0x2076920, 0x46}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002848e50, {0x2392298, 0xc001f51f80}, {0x2076920, 0x2e40fa347}, {0x1f33320, 0xc001241d80}, {0x1f736c0, 0xc001207e60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc001f51f80}, 0x1e32ea0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010f48f0, {0x2392298, 0xc001f51f80})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1480930 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0014195e0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1661083 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0009f67e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012fa790, {0x1f7c860, 0xc0012fa580}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6300, 0xc0017a55e8, 0xc0017a55b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6300, {0x1f7c860, 0xc0012fa580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002245860}, {0x20768da, 0x8}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x1c75901, {0xc000ebb760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x7f00048b45b8, {0xc000ebb760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x20, 0xc00206d050, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0xc000f5fa00, {0xc002072980, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x0, 0xc00206d020, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x1e0cfc0, 0xc00206cf90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002245860}, {0x20768da, 0x203000}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x8, 0xc00206cf60, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002245860}, {0x20768da, 0x46}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, 0x7, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc002245860}, {0x20768da, 0x0}, {0x1f33180, 0xc00206cf00}, {0x1f7c860, 0xc0012fa580}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc002245860}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc001198e00, {0x2392298, 0xc002245860})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658125 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005c46c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116c580, {0x1f736c0, 0xc000b1e240}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40b00, 0xc00180b5c0, 0xc00180b590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40b00, {0x1f736c0, 0xc000b1e240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00263e4e0}, {0x2076920, 0x8}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x1c75901, {0xc000f818a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x7f00048b4108, {0xc000f818a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x20, 0xc00200bb00, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0xc000bfb110, {0xc0010dc0c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x0, 0xc00200bad0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x1e0cfc0, 0xc00200ba40, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0xc0034e60fa, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00263e4e0}, {0x2076920, 0x203000}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x203000, 0xc00200ba10, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00263e4e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001215650, {0x2392298, 0xc00263e4e0}, {0x2076920, 0x12f314c4f}, {0x1f33320, 0xc0010dc080}, {0x1f736c0, 0xc000b1e240}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc00263e4e0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc00263e4e0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658008 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b4a7e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2370, {0x1f7b560, 0xc001f4c5a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40e00, 0xc002882440, 0xc002882410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40e00, {0x1f7b560, 0xc001f4c5a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002112300}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0x20, {0xc001c64ca0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002112300}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002112300}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0xc0010d3b80, {0xc001c64ca0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002112300}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0xc0023f4400, 0xc002112330, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002112300}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0020e9e40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0x237ac70, 0xc002112270, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0020e9e40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0020e9e40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0020e9dc0}, {0x1f7b560, 0xc001f4c5a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0020e9e40}, 0xc0003ef026, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc002428960}, 0xc0020e9dc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc002428960}, 0xc0020e9dc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0023aa9c0, {0x2392298, 0xc002428960})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023aac00, {0x2392298, 0xc002428960}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005f1680, {0x23922d0, 0xc001f087e0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001f087e0}, 0xc0013b2cd0, 0x1bd58c0, 0x13b2ce0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001f08780, 0xc001f82f40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001f084e0}, 0xc001f82f40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f084e0}, {0x1f353a0, 0xc001f82f40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001f084e0}, {0x1f353a0, 0xc001f82f40}, 0x237bc38, 0xc002425a28)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f084e0}, {0x1f353a0, 0xc001f82f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001f08360, {0x23922d0, 0xc001f08390}, {0x1f353a0, 0xc001f82f40}, 0xc001291470, 0xc001f82f80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f08390}, {0x1f353a0, 0xc001f82f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f08390}, {0x1f353a0, 0xc001f82f40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f08390}, {0x1f353a0, 0xc001f82f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f08300}, {0x1f353a0, 0xc001f82f40}, 0x40, 0xc001f82f80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f08300}, {0x1f353a0, 0xc001f82f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f08300}, {0x1f353a0, 0xc001f82f40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f08300}, {0x1f353a0, 0xc001f82f40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f08300}, {0x1f353a0, 0xc001f82f40}, 0xc000be4be0, 0xc002425a28)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001f08300}, 0xc002428900, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019070e0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019070e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1674690 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:627 +0x152
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 1678708 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bbe6c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b808f0, {0x1f736c0, 0xc001206fc0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7d00, 0xc0017d55c0, 0xc0017d5590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7d00, {0x1f736c0, 0xc001206fc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001f50960}, {0x2076920, 0x8}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x1c75901, {0xc000ebab40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x7f00048b45b8, {0xc000ebab40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x20, 0xc0020fe480, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0xc000bfb110, {0xc001240040, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x0, 0xc0020fe450, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x41774b, 0xc0020fe3c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x4175cd, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001f50960}, {0x2076920, 0x203000}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x203000, 0xc0020fe390, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001f50960}, {0x2076920, 0x46}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002848e50, {0x2392298, 0xc001f50960}, {0x2076920, 0x2011458b8}, {0x1f33320, 0xc001240000}, {0x1f736c0, 0xc001206fc0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc001f50960}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010f48f0, {0x2392298, 0xc001f50960})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1829688 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019fd7a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0c60, {0x1f736c0, 0xc002128240}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5f)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446e00, 0xc0017d75c0, 0xc0017d7590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446e00, {0x1f736c0, 0xc002128240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00241b200}, {0x2076920, 0x8}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x1c75901, {0xc000f80f80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x7f00048b4108, {0xc000f80f80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x20, 0xc000a05c80, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0xc000bfb110, {0xc000f1e000, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x0, 0xc000a05c50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x1e0cfc0, 0xc000a05b90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0xc015c64ea7, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00241b200}, {0x2076920, 0x203000}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x203000, 0xc000a05b60, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00241b200}, {0x2076920, 0x46}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000a6ee50, {0x2392298, 0xc00241b200}, {0x2076920, 0x100bfa8f0}, {0x1f33320, 0xc00212dfc0}, {0x1f736c0, 0xc002128240}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc00241b200}, 0xc0007e7b80, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010f48f0, {0x2392298, 0xc00241b200})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1654703 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1ec60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116da20, {0x1f736c0, 0xc0021a85a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8dc00, 0xc0019595c0, 0xc001959590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8dc00, {0x1f736c0, 0xc0021a85a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002362240}, {0x2076920, 0x8}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x1c75901, {0xc001bc5ac0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x7f00048b45b8, {0xc001bc5ac0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x20, 0xc0021accc0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0xc000bfb110, {0xc0021aa6c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x0, 0xc0021acc90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0xc0011b6418, 0xc0021acc00, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x3d6034e60fa, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002362240}, {0x2076920, 0x203000}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x203000, 0xc0021acbd0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002362240}, {0x2076920, 0x46}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0011b6650, {0x2392298, 0xc002362240}, {0x2076920, 0x142974201}, {0x1f33320, 0xc0021aa680}, {0x1f736c0, 0xc0021a85a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc002362240}, 0x11b66a0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc002362240})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1475043 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc000f97b00, {0x2392228, 0xc001737b40})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1661131 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003cf570, {0x2392228, 0xc002190840})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1654716 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bba480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116dd90, {0x1f736c0, 0xc0021a9320}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446800, 0xc001c195c0, 0xc001c19590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446800, {0x1f736c0, 0xc0021a9320})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0023632c0}, {0x2076920, 0x8}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x1c75901, {0xc000084ea0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x7f00048b45b8, {0xc000084ea0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x20, 0xc0021c47e0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0xc000bfb110, {0xc0021c2580, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x0, 0xc0021c47b0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x1e0cfc0, 0xc0021c4720, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x3d6034e60fa, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0023632c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x203000, 0xc0021c46f0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0023632c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0011b6650, {0x2392298, 0xc0023632c0}, {0x2076920, 0x1011458b8}, {0x1f33320, 0xc0021c2540}, {0x1f736c0, 0xc0021a9320}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc0023632c0}, 0x11b66a0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001194d00, {0x2392298, 0xc0023632c0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1654719 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00250de60, {0x2392298, 0xc002363a40}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000fe85a0, {0x23922d0, 0xc0021ce870}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001e17230, {0x23922d0, 0xc0021ce870}, 0xc0021c8820, 0x1, 0x21c8830)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0021ce660, 0xc0021c3280)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0021ce3c0}, 0xc0021c3280)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021ce3c0}, {0x1f35540, 0xc0021c3280})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0021ce3c0}, {0x1f35540, 0xc0021c3280}, 0x237bc38, 0xc0021ca210)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ce3c0}, {0x1f35540, 0xc0021c3280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0021ce270}, {0x1f35540, 0xc0021c3280}, 0xc0021c2780, 0xc0021c3300)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ce270}, {0x1f35540, 0xc0021c3280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021ce270}, {0x1f35540, 0xc0021c3280}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ce270}, {0x1f35540, 0xc0021c3280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021ce240}, {0x1f35540, 0xc0021c3280}, 0x40, 0xc0021c3300)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ce240}, {0x1f35540, 0xc0021c3280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021ce240}, {0x1f35540, 0xc0021c3280}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ce240}, {0x1f35540, 0xc0021c3280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021ce240}, {0x1f35540, 0xc0021c3280}, 0xc00230a2c0, 0xc0021ca210)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0021ce240}, 0xc0023639e0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001af98c0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001af98c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658142 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0003bcd80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bacf20, {0x1f7b560, 0xc0019417c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8de00, 0xc001990440, 0xc001990410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8de00, {0x1f7b560, 0xc0019417c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00133c030}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0x20, {0xc000eba420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00133c030}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00133c030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0xc002084840, {0xc000eba420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00133c030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0xc000c61270, 0xc00133c060, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00133c030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0011704c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0x237ac70, 0xc002127f80, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0011704c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0011704c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001170440}, {0x1f7b560, 0xc0019417c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0011704c0}, 0xc001c09cc6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc002398120}, 0xc001170440, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc002398120}, 0xc001170440, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0021d2000, {0x2392298, 0xc002398120})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021d22a0, {0x2392298, 0xc002398120}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001c1efc0, {0x23922d0, 0xc002127c80}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002127c80}, 0xc001dd7d80, 0x1bd58c0, 0x1dd7d90)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc002127c20, 0xc001170240)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc002127980}, 0xc001170240)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002127980}, {0x1f353a0, 0xc001170240})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002127980}, {0x1f353a0, 0xc001170240}, 0x237bc38, 0xc00209b650)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002127980}, {0x1f353a0, 0xc001170240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002127800, {0x23922d0, 0xc002127830}, {0x1f353a0, 0xc001170240}, 0xc002126ae0, 0xc001170280)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002127830}, {0x1f353a0, 0xc001170240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002127830}, {0x1f353a0, 0xc001170240}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002127830}, {0x1f353a0, 0xc001170240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021277a0}, {0x1f353a0, 0xc001170240}, 0x40, 0xc001170280)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021277a0}, {0x1f353a0, 0xc001170240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021277a0}, {0x1f353a0, 0xc001170240}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021277a0}, {0x1f353a0, 0xc001170240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021277a0}, {0x1f353a0, 0xc001170240}, 0xc000eba380, 0xc00209b650)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0021277a0}, 0xc0023980c0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019759e0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019759e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678007 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd864b10, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ca9d00, 0xc000db4000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ca9d00, {0xc000db4000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ca9d00, {0xc000db4000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278ebd0, {0xc000db4000, 0x9, 0xc015c64ea7})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0cea0, {0xc0006bac80, 0x9, 0xc00085a000})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0cea0}, {0xc0006bac80, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006bac80, 0x9, 0xc00131c300}, {0x23585c0, 0xc001c0cea0})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006bac40)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001259b00, 0x4f9b01, 0x1418c25)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a41c0, {0x23d1088, 0xc001259b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 1269 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000e85180)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1270 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000fdc70, {0x2392228, 0xc000ea6b40})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1271 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000e886c0, {0x2392228, 0xc000ea6c00})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1272 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc000e886c0, {0x2392228, 0xc000ea6c00})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654722 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353110, {0x2392228, 0xc002123180})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1661171 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003d9420, {0x2392228, 0xc0022e2dc0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1678774 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0019ab560, {0x2392298, 0xc0020a6a20}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019c30e0, {0x23922d0, 0xc0022ffef0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0026895c0, {0x23922d0, 0xc0022ffef0}, 0xc0012af940, 0x1, 0x12af950)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0022ffe00, 0xc0012cc440)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022ffb90}, 0xc0012cc440)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022ffb90}, {0x1f35540, 0xc0012cc440})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022ffb90}, {0x1f35540, 0xc0012cc440}, 0x237bc38, 0xc00237ce28)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ffb90}, {0x1f35540, 0xc0012cc440})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0022ffa40}, {0x1f35540, 0xc0012cc440}, 0xc001283ec0, 0xc0012cc4c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ffa40}, {0x1f35540, 0xc0012cc440})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022ffa40}, {0x1f35540, 0xc0012cc440}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ffa40}, {0x1f35540, 0xc0012cc440})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022ffa10}, {0x1f35540, 0xc0012cc440}, 0x40, 0xc0012cc4c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ffa10}, {0x1f35540, 0xc0012cc440})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022ffa10}, {0x1f35540, 0xc0012cc440}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022ffa10}, {0x1f35540, 0xc0012cc440})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022ffa10}, {0x1f35540, 0xc0012cc440}, 0xc001c64a60, 0xc00237ce28)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0022ffa10}, 0xc0020a69c0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001dcd440, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001dcd440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657989 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bcb8c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e82b00, {0x1f7b560, 0xc001b159a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446100, 0xc000ee3d18, 0xc000ee3ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446100, {0x1f7b560, 0xc001b159a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f139e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0x20, {0xc00230b900, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f139e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f139e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0xc000cda2c0, {0xc00230b900, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f139e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0xc00119e640, 0xc001f13a70, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f139e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0024ff640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0x237ac70, 0xc001f13920, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0024ff640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0024ff640}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0024ff5c0}, {0x1f7b560, 0xc001b159a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0024ff640}, 0xc00204c720, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc001f13200}, 0xc0024ff5c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc001f13200}, 0xc0024ff5c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000ee45a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000ee4648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd818a18, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc001f13200}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000c60360})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000ee4898, {0x235a640, 0xc000045ad0}, 0x392d646535342d37)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc001f13200}, 0xc001f12f60)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000772db8, 0x4c1cb4}, {0xc057f6d935edebfd, 0x8a6765bf48}, {0xc000772db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc001f13200}, 0xc001f12f60)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60}, 0xc00230b820, 0xc00205dea8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60}, 0x15, 0xc0024ff380)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60}, 0x207690b, 0xc0024ff380)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60}, 0x237bc38, 0xc0024ff380)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f13200}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f13020}, {0x1f33180, 0xc001f12f60}, 0x1e5a940, 0xc0024ff380)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f13020}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60}, 0x9, 0xc0024ff380)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60}, 0xc001bf5768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f12f30}, {0x1f33180, 0xc001f12f60}, 0xc00230b820, 0xc00205dea8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc001f12f30}, 0xc001fddc80, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bca000, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bca000, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1829723 [IO wait]:
internal/poll.runtime_pollWait(0x7effdda99660, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0024b0880, 0xc000d12000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0024b0880, {0xc000d12000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0024b0880, {0xc000d12000, 0xc0024b0898, 0xc000d12000})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0020cb108, {0xc000d12000, 0x448e71, 0x466a4e})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc001906000, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc001906000, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc001906000)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlConn).readResultSetHeaderPacket(0xc0023e5c20)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:537 +0x25
github.com/go-sql-driver/mysql.(*mysqlStmt).query(0xc0023e5c20, {0xc000e96a00, 0xc0021d29c0, 0xc000e9a720})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/statement.go:108 +0x125
github.com/go-sql-driver/mysql.(*mysqlStmt).QueryContext(0xc0023e5c20, {0x2392298, 0xc0021d29c0}, {0xc000e9a680, 0x94, 0x208f76a})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:565 +0x91
database/sql.ctxDriverStmtQuery({0x2392298, 0xc0021d29c0}, {0x2392458, 0xc0023e5c20}, {0xc000e9a680, 0x5, 0x5})
/usr/local/go/src/database/sql/ctxutil.go:82 +0x132
database/sql.rowsiFromStatement({0x2392298, 0xc0021d29c0}, {0x237efb0, 0xc001906000}, 0xc000e8d240, {0xc000e4ba90, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:2794 +0x17e
database/sql.(*DB).queryDC(0x203001, {0x2392298, 0xc0021d29c0}, {0x0, 0x0}, 0xc00249fb00, 0xc00127dd70, {0x208f76a, 0x94}, {0xc000e4ba90, ...})
/usr/local/go/src/database/sql/sql.go:1752 +0x3d7
database/sql.(*DB).query(0x1df9548, {0x2392298, 0xc0021d29c0}, {0x208f76a, 0x94}, {0xc000e4ba90, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0021d29c0}, {0x208f76a, 0x94}, {0xc000e4ba90, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0021d29c0}, {0x208f76a, 0xc00004e800}, {0xc000e4ba90, 0xc001df96c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0021d29c0}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0025f12c0}, {0x208f76a, 0x0}, {0xc000e4ba90, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x6, {0x2392298, 0xc0021d29c0}, {0x1b5d8e0, 0xc0025f12c0}, {0x208f76a, 0xc001df97e0}, {0xc000e4ba90, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0021d29c0}, {0x8c92cc73, {0xc001aedb90, 0x2d, 0x2d}, 0x0, 0xc000e3fba8, 0xc00210eb18, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e3fb80)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x1c)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0024ef500, 0xfed260, 0x1005900, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc00085a000, 0xc00201c030})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001df9c50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc001df9d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001df9de2, 0xc000749860)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001df9d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc0024ef680)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0024ef680, {0x2392228, 0xc0019708c0})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1661216 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002244780, {0x2392298, 0xc00257b6e0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0000be240, {0x23922d0, 0xc002315530}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002315530}, 0xc002303590, 0x1bd58c0, 0x23035a0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0023154d0, 0xc002301680)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc002315230}, 0xc002301680)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002315230}, {0x1f353a0, 0xc002301680})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002315230}, {0x1f353a0, 0xc002301680}, 0x237bc38, 0xc00230c918)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315230}, {0x1f353a0, 0xc002301680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0023150b0, {0x23922d0, 0xc0023150e0}, {0x1f353a0, 0xc002301680}, 0xc0023143f0, 0xc0023016c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023150e0}, {0x1f353a0, 0xc002301680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023150e0}, {0x1f353a0, 0xc002301680}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023150e0}, {0x1f353a0, 0xc002301680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002315050}, {0x1f353a0, 0xc002301680}, 0x40, 0xc0023016c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315050}, {0x1f353a0, 0xc002301680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002315050}, {0x1f353a0, 0xc002301680}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315050}, {0x1f353a0, 0xc002301680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002315050}, {0x1f353a0, 0xc002301680}, 0xc001af6200, 0xc00230c918)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002315050}, 0xc00257b680, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001b39d40, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001b39d40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661084 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0009f79e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012fa8f0, {0x1f7b560, 0xc001941e00}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6400, 0xc000d09d18, 0xc000d09ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6400, {0x1f7b560, 0xc001941e00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00206dd10}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0x20, {0xc000ebb900, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00206dd10}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00206dd10}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0xc002085a20, {0xc000ebb900, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00206dd10}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0xc00119e640, 0xc00206dd40, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00206dd10}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002072e40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0x237ac70, 0xc00206dc80, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002072e40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002072e40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc002072dc0}, {0x1f7b560, 0xc001941e00}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc002072e40}, 0xc00250a720, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc00206d5c0}, 0xc002072dc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc00206d5c0}, 0xc002072dc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000d0a5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000d0a648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd82b710, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc00206d5c0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021fe170})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000d0a898, {0x235a640, 0xc000045ad0}, 0x612d313666342d36)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc00206d5c0}, 0xc00206d410)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001b82db8, 0x4c1cb4}, {0xc057f6d975b96008, 0x8aa2cbfd55}, {0xc001b82db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc00206d5c0}, 0xc00206d410)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410}, 0xc000ebb840, 0xc0020765b8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410}, 0x15, 0xc002072b80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410}, 0x207690b, 0xc002072b80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410}, 0x237bc38, 0xc002072b80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d5c0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00206d470}, {0x1f33180, 0xc00206d410}, 0x1e5a940, 0xc002072b80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d470}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410}, 0x9, 0xc002072b80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410}, 0xc000850768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000850b60, {0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00206d3e0}, {0x1f33180, 0xc00206d410}, 0xc000ebb840, 0xc0020765b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc00206d3e0}, 0xc002245b00, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0009f6900, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc0009f6900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1293 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc000be5a00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcc30)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcc30)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1263 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000a4c500, 0xc0012e6e50)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000a4c500)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1297 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000a4c500, 0xc002114620)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000a4c500)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1298 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000a4c500, 0xc001f0eac0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000a4c500)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1299 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000a554a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1657997 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00067f9e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e82d10, {0x1f7c860, 0xc001e82c60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8c300, 0xc00181f5e8, 0xc00181f5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8c300, {0x1f7c860, 0xc001e82c60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0019aa840}, {0x20768da, 0x8}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x1c75901, {0xc00230bd60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x7f00048b4108, {0xc00230bd60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x20, 0xc0023795f0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0xc000f5fa00, {0xc0024ca540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x0, 0xc0023795c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x1e0cfc0, 0xc002379530, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0019aa840}, {0x20768da, 0x203000}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0x8, 0xc002379500, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0019aa840}, {0x20768da, 0x46}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, 0xc00002ae50, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc0019aa840}, {0x20768da, 0x0}, {0x1f33180, 0xc0023794a0}, {0x1f7c860, 0xc001e82c60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0019aa840}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000a4c500, {0x2392298, 0xc0019aa840})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1477991 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001e51f40)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1477992 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000141030, {0x2392228, 0xc0024655c0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1657981 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353c00, {0x2392228, 0xc001319cc0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1661139 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0023b35c0, {0x2392228, 0xc002191600})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658105 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fda5a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e83a20, {0x1f7b560, 0xc001370c80}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40600, 0xc001dadd18, 0xc001dadce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40600, {0x1f7b560, 0xc001370c80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0013025a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0x20, {0xc000f80740, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0013025a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013025a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0xc001ec5080, {0xc000f80740, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013025a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0xc00119e640, 0xc0013025d0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013025a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e65b80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0x237ac70, 0xc001302510, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e65b80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e65b80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e65b00}, {0x1f7b560, 0xc001370c80}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc000e65b80}, 0xc000ba4720, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0020ade00}, 0xc000e65b00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0020ade00}, 0xc000e65b00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001dae5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001dae648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8c3e20, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0020ade00}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000f6e800})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001dae898, {0x235a640, 0xc000045ad0}, 0x622d616334342d37)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0020ade00}, 0xc0020adb60)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0019d6db8, 0x4c1cb4}, {0xc057f6d937a8a067, 0x8a692073b2}, {0xc0019d6db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0020ade00}, 0xc0020adb60)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60}, 0xc000f80540, 0xc0024e5de8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60}, 0x15, 0xc000e658c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60}, 0x207690b, 0xc000e658c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60}, 0x237bc38, 0xc000e658c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ade00}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0020adc80}, {0x1f33180, 0xc0020adb60}, 0x1e5a940, 0xc000e658c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020adc80}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60}, 0x9, 0xc000e658c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60}, 0xc001bf5768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bf5b60, {0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020adb30}, {0x1f33180, 0xc0020adb60}, 0xc000f80540, 0xc0024e5de8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0020adb30}, 0xc00196e600, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00077e900, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00077e900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658005 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00086fd40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022ebb80, {0x1f7c860, 0xc0022ebad0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8c600, 0xc0018f55e8, 0xc0018f55b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8c600, {0x1f7c860, 0xc0022ebad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002428420}, {0x20768da, 0x8}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x1c75901, {0xc000be4880, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x7f00048b45b8, {0xc000be4880, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x20, 0xc0012910b0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0xc000f5fa00, {0xc001f82800, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x0, 0xc001291080, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x1e0cfc0, 0xc001290ff0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002428420}, {0x20768da, 0x203000}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x8, 0xc001290fc0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002428420}, {0x20768da, 0x46}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc002428420}, {0x20768da, 0x0}, {0x1f33180, 0xc001290f60}, {0x1f7c860, 0xc0022ebad0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc002428420}, 0xc001216f20, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000a4c500, {0x2392298, 0xc002428420})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658062 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003d8850, {0x2392228, 0xc000e16900})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1769156 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 1654724 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c2c0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00243e900, 0xc001373000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00243e900, {0xc001373000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00243e900, {0xc001373000, 0x1, 0xc001b0efd8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0020cafe0, {0xc001373000, 0x448e71, 0x7effdd897898})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc0005f0000, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc0005f0000, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc0005f0000)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlConn).readResultSetHeaderPacket(0xc0023e5ad0)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:537 +0x25
github.com/go-sql-driver/mysql.(*mysqlStmt).query(0xc0023e5ad0, {0xc000e96550, 0xc0012f9b60, 0xc000e9a0a0})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/statement.go:108 +0x125
github.com/go-sql-driver/mysql.(*mysqlStmt).QueryContext(0xc0023e5ad0, {0x2392298, 0xc0012f9b60}, {0xc000e9a000, 0x94, 0x208f76a})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:565 +0x91
database/sql.ctxDriverStmtQuery({0x2392298, 0xc0012f9b60}, {0x2392458, 0xc0023e5ad0}, {0xc000e9a000, 0x5, 0x5})
/usr/local/go/src/database/sql/ctxutil.go:82 +0x132
database/sql.rowsiFromStatement({0x2392298, 0xc0012f9b60}, {0x237efb0, 0xc0005f0000}, 0xc000e8d000, {0xc000dff6d0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:2794 +0x17e
database/sql.(*DB).queryDC(0x203001, {0x2392298, 0xc0012f9b60}, {0x0, 0x0}, 0xc002433d40, 0xc00127dc40, {0x208f76a, 0x94}, {0xc000dff6d0, ...})
/usr/local/go/src/database/sql/sql.go:1752 +0x3d7
database/sql.(*DB).query(0x1b0f548, {0x2392298, 0xc0012f9b60}, {0x208f76a, 0x94}, {0xc000dff6d0, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0012f9b60}, {0x208f76a, 0x94}, {0xc000dff6d0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc0012f9b60}, {0x208f76a, 0xc00004c000}, {0xc000dff6d0, 0xc001b0f6c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0012f9b60}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0023e5830}, {0x208f76a, 0xc000ee9300}, {0xc000dff6d0, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x23, {0x2392298, 0xc0012f9b60}, {0x1b5d8e0, 0xc0023e5830}, {0x208f76a, 0xc001b0f7e0}, {0xc000dff6d0, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc0012f9b60}, {0xfb32f745, {0xc0013c8840, 0x34, 0x34}, 0x0, 0xc000e8c8a8, 0xc00245f1b0, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e8c880)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00085a000)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x23)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc00250db60, 0x403030a0, 0x4031b740, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc00005e800, 0xc00211d390})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001b0fc50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc001b0fd10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001b0fde2, 0xc001125860)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001b0fd60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc00250dce0)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00250dce0, {0x2392228, 0xc0021231c0})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658087 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000353c70, {0x2392228, 0xc000e90380})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1661170 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0022916d0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1829768 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006c4380, {0x2392228, 0xc0019fe940})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1654837 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001113bc0, {0x2392298, 0xc002318c00}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00195a480, {0x23922d0, 0xc00229e3f0}, 0x2a)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001aec540, {0x23922d0, 0xc00229e3f0}, 0xc002286e50, 0x1, 0x2286e60)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc00229e390, 0xc0021f5a40)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc00229e0f0}, 0xc0021f5a40)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00229e0f0}, {0x1f35540, 0xc0021f5a40})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00229e0f0}, {0x1f35540, 0xc0021f5a40}, 0x237bc38, 0xc0021ef350)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00229e0f0}, {0x1f35540, 0xc0021f5a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc002281f80}, {0x1f35540, 0xc0021f5a40}, 0xc0021f5480, 0xc0021f5ac0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281f80}, {0x1f35540, 0xc0021f5a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002281f80}, {0x1f35540, 0xc0021f5a40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281f80}, {0x1f35540, 0xc0021f5a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002281f50}, {0x1f35540, 0xc0021f5a40}, 0x40, 0xc0021f5ac0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281f50}, {0x1f35540, 0xc0021f5a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002281f50}, {0x1f35540, 0xc0021f5a40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281f50}, {0x1f35540, 0xc0021f5a40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002281f50}, {0x1f35540, 0xc0021f5a40}, 0xc00230b1e0, 0xc0021ef350)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002281f50}, 0xc002318ba0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001bfcc60, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001bfcc60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1370 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000688700, 0xc0022bda80)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000688700)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1829714 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019c3560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116cbb0, {0x1f7c860, 0xc00116cb00}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8d900, 0xc00194d5e8, 0xc00194d5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8d900, {0x1f7c860, 0xc00116cb00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0024eeea0}, {0x20768da, 0x8}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x1c75901, {0xc0022b41a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x7f00048b45b8, {0xc0022b41a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x20, 0xc0006a3a70, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0xc000f5fa00, {0xc000efb780, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x0, 0xc0006a39e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x1e0cfc0, 0xc0006a3950, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x60, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0024eeea0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x8, 0xc0006a3920, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0024eeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, 0x144aacf, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b45b8, {0x2392298, 0xc0024eeea0}, {0x20768da, 0x0}, {0x1f33180, 0xc0006a38c0}, {0x1f7c860, 0xc00116cb00}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0024eeea0}, 0xc00080b6e0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000a4c500, {0x2392298, 0xc0024eeea0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1294 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000cfcc30)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1651755 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00195a000)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bad290, {0x1f7b7c0, 0xc001940640}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8d600, 0xc0017f7bf8, 0xc0017f7bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8d600, {0x1f7b7c0, 0xc001940640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002581410}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0x20, {0xc000be5520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002581410}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002581410}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0xc0019bc2c0, {0xc000be5520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002581410}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0xc00119e640, 0xc002581440, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002581410}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0015f3e40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0x237ac70, 0xc002581380, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0015f3e40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0015f3e40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0015f3cc0}, {0x1f7b7c0, 0xc001940640}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0015f3e40}, 0xc000ba5530, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc002580810}, 0xc0015f3cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc002580810}, 0xc0015f3cc0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017f8480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017f8528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8d2f70, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002580810}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000a4b020})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0017f8778, {0x235a640, 0xc000045ad0}, 0x622d313734342d35)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002580810}, 0xc0015f3840)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001b66db8, 0x4c1cb4}, {0xc057f6d8b88aae77, 0x89f2ccedc1}, {0xc001b66db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc002580810}, 0xc0015f3840)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840}, 0xc000be5360, 0xc0026c3938)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840}, 0x15, 0xc0015f3880)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840}, 0x2076951, 0xc0015f3880)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840}, 0x237bc38, 0xc0015f3880)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002580810}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002580600}, {0x1f33320, 0xc0015f3840}, 0x1e5a940, 0xc0015f3880)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002580600}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840}, 0x23c27b0, 0xc0015f3880)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0019a0b60, {0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0025805d0}, {0x1f33320, 0xc0015f3840}, 0xc000be5360, 0xc0026c3938)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0025805d0}, 0xc0023ed2c0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00112ea20, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00112ea20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1475042 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000f97b00, {0x2392228, 0xc001737b40})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1829742 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001236f00)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1295 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010e35f0, 0xc0012ae860)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010e35f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1296 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010e35f0, 0xc001474110)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010e35f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1329 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010e35f0, 0xc001256350)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010e35f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1330 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0010e35f0, 0xc001f867e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0010e35f0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1331 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000cfcb40)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1678748 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ae3b00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0000, {0x1f736c0, 0xc0011b3a70}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078b700, 0xc0019815c0, 0xc001981590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078b700, {0x1f736c0, 0xc0011b3a70})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00243be60}, {0x2076920, 0x8}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x1c75901, {0xc001c642e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x7f00048b4108, {0xc001c642e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x20, 0xc002314900, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0xc000bfb110, {0xc0012830c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x0, 0xc0023148d0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x0, 0xc002314840, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00243be60}, {0x2076920, 0x203000}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x203000, 0xc002314810, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00243be60}, {0x2076920, 0x46}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002849e50, {0x2392298, 0xc00243be60}, {0x2076920, 0x2011458b8}, {0x1f33320, 0xc001283080}, {0x1f736c0, 0xc0011b3a70}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc00243be60}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010e35f0, {0x2392298, 0xc00243be60})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1371 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000688700, 0xc001f1d6b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000688700)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1337 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0019c0370)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1338 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000fdf10, {0x2392228, 0xc0019a7880})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1339 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0019ab3e0, {0x2392228, 0xc0019a78c0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1340 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0019ab3e0, {0x2392228, 0xc0019a78c0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658038 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1e6c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac580, {0x1f736c0, 0xc0009e5d40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x69)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8ca00, 0xc0019b15c0, 0xc0019b1590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8ca00, {0x1f736c0, 0xc0009e5d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002429320}, {0x2076920, 0x8}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x1c75901, {0xc000be5020, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x7f00048b45b8, {0xc000be5020, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x20, 0xc0023a41e0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0xc000bfb110, {0xc0012e0280, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x0, 0xc0023a4150, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x1e0cfc0, 0xc0023a40c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x1e0cfc0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002429320}, {0x2076920, 0x203000}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x203000, 0xc0023a4090, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002429320}, {0x2076920, 0x46}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001216e50, {0x2392298, 0xc002429320}, {0x2076920, 0x100000000}, {0x1f33320, 0xc0012e0240}, {0x1f736c0, 0xc0009e5d40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc002429320}, 0xc001216f68, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010e35f0, {0x2392298, 0xc002429320})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1654762 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0012dd3e0, {0x2392228, 0xc002202d80})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658128 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001bfd7a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116c9a0, {0x1f7b7c0, 0xc0007c1ae0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40d00, 0xc000ee04d0, 0xc000ee04a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40d00, {0x1f7b7c0, 0xc0007c1ae0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00214d3e0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0x20, {0xc000e80660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00214d3e0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00214d3e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0xc001bfb340, {0xc000e80660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00214d3e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0xc000c61270, 0xc00214d410, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00214d3e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0010dcbc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0x237ac70, 0xc00214d350, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0010dcbc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0010dcbc0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0010dcb00}, {0x1f7b7c0, 0xc0007c1ae0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc0010dcbc0}, 0xc001ee3056, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc00263ea20}, 0xc0010dcb00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc00263ea20}, 0xc0010dcb00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0012f8d80, {0x2392298, 0xc00263ea20})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0012f8fc0, {0x2392298, 0xc00263ea20}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00066cd80, {0x23922d0, 0xc00214d110}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0024766c0, {0x23922d0, 0xc00214d110}, 0xc001eb56b0, 0x1, 0x1eb56c0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc00214d0b0, 0xc0010dc8c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc00214cd20}, 0xc0010dc8c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00214cd20}, {0x1f35540, 0xc0010dc8c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00214cd20}, {0x1f35540, 0xc0010dc8c0}, 0x237bc38, 0xc0023f6630)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214cd20}, {0x1f35540, 0xc0010dc8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00214cbd0}, {0x1f35540, 0xc0010dc8c0}, 0xc0010dc300, 0xc0010dc940)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214cbd0}, {0x1f35540, 0xc0010dc8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00214cbd0}, {0x1f35540, 0xc0010dc8c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214cbd0}, {0x1f35540, 0xc0010dc8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00214cba0}, {0x1f35540, 0xc0010dc8c0}, 0x40, 0xc0010dc940)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214cba0}, {0x1f35540, 0xc0010dc8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00214cba0}, {0x1f35540, 0xc0010dc8c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214cba0}, {0x1f35540, 0xc0010dc8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00214cba0}, {0x1f35540, 0xc0010dc8c0}, 0xc000e805a0, 0xc0023f6630)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00214cba0}, 0xc00263e960, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001bfc900, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001bfc900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658039 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1fe60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac630, {0x1f7b7c0, 0xc0019405a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xbf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8cd00, 0xc001c23bf8, 0xc001c23bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8cd00, {0x1f7b7c0, 0xc0019405a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0023a52f0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0x20, {0xc000be5460, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0023a52f0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023a52f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0xc0010c6b00, {0xc000be5460, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023a52f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0xc00119e640, 0xc0023a5350, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023a52f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012e0800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0x237ac70, 0xc0023a5230, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012e0800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012e0800}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012e0780}, {0x1f7b7c0, 0xc0019405a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012e0800}, 0xc00111cf00, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0023a4840}, 0xc0012e0780, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0023a4840}, 0xc0012e0780, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001c24480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c24528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8214d8, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0023a4840}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10101c000a4b220})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c24778, {0x235a640, 0xc000045ad0}, 0x382d636163342d39)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0023a4840}, 0xc0012e0480)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0018f0db8, 0x4c1cb4}, {0xc057f6d936f3f149, 0x8a686bc4a0}, {0xc0018f0db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0023a4840}, 0xc0012e0480)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480}, 0xc000be5220, 0xc002080720)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480}, 0x15, 0xc0012e04c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480}, 0x2076951, 0xc0012e04c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480}, 0x237bc38, 0xc0012e04c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a4840}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0023a46f0}, {0x1f33320, 0xc0012e0480}, 0x1e5a940, 0xc0012e04c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a46f0}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480}, 0x23c27b0, 0xc0012e04c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002888b60, {0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023a46c0}, {0x1f33320, 0xc0012e0480}, 0xc000be5220, 0xc002080720)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0023a46c0}, 0xc002429560, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c1e7e0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001c1e7e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661167 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005fe000)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2fd0, {0x1f7b560, 0xc001f4cfa0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41300, 0xc001966440, 0xc001966410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41300, {0x1f7b560, 0xc001f4cfa0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022b16b0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0x20, {0xc001c65720, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022b16b0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b16b0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0xc001b226e0, {0xc001c65720, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022b16b0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0xc000c61270, 0xc0022b16e0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b16b0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0022e2340}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0x237ac70, 0xc0022b1620, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0022e2340}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0022e2340}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0022e22c0}, {0x1f7b560, 0xc001f4cfa0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e8c0, {0x2392228, 0xc0022e2340}, 0xc0026b6f96, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0007de270, {0x2392298, 0xc0025ead80}, 0xc0022e22c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007c2160, {0x2392298, 0xc0025ead80}, 0xc0022e22c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002633860, {0x2392298, 0xc0025ead80})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002633e00, {0x2392298, 0xc0025ead80}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001922120, {0x23922d0, 0xc0022b1320}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0022b1320}, 0xc0022bc220, 0x1bd58c0, 0x22bc230)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0022b12c0, 0xc0022e20c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0022b1020}, 0xc0022e20c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022b1020}, {0x1f353a0, 0xc0022e20c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0022b1020}, {0x1f353a0, 0xc0022e20c0}, 0x237bc38, 0xc00229c828)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b1020}, {0x1f353a0, 0xc0022e20c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0022b0ea0, {0x23922d0, 0xc0022b0ed0}, {0x1f353a0, 0xc0022e20c0}, 0xc0022b0090, 0xc0022e2100)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0ed0}, {0x1f353a0, 0xc0022e20c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022b0ed0}, {0x1f353a0, 0xc0022e20c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0ed0}, {0x1f353a0, 0xc0022e20c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022b0e40}, {0x1f353a0, 0xc0022e20c0}, 0x40, 0xc0022e2100)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0e40}, {0x1f353a0, 0xc0022e20c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022b0e40}, {0x1f353a0, 0xc0022e20c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0e40}, {0x1f353a0, 0xc0022e20c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022b0e40}, {0x1f353a0, 0xc0022e20c0}, 0xc001c65680, 0xc00229c828)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0022b0e40}, 0xc0025ead20, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000ff8d80, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000ff8d80, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678760 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00008e900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b81b80, {0x1f736c0, 0xc00124f560}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446400, 0xc0019db5c0, 0xc0019db590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446400, {0x1f736c0, 0xc00124f560})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001aeb920}, {0x2076920, 0x8}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x1c75901, {0xc000ebb6e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x7f00048b45b8, {0xc000ebb6e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x20, 0xc0021c6e10, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0xc000bfb110, {0xc001251a40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x0, 0xc0021c6de0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x0, 0xc0021c6d50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001aeb920}, {0x2076920, 0x203000}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x203000, 0xc0021c6d20, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001aeb920}, {0x2076920, 0x46}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002848e50, {0x2392298, 0xc001aeb920}, {0x2076920, 0x200000000}, {0x1f33320, 0xc001251a00}, {0x1f736c0, 0xc00124f560}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc001aeb920}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010e35f0, {0x2392298, 0xc001aeb920})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658007 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001906fc0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022ebe40, {0x1f7b560, 0xc001de3cc0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8c700, 0xc000733d18, 0xc000733ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8c700, {0x1f7b560, 0xc001de3cc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001291ec0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0x20, {0xc000be4b00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001291ec0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001291ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0xc000786580, {0xc000be4b00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001291ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0xc00119e640, 0xc001291ef0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001291ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001f82cc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0x237ac70, 0xc001291e30, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001f82cc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001f82cc0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001f82c40}, {0x1f7b560, 0xc001de3cc0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc001f82cc0}, 0xc000cfab88, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc001291620}, 0xc001f82c40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc001291620}, 0xc001f82c40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0007345a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000734648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd9cbc08, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc001291620}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000a4a900})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000734898, {0x235a640, 0xc000045ad0}, 0x392d386464342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc001291620}, 0xc001291470)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc00181adb8, 0x4c1cb4}, {0xc057f6d9368c0a7e, 0x8a6803ddcb}, {0xc00181adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc001291620}, 0xc001291470)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470}, 0xc000be4a40, 0xc002425470)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470}, 0x15, 0xc001f82a00)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470}, 0x207690b, 0xc001f82a00)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470}, 0x237bc38, 0xc001f82a00)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291620}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0012914d0}, {0x1f33180, 0xc001291470}, 0x1e5a940, 0xc001f82a00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0012914d0}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470}, 0x9, 0xc001f82a00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470}, 0xc002888768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002888b60, {0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001291440}, {0x1f33180, 0xc001291470}, 0xc000be4a40, 0xc002425470)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc001291440}, 0xc0024286c0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00086fe60, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00086fe60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1369 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000688700, 0xc0021159c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000688700)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1372 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000688700, 0xc001475010)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000688700)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1373 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000cfcd20)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1661163 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00065a900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2dc0, {0x1f7c860, 0xc0007d2d10}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41100, 0xc0019555e8, 0xc0019555b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41100, {0x1f7c860, 0xc0007d2d10})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x8}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x1c75901, {0xc001c653c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x7f00048b4108, {0xc001c653c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x20, 0xc00213dcb0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0xc000f5fa00, {0xc0021218c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x0, 0xc00213dc80, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x1e0cfc0, 0xc00213dbf0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x8, 0xc00213dbc0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0025ea3c0}, {0x20768da, 0x46}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, 0x1cab380, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f00048b4108, {0x2392298, 0xc0025ea3c0}, {0x20768da, 0x0}, {0x1f33180, 0xc00213db60}, {0x1f7c860, 0xc0007d2d10}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e3c0, {0x2392298, 0xc0025ea3c0}, 0x10, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000688700, {0x2392298, 0xc0025ea3c0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658057 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bbf560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e83340, {0x1f736c0, 0xc000967560}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x69)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001b8d300, 0xc0019575c0, 0xc001957590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001b8d300, {0x1f736c0, 0xc000967560})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x8}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x1c75901, {0xc001bc4e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x7f00048b4108, {0xc001bc4e60, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x20, 0xc001f78bd0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0xc000bfb110, {0xc001367840, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x0, 0xc001f78ba0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x1e0cfc0, 0xc001f78b10, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x2, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x203000, 0xc001f78ae0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001e8d6e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000a6ce50, {0x2392298, 0xc001e8d6e0}, {0x2076920, 0x100000000}, {0x1f33320, 0xc001367800}, {0x1f736c0, 0xc000967560}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc001e8d6e0}, 0xa6cea0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0010e35f0, {0x2392298, 0xc001e8d6e0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1661136 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023b3740, {0x2392298, 0xc0023b37a0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001c68480, {0x23922d0, 0xc002197da0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002197da0}, 0xc0021b2040, 0x1bd58c0, 0x21b2050)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc002197b90, 0xc002190e40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc0021978f0}, 0xc002190e40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0021978f0}, {0x1f353a0, 0xc002190e40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0021978f0}, {0x1f353a0, 0xc002190e40}, 0x237bc38, 0xc002077b18)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021978f0}, {0x1f353a0, 0xc002190e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021977a0}, {0x1f353a0, 0xc002190e40}, 0x203000, 0xc002190e80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021977a0}, {0x1f353a0, 0xc002190e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021977a0}, {0x1f353a0, 0xc002190e40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021977a0}, {0x1f353a0, 0xc002190e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002197740}, {0x1f353a0, 0xc002190e40}, 0x40, 0xc002190e80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002197740}, {0x1f353a0, 0xc002190e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002197740}, {0x1f353a0, 0xc002190e40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002197740}, {0x1f353a0, 0xc002190e40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002197740}, {0x1f353a0, 0xc002190e40}, 0xc000ebbf20, 0xc002077b18)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002197740}, 0xc0023b33e0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001349320, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001349320, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657982 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002571020, {0x2392228, 0xc001319d00})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1678682 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001112540, {0x2392298, 0xc000f97920}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005bb8c0, {0x23922d0, 0xc002196b70}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0024761e0, {0x23922d0, 0xc002196b70}, 0xc001208590, 0x1, 0x12085a0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc002196ab0, 0xc0012013c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc002196720}, 0xc0012013c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002196720}, {0x1f35540, 0xc0012013c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002196720}, {0x1f35540, 0xc0012013c0}, 0x237bc38, 0xc0023c18c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002196720}, {0x1f35540, 0xc0012013c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0021965a0}, {0x1f35540, 0xc0012013c0}, 0xc001200e40, 0xc001201440)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021965a0}, {0x1f35540, 0xc0012013c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021965a0}, {0x1f35540, 0xc0012013c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021965a0}, {0x1f35540, 0xc0012013c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002196570}, {0x1f35540, 0xc0012013c0}, 0x40, 0xc001201440)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002196570}, {0x1f35540, 0xc0012013c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002196570}, {0x1f35540, 0xc0012013c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002196570}, {0x1f35540, 0xc0012013c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002196570}, {0x1f35540, 0xc0012013c0}, 0xc000ebaa00, 0xc0023c18c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002196570}, 0xc000f977a0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019fcea0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc0019fcea0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1681109 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001112540, {0x2392298, 0xc002248120}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005bb8c0, {0x23922d0, 0xc0006a5920}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001f15590, {0x23922d0, 0xc0006a5920}, 0xc002457c30, 0x1, 0x2457c40)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc0006a58c0, 0xc002082340)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc0006a5620}, 0xc002082340)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0006a5620}, {0x1f35540, 0xc002082340})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc0006a5620}, {0x1f35540, 0xc002082340}, 0x237bc38, 0xc001b9c258)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a5620}, {0x1f35540, 0xc002082340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0006a54d0}, {0x1f35540, 0xc002082340}, 0xc001fa5b80, 0xc002082400)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a54d0}, {0x1f35540, 0xc002082340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0006a54d0}, {0x1f35540, 0xc002082340}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a54d0}, {0x1f35540, 0xc002082340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0006a54a0}, {0x1f35540, 0xc002082340}, 0x40, 0xc002082400)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a54a0}, {0x1f35540, 0xc002082340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0006a54a0}, {0x1f35540, 0xc002082340}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006a54a0}, {0x1f35540, 0xc002082340})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0006a54a0}, {0x1f35540, 0xc002082340}, 0xc000f807a0, 0xc001b9c258)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0006a54a0}, 0xc002248000, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c68900, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c68900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658106 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001f8f620, {0x2392298, 0xc00196ec00}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d93320, {0x23922d0, 0xc001302de0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001302de0}, 0xc001f1cad0, 0x1bd58c0, 0x1f1cae0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001302d80, 0xc000e65dc0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001302ae0}, 0xc000e65dc0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001302ae0}, {0x1f353a0, 0xc000e65dc0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001302ae0}, {0x1f353a0, 0xc000e65dc0}, 0x237bc38, 0xc001f16210)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001302ae0}, {0x1f353a0, 0xc000e65dc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001302960, {0x23922d0, 0xc001302990}, {0x1f353a0, 0xc000e65dc0}, 0xc0020adb60, 0xc000e65e00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001302990}, {0x1f353a0, 0xc000e65dc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001302990}, {0x1f353a0, 0xc000e65dc0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001302990}, {0x1f353a0, 0xc000e65dc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001302900}, {0x1f353a0, 0xc000e65dc0}, 0x40, 0xc000e65e00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001302900}, {0x1f353a0, 0xc000e65dc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001302900}, {0x1f353a0, 0xc000e65dc0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001302900}, {0x1f353a0, 0xc000e65dc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001302900}, {0x1f353a0, 0xc000e65dc0}, 0xc000f80860, 0xc001f16210)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001302900}, 0xc00196eba0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000fda6c0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000fda6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658060 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a57560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001e834a0, {0x1f7b7c0, 0xc001fbd5e0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c40100, 0xc00171c4d0, 0xc00171c4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c40100, {0x1f7b7c0, 0xc001fbd5e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022cef60}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0x20, {0xc001bc5280, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022cef60}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022cef60}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0x7effdd8405f0, {0xc001bc5280, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022cef60}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0xc000c61270, 0xc0022cef90, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022cef60}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0x8, {0xc0005fb7d0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e16ac0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0x237ac70, 0xc0022ceed0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e16ac0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e16ac0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000e16a00}, {0x1f7b7c0, 0xc001fbd5e0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e8c0, {0x2392228, 0xc000e16ac0}, 0xc00204da16, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0007de270, {0x2392298, 0xc0026420c0}, 0xc000e16a00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007c2160, {0x2392298, 0xc0026420c0}, 0xc000e16a00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001e8de00, {0x2392298, 0xc0026420c0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002642060, {0x2392298, 0xc0026420c0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001c35e60, {0x23922d0, 0xc0022ce060}, 0x1e)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc00204d980, {0x23922d0, 0xc0022ce060}, 0xc001f86f10, 0x1, 0x1f86f20)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc001f79fb0, 0xc000e16040)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc001f79d10}, 0xc000e16040)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001f79d10}, {0x1f35540, 0xc000e16040})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001f79d10}, {0x1f35540, 0xc000e16040}, 0x237bc38, 0xc002289878)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f79d10}, {0x1f35540, 0xc000e16040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001f79bc0}, {0x1f35540, 0xc000e16040}, 0xc001367a40, 0xc000e160c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f79bc0}, {0x1f35540, 0xc000e16040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f79bc0}, {0x1f35540, 0xc000e16040}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f79bc0}, {0x1f35540, 0xc000e16040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f79b90}, {0x1f35540, 0xc000e16040}, 0x40, 0xc000e160c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f79b90}, {0x1f35540, 0xc000e16040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f79b90}, {0x1f35540, 0xc000e16040}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f79b90}, {0x1f35540, 0xc000e16040})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f79b90}, {0x1f35540, 0xc000e16040}, 0xc001bc5160, 0xc002289878)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001f79b90}, 0xc001e8db60, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c34d80, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc001c34d80, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1661165 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ff8b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0007d2f20, {0x1f7b560, 0xc001f4ce60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000c41200, 0xc00192bd18, 0xc00192bce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c41200, {0x1f7b560, 0xc001f4ce60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022b0990}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0x20, {0xc001c65580, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022b0990}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b0990}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0xc001b222c0, {0xc001c65580, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022b0990}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0xc00119e640, 0xc0022b09c0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022b0990}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002121d80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0x237ac70, 0xc0022b0900, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002121d80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002121d80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc002121d00}, {0x1f7b560, 0xc001f4ce60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc002121d80}, 0xc0000c3d40, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022b0240}, 0xc002121d00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022b0240}, 0xc002121d00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00192c5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00192c648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8d3ee8, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0022b0240}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0022071a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00192c898, {0x235a640, 0xc000045ad0}, 0x622d313461342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022b0240}, 0xc0022b0090)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc001728db8, 0x4c1cb4}, {0xc057f6d976c4856a, 0x8aa3d722b8}, {0xc001728db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0022b0240}, 0xc0022b0090)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090}, 0xc001c654c0, 0xc00229c390)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090}, 0x15, 0xc002121ac0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090}, 0x207690b, 0xc002121ac0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090}, 0x237bc38, 0xc002121ac0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0240}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0022b00f0}, {0x1f33180, 0xc0022b0090}, 0x1e5a940, 0xc002121ac0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b00f0}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090}, 0x9, 0xc002121ac0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090}, 0xc001d5e768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001d5eb60, {0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022b0060}, {0x1f33180, 0xc0022b0090}, 0xc001c654c0, 0xc00229c390)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0022b0060}, 0xc0025ea840, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00065ab40, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc00065ab40, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657945 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00112ed80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022eb550, {0x1f7b560, 0xc001de3220}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078bb00, 0xc001c27d18, 0xc001c27ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078bb00, {0x1f7b560, 0xc001de3220})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00240e870}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0x20, {0xc0007c3460, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00240e870}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00240e870}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0xc0020af760, {0xc0007c3460, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00240e870}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0xc00119e640, 0xc00240e8a0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00240e870}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013c7740}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0x237ac70, 0xc00240e7e0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013c7740}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013c7740}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0013c7680}, {0x1f7b560, 0xc001de3220}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0013c7740}, 0xc001c20a80, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc002581da0}, 0xc0013c7680, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc002581da0}, 0xc0013c7680, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001c285a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c28648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd9614f8, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc002581da0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0020a18a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001c28898, {0x235a640, 0xc000045ad0}, 0x612d613235342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002581da0}, 0xc0025819e0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000848db8, 0x4c1cb4}, {0xc057f6d9359e5025, 0x8a6716237c}, {0xc000848db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc002581da0}, 0xc0025819e0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0}, 0xc0007c29c0, 0xc002004f18)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0}, 0x15, 0xc0013c7440)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0}, 0x207690b, 0xc0013c7440)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0}, 0x237bc38, 0xc0013c7440)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581da0}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002581a70}, {0x1f33180, 0xc0025819e0}, 0x1e5a940, 0xc0013c7440)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581a70}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0}, 0x1bd58c0, 0xc0013c7440)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0}, 0xc002888768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002888b60, {0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0}, 0x40, 0xc00005e400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002581980}, {0x1f33180, 0xc0025819e0}, 0xc0007c29c0, 0xc002004f18)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc002581980}, 0xc0023ec0c0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001119680, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001119680, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1657952 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023aafc0, {0x2392298, 0xc0023edc20}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001127200, {0x23922d0, 0xc001fd3230}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fd3230}, 0xc00133a320, 0x1bd58c0, 0x133a330)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc001fd2f90, 0xc0013187c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc001fd2bd0}, 0xc0013187c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001fd2bd0}, {0x1f353a0, 0xc0013187c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc001fd2bd0}, {0x1f353a0, 0xc0013187c0}, 0x237bc38, 0xc002005950)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd2bd0}, {0x1f353a0, 0xc0013187c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001fd2900}, {0x1f353a0, 0xc0013187c0}, 0x203000, 0xc001318800)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd2900}, {0x1f353a0, 0xc0013187c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd2900}, {0x1f353a0, 0xc0013187c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd2900}, {0x1f353a0, 0xc0013187c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001fd28a0}, {0x1f353a0, 0xc0013187c0}, 0x40, 0xc001318800)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd28a0}, {0x1f353a0, 0xc0013187c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001fd28a0}, {0x1f353a0, 0xc0013187c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd28a0}, {0x1f353a0, 0xc0013187c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001fd28a0}, {0x1f353a0, 0xc0013187c0}, 0xc0007c3a20, 0xc002005950)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc001fd28a0}, 0xc0023edbc0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc000bc86c0, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc000bc86c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1677979 [IO wait]:
internal/poll.runtime_pollWait(0x7effdd95c830, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ca9a80, 0xc000d3a000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ca9a80, {0xc000d3a000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ca9a80, {0xc000d3a000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00278eb90, {0xc000d3a000, 0xff90, 0x3e300025ce8})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c0c900, {0xc0006ba4a0, 0x9, 0xc00005e800})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c0c900}, {0xc0006ba4a0, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0006ba4a0, 0x9, 0xc0012e56b0}, {0x23585c0, 0xc001c0c900})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ba460)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001259800, 0xc000025f98, 0x1418c25)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0007a41c0, {0x23d1088, 0xc001259800})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 1661173 [select]:
database/sql.(*DB).conn(0xc0006de8f0, {0x2392298, 0xc000ba2060}, 0x1)
/usr/local/go/src/database/sql/sql.go:1312 +0x413
database/sql.(*DB).query(0x1729548, {0x2392298, 0xc000ba2060}, {0x208f76a, 0x94}, {0xc000e964b0, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1695 +0x5d
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc000ba2060}, {0x208f76a, 0x94}, {0xc000e964b0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc000a218f0, {0x2392298, 0xc000ba2060}, {0x208f76a, 0xc00004c000}, {0xc000e964b0, 0xc0017296c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc000ba2060}, {0x2380120, 0xc000a218f0}, {0x1b5d8e0, 0xc0023e5bd8}, {0x208f76a, 0x0}, {0xc000e964b0, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x6, {0x2392298, 0xc000ba2060}, {0x1b5d8e0, 0xc0023e5bd8}, {0x208f76a, 0xc0017297e0}, {0xc000e964b0, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc000569680, {0x2392298, 0xc000ba2060}, {0xb884e2b6, {0xc000ee9470, 0x2f, 0x2f}, 0x0, 0xc000e8cfa8, 0xc00245f568, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0006f76e0, 0xc000e8cf80)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000085f80, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0006f7710, 0xc00005e400)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0006f7740, 0x1e)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0025eb2c0, 0x3e541da0, 0x3e55a440, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc000716400, 0xc0022bcec0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001729c50, {0x235a640, 0xc0000a9f50}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0000a9f50}, 0xc001729d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001729de2, 0xc001af11e0)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001729d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc0025eb5c0)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0025eb5c0, {0x2392228, 0xc0022e2e00})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1829722 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0024ef680, {0x2392228, 0xc0019708c0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1477994 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001112840, {0x2392228, 0xc002465600})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1658086 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0025e3770)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1657990 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023aafc0, {0x2392298, 0xc001fddf20}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001127200, {0x23922d0, 0xc002378540}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002378540}, 0xc0012e64b0, 0x1bd58c0, 0x12e64c0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc0006ba2a0, 0xc0023784e0, 0xc0024ff8c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc00074baa0, {0x23922d0, 0xc002378240}, 0xc0024ff8c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002378240}, {0x1f353a0, 0xc0024ff8c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002378240}, {0x1f353a0, 0xc0024ff8c0}, 0x237bc38, 0xc0019ba2d0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002378240}, {0x1f353a0, 0xc0024ff8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0023780c0, {0x23922d0, 0xc0023780f0}, {0x1f353a0, 0xc0024ff8c0}, 0xc001f12f60, 0xc0024ff900)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023780f0}, {0x1f353a0, 0xc0024ff8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023780f0}, {0x1f353a0, 0xc0024ff8c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023780f0}, {0x1f353a0, 0xc0024ff8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002378060}, {0x1f353a0, 0xc0024ff8c0}, 0x40, 0xc0024ff900)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002378060}, {0x1f353a0, 0xc0024ff8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002378060}, {0x1f353a0, 0xc0024ff8c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002378060}, {0x1f353a0, 0xc0024ff8c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002378060}, {0x1f353a0, 0xc0024ff8c0}, 0xc00230ba00, 0xc0019ba2d0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc002378060}, 0xc001fddec0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000bcbe60, 0xc001339560, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc000bcbe60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658084 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fe8b40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac8f0, {0x1f7b560, 0xc001940b40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001d8d200, 0xc0007bdd18, 0xc0007bdce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001d8d200, {0x1f7b560, 0xc001940b40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002210240}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0x20, {0xc000be5f80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002210240}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002210240}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0xc00105ab00, {0xc000be5f80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002210240}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0xc00119e640, 0xc002210270, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002210240}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012e19c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0x237ac70, 0xc0022101b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012e19c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012e19c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0012e1940}, {0x1f7b560, 0xc001940b40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012e19c0}, 0xc00111d080, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022a9aa0}, 0xc0012e1940, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022a9aa0}, 0xc0012e1940, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0007be5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0007be648, {0x235a640, 0xc000a5a780}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8214d8, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001441920, {0x23922d0, 0xc0022a9aa0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000a4b830})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0007be898, {0x235a640, 0xc000045ad0}, 0x622d643637342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022a9aa0}, 0xc0022a97d0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0018f0db8, 0x4c1cb4}, {0xc057f6d9376e6a34, 0x8a68e63d7e}, {0xc0018f0db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000087140, {0x23922d0, 0xc0022a9aa0}, 0xc0022a97d0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0}, 0xc000be5ec0, 0xc002081968)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0}, 0x15, 0xc0012e1700)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0}, 0x207690b, 0xc0012e1700)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0}, 0x237bc38, 0xc0012e1700)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9aa0}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0022a9860}, {0x1f33180, 0xc0022a97d0}, 0x1e5a940, 0xc0012e1700)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9860}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0}, 0x9, 0xc0012e1700)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0}, 0xc002888768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002888b60, {0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022a9710}, {0x1f33180, 0xc0022a97d0}, 0xc000be5ec0, 0xc002081968)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0022a9710}, 0xc002548060, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001af8fc0, 0xc0008fb9b0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001af8fc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1475041 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000141650, {0x2392228, 0xc001737b00})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1678750 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bba120)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae00b0, {0x1f7b7c0, 0xc001f4d400}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078b800, 0xc0009b1bf8, 0xc0009b1bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078b800, {0x1f7b7c0, 0xc001f4d400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002315a10}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0x20, {0xc001c645a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002315a10}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002315a10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0xc0020ae420, {0xc001c645a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002315a10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0xc00119e640, 0xc002315a40, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002315a10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001283640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0x237ac70, 0xc002315980, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001283640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001283640}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0012835c0}, {0x1f7b7c0, 0xc001f4d400}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc001283640}, 0xc002201180, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc002315170}, 0xc0012835c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc002315170}, 0xc0012835c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0009b2480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0009b2528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd898998, {0x235a640, 0xc000a5a780}, 0x7f00048b4108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002315170}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00119e340})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0009b2778, {0x235a640, 0xc000045ad0}, 0x612d623633342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc002315170}, 0xc0012832c0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000b60db8, 0x4c1cb4}, {0xc057f6daf64515d4, 0x8c08f86f1e}, {0xc000b60db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc002315170}, 0xc0012832c0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0}, 0xc001c644a0, 0xc00237c180)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0}, 0x15, 0xc001283300)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0}, 0x2076951, 0xc001283300)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0}, 0x237bc38, 0xc001283300)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002315170}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002314e70}, {0x1f33320, 0xc0012832c0}, 0x1e5a940, 0xc001283300)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002314e70}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0}, 0x23c27b0, 0xc001283300)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002849b60, {0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002314d50}, {0x1f33320, 0xc0012832c0}, 0xc001c644a0, 0xc00237c180)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc002314d50}, 0xc0020a6120, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001ae3c20, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc001ae3c20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1658061 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0025c60a0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1678782 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bc50e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0210, {0x1f736c0, 0xc0012c0b40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078ba00, 0xc001b275c0, 0xc001b27590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078ba00, {0x1f736c0, 0xc0012c0b40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0020a70e0}, {0x2076920, 0x8}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x1c75901, {0xc001c64b80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x7f00048b4108, {0xc001c64b80, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x20, 0xc002444de0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0xc000bfb110, {0xc0012cce40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x0, 0xc002444db0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x0, 0xc002444d20, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0020a70e0}, {0x2076920, 0x203000}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x203000, 0xc002444cf0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0020a70e0}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002849e50, {0x2392298, 0xc0020a70e0}, {0x2076920, 0x200000000}, {0x1f33320, 0xc0012cce00}, {0x1f736c0, 0xc0012c0b40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc0020a70e0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001951a00, {0x2392298, 0xc0020a70e0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1417 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc001142fe0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfd1d0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfd1d0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1418 [select, 10 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000cfd1d0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1419 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001951a00, 0xc0012d2240)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001951a00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1420 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001951a00, 0xc002286730)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001951a00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1421 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001951a00, 0xc0012d2e60)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001951a00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1422 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001951a00, 0xc0013d0cf0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001951a00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1423 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc000cfd0e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 1403 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000edd100, 0xc002236c90)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000edd100)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000cfd2c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000cfd2c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1678710 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c34360)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b809a0, {0x1f7b7c0, 0xc001b15400}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7e00, 0xc001d6fbf8, 0xc001d6fbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7e00, {0x1f7b7c0, 0xc001b15400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0020ff3b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0x20, {0xc000ebad20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0020ff3b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020ff3b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0xc001c571e0, {0xc000ebad20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020ff3b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0xc00119e640, 0xc0020ff3e0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020ff3b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0012405c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0x237ac70, 0xc0020ff320, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0012405c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0012405c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001240540}, {0x1f7b7c0, 0xc001b15400}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0012405c0}, 0xc0021a7d40, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0020fecc0}, 0xc001240540, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0020fecc0}, 0xc001240540, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001d70480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001d70528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd818110, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0020fecc0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0014300e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001d70778, {0x235a640, 0xc000045ad0}, 0x612d373564342d37)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0020fecc0}, 0xc001240240)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000fd4db8, 0x4c1cb4}, {0xc057f6daf58c1730, 0x8c083f707c}, {0xc000fd4db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0020fecc0}, 0xc001240240)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240}, 0xc000ebac40, 0xc0023fc6c0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240}, 0x15, 0xc001240280)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240}, 0x2076951, 0xc001240280)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240}, 0x237bc38, 0xc001240280)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fecc0}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0020fea80}, {0x1f33320, 0xc001240240}, 0x1e5a940, 0xc001240280)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fea80}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240}, 0x23c27b0, 0xc001240280)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240}, 0x9, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002848b60, {0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fea50}, {0x1f33320, 0xc001240240}, 0xc000ebac40, 0xc0023fc6c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0020fea50}, 0xc001f50ba0, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbe900, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bbe900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1430 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001b749b0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1678918 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:627 +0x152
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 1431 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003663f0, {0x2392228, 0xc001b7ac80})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1432 [select, 10 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001b5ff20, {0x2392228, 0xc001b7acc0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1433 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001b5ff20, {0x2392228, 0xc001b7acc0})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1654834 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fdb7a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80420, {0x1f736c0, 0xc0021f8d80}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x75)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001390b00, 0xc001b635c0, 0xc001b63590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001390b00, {0x1f736c0, 0xc0021f8d80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002318720}, {0x2076920, 0x8}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x1c75901, {0xc00230aea0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x7f00048b45b8, {0xc00230aea0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x20, 0xc002280f90, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0xc000bfb110, {0xc0021f5280, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x0, 0xc002280f60, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x1e0cfc0, 0xc002280ed0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0xc0d3f1fdc7, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002318720}, {0x2076920, 0x203000}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x203000, 0xc002280ea0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002318720}, {0x2076920, 0x46}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, 0x7f00048b45b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0011b9e50, {0x2392298, 0xc002318720}, {0x2076920, 0x100000000}, {0x1f33320, 0xc0021f5240}, {0x1f736c0, 0xc0021f8d80}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc002318720}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001951a00, {0x2392298, 0xc002318720})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1654717 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001af8360)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00116de40, {0x1f7b7c0, 0xc001fbd680}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001446a00, 0xc001913bf8, 0xc001913bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001446a00, {0x1f7b7c0, 0xc001fbd680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021c5440}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0x20, {0xc000085bc0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021c5440}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021c5440}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0xc000787b80, {0xc000085bc0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021c5440}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0xc00119e640, 0xc0021c5470, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021c5440}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021c2b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0x237ac70, 0xc0021c53b0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021c2b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021c2b00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0021c2a80}, {0x1f7b7c0, 0xc001fbd680}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0021c2b00}, 0xc001e16e10, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0021c4cf0}, 0xc0021c2a80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0021c4cf0}, 0xc0021c2a80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001914480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001914528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd817ab0, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0021c4cf0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021b48f0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001914778, {0x235a640, 0xc000045ad0}, 0x622d383831342d34)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0021c4cf0}, 0xc0021c2780)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc0017b0db8, 0x4c1cb4}, {0xc057f6d8f5cee765, 0x8a2babf0ae}, {0xc0017b0db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0021c4cf0}, 0xc0021c2780)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780}, 0xc0000854c0, 0xc0021bbab8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780}, 0x15, 0xc0021c27c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780}, 0x2076951, 0xc0021c27c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780}, 0x237bc38, 0xc0021c27c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4cf0}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021c4ba0}, {0x1f33320, 0xc0021c2780}, 0x1e5a940, 0xc0021c27c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4ba0}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780}, 0x23c27b0, 0xc0021c27c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001bbdb60, {0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780}, 0x40, 0xc000716400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021c4b70}, {0x1f33320, 0xc0021c2780}, 0xc0000854c0, 0xc0021bbab8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc0021c4b70}, 0xc002363500, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bba6c0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000bba6c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1654836 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001bfcb40)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001b80580, {0x1f7b7c0, 0xc001b14a00}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc3)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001390c00, 0xc0007b9bf8, 0xc0007b9bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001390c00, {0x1f7b7c0, 0xc001b14a00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002281bf0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0x20, {0xc00230b100, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002281bf0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002281bf0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0xc00105b4a0, {0xc00230b100, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002281bf0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0xc00119e640, 0xc002281c20, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002281bf0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0x8, {0xc000833de0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021f5800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0x237ac70, 0xc002281b60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021f5800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021f5800}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0021f5780}, {0x1f7b7c0, 0xc001b14a00}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000aacd8, {0x2392228, 0xc0021f5800}, 0xc001aec360, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0000220f0, {0x23922d0, 0xc0022814a0}, 0xc0021f5780, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0006c0060, {0x23922d0, 0xc0022814a0}, 0xc0021f5780, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0007ba480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0007ba528, {0x235a640, 0xc000a5a780}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7effdd8ded48, {0x235a640, 0xc000a5a780}, 0x7f00048b45b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0022814a0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021fe490})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0007ba778, {0x235a640, 0xc000045ad0}, 0x622d383666342d30)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fa80, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0000fc8c0, {0x23922d0, 0xc0022814a0}, 0xc0021f5480)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006e9cb0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000087140, {0xc000a3adb8, 0x4c1cb4}, {0xc057f6d8f7ac6355, 0x8a2d896ca5}, {0xc000a3adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000087140, {0x23922d0, 0xc0022814a0}, 0xc0021f5480)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000088a00, {0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480}, 0xc00230b020, 0xc0021eef30)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000023950, {0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480}, 0x15, 0xc0021f54c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0000237a0, {0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480}, 0x2076951, 0xc0021f54c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0002ca3f0, {0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480}, 0x237bc38, 0xc0021f54c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022814a0}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002281350}, {0x1f33320, 0xc0021f5480}, 0x1e5a940, 0xc0021f54c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281350}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480}, 0x23c27b0, 0xc0021f54c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00072ab60, {0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480}, 0x40, 0xc00085a000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002281320}, {0x1f33320, 0xc0021f5480}, 0xc00230b020, 0xc0021eef30)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000087140}, {0x23922d0, 0xc002281320}, 0xc002318960, 0xc0006c0540)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000fdb8c0, 0xc0008fb9b0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a41c0, {0x23d1088, 0xc001258600}, 0xc000fdb8c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1651756 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023ed860, {0x2392298, 0xc0023ed8c0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00195ad80, {0x23922d0, 0xc002581c20}, 0x2d)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc000ba5830, {0x23922d0, 0xc002581c20}, 0xc0012d6810, 0x1, 0x12d6820)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc002581bc0, 0xc001402100)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc002581920}, 0xc001402100)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002581920}, {0x1f35540, 0xc001402100})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc002581920}, {0x1f35540, 0xc001402100}, 0x237bc38, 0xc0026c3d28)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002581920}, {0x1f35540, 0xc001402100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0025817d0}, {0x1f35540, 0xc001402100}, 0xc0015f3840, 0xc001402180)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025817d0}, {0x1f35540, 0xc001402100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0025817d0}, {0x1f35540, 0xc001402100}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025817d0}, {0x1f35540, 0xc001402100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0025817a0}, {0x1f35540, 0xc001402100}, 0x40, 0xc001402180)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025817a0}, {0x1f35540, 0xc001402100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0025817a0}, {0x1f35540, 0xc001402100}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0025817a0}, {0x1f35540, 0xc001402100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0025817a0}, {0x1f35540, 0xc001402100}, 0xc000be57a0, 0xc0026c3d28)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc0025817a0}, 0xc0023ed500, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc00195a120, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc0006efe00}, 0xc00195a120, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1678789 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fdb320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ae0370, {0x1f736c0, 0xc0012c1320}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00078be00, 0xc001c155c0, 0xc001c15590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00078be00, {0x1f736c0, 0xc0012c1320})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0020a7980}, {0x2076920, 0x8}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x1c75901, {0xc001c65240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x7f00048b4108, {0xc001c65240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x20, 0xc00246cb40, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0xc000bfb110, {0xc0012cdc40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x0, 0xc00246cb10, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x0, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x0, 0xc00246ca80, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x40ce34, {0xc0006c13c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0020a7980}, {0x2076920, 0x203000}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x203000, 0xc00246ca50, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0020a7980}, {0x2076920, 0x46}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, 0x7f00048b4108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002849e50, {0x2392298, 0xc0020a7980}, {0x2076920, 0x20195c538}, {0x1f33320, 0xc0012cdc00}, {0x1f736c0, 0xc0012c1320}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e3c0, {0x2392298, 0xc0020a7980}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001951a00, {0x2392298, 0xc0020a7980})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1658145 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00250de60, {0x2392298, 0xc00263ee40}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000fe85a0, {0x23922d0, 0xc00214dfb0}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0024768d0, {0x23922d0, 0xc00214dfb0}, 0xc001eb5c00, 0x1, 0x1eb5c10)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc0006ba2a0, 0xc00214dda0, 0xc0010dce40)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc00074baa0, {0x23922d0, 0xc00214db00}, 0xc0010dce40)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00214db00}, {0x1f35540, 0xc0010dce40})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a24c60, {0x23922d0, 0xc00214db00}, {0x1f35540, 0xc0010dce40}, 0x237bc38, 0xc0023f6990)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214db00}, {0x1f35540, 0xc0010dce40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00214d9b0}, {0x1f35540, 0xc0010dce40}, 0xc0010dc300, 0xc0010dcec0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214d9b0}, {0x1f35540, 0xc0010dce40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00214d9b0}, {0x1f35540, 0xc0010dce40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214d9b0}, {0x1f35540, 0xc0010dce40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00214d980}, {0x1f35540, 0xc0010dce40}, 0x40, 0xc0010dcec0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214d980}, {0x1f35540, 0xc0010dce40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00214d980}, {0x1f35540, 0xc0010dce40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00214d980}, {0x1f35540, 0xc0010dce40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00214d980}, {0x1f35540, 0xc0010dce40}, 0xc000e80a60, 0xc0023f6990)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc00074baa0}, {0x23922d0, 0xc00214d980}, 0xc00263ecc0, 0xc0007c2220)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001bfd8c0, 0xc001339560, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0007a4540, {0x23d1088, 0xc00081b500}, 0xc001bfd8c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment