goroutine.txt

goroutine 1358679 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc000828a80})
	/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc000828a80}, 0x0)
	/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc000828a80}, 0xc)
	/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc001c38161, 0x34934e0}, {0x2386300, 0xc000828a80}, 0xc001c38154)
	/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc000828a80}, 0xc000734900)
	/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc000828a80}, 0x0)
	/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc001c3816b, {0x2386300, 0xc000828a80}, 0xc000734900)
	/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc00247f1a0}, {0x2386300, 0xc000828a80}, 0xc000734900)
	/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc00156e000, {0x23922d0, 0xc0006eb5c0})
	/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3033 +0x4e8

goroutine 1 [chan receive, 13 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc00073c000)
	/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc00009b340)
	/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc0001270e0, 0xc00009b300)
	/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc00053b1e0, {0x2392260, 0xc0000b0000}, {0xc0000ae000, 0x8, 0x8})
	/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
	/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
	/temporal/cmd/server/main.go:50 +0x45

goroutine 27 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00003a380)
	/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
	/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92

goroutine 3 [syscall, 13 minutes]:
os/signal.signal_recv()
	/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x2c

goroutine 4 [chan receive, 13 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
	/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
	/temporal/temporal/interrupt.go:38 +0xef

goroutine 5 [chan receive, 13 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
	/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
	/temporal/temporal/interrupt.go:38 +0xef

goroutine 65 [select]:
database/sql.(*DB).connectionCleaner(0xc000533380, 0x0)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 122 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc0005796c0, {0x2392228, 0xc000c696c0})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 68 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef1830, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003bc00, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003bc00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003bc00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000d890)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000d890)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0000859e0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790000)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 76 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00056b4a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 58 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 51 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc000533380, {0x2392228, 0xc000753d00})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 1266200 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0x0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 49 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000088a50)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 16 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef1a00, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003ab00, 0x416f26, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003ab00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003ab00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000c1c8)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000c1c8)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc0003280e0, {0x23860f0, 0xc00000c1c8})
	/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc0003280e0)
	/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
	/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
	/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
	/temporal/common/pprof/pprof.go:72 +0x9c

goroutine 1266153 [select]:
go.temporal.io/server/service/history.(*timerQueueProcessorBase).internalProcessor(0xc00127b080)
	/temporal/service/history/timerQueueProcessorBase.go:271 +0x37f
go.temporal.io/server/service/history.(*timerQueueProcessorBase).processorPump(0xc00127b080)
	/temporal/service/history/timerQueueProcessorBase.go:196 +0x9e
created by go.temporal.io/server/service/history.(*timerQueueProcessorBase).Start
	/temporal/service/history/timerQueueProcessorBase.go:163 +0xaf

goroutine 584 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060ec60)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 1282761 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001ba4ae0, {0x2392298, 0xc0008a9f80}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000640a20, {0x23922d0, 0xc001e99ad0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001e99ad0}, 0xc001e96cb0, 0x1bd58c0, 0x1e96cc0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001e99a70, 0xc001e91580)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001e997d0}, 0xc001e91580)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e997d0}, {0x1f353a0, 0xc001e91580})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e997d0}, {0x1f353a0, 0xc001e91580}, 0x237bc38, 0xc001e89818)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e997d0}, {0x1f353a0, 0xc001e91580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001e99650, {0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580}, 0xc001e98990, 0xc001e915c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580}, 0x40, 0xc001e915c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580}, 0xc0019a5620, 0xc001e89818)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e995f0}, 0xc0008a9f20, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000681c20, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000681c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 69 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc0006c0d00, {0x2392228, 0xc0007966c0})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 75 [IO wait, 12 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef1748, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00079a080, 0xc00004e800, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00079a080)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00079a080)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000dbd8)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000dbd8)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0007501c0, {0x23860f0, 0xc00000dbd8})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc0007137d0)
	/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc00056b360}, 0xc00072001c)
	/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
	/temporal/service/history/fx.go:168 +0x9a

goroutine 1266165 [select, 1 minutes]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001be0bb0)
	/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
	/temporal/service/history/visibilityQueueProcessor.go:203 +0x85

goroutine 78 [select]:
database/sql.(*DB).connectionOpener(0xc000aa20d0, {0x2392228, 0xc000a948c0})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 59 [sleep]:
time.Sleep(0x306dc4200)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 85 [select]:
database/sql.(*DB).connectionCleaner(0xc000aa20d0, 0xc00056b4a0)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 1286354 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0023bd440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f1e0, {0x1f7b560, 0xc000630dc0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa600, 0xc00160e440, 0xc00160e410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa600, {0x1f7b560, 0xc000630dc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00212f920}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x20, {0xc001fa1520, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00212f920}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00212f920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0xc000a2f8c0, {0xc001fa1520, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00212f920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0xc0021b2800, 0xc00212f950, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00212f920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e90100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x33, 0xc00212f890, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e90100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e90100}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001e90100}, 0xc0011d09c6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc00286bec0}, 0xc001e90040, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc00286bec0}, 0xc001e90040, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00286bbc0, {0x2392298, 0xc00286bec0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00286be60, {0x2392298, 0xc00286bec0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0023bcea0, {0x23922d0, 0xc00212ec90}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00212ec90}, 0xc001e1e740, 0x1bd58c0, 0x1e1e750)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc00212ec30, 0xc001d73480)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc00212e990}, 0xc001d73480)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00212e990}, {0x1f353a0, 0xc001d73480})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc00212e990}, {0x1f353a0, 0xc001d73480}, 0x237bc38, 0xc0020c1260)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e990}, {0x1f353a0, 0xc001d73480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00212e720, {0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480}, 0xc0021152c0, 0xc001d734c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480}, 0x40, 0xc001d734c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480}, 0xc001fa1220, 0xc0020c1260)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc00212e6c0}, 0xc00286b920, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0008cfe60, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0008cfe60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1174605 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002572360, {0x2392228, 0xc002576640})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 88 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
	/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
	/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8

goroutine 89 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 90 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 120 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 1300594 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002481e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5760, {0x1f7b560, 0xc0000203c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000734600, 0xc002789d18, 0xc002789ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000734600, {0x1f7b560, 0xc0000203c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x20, {0xc0021d7760, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x7fb42acf4f50, {0xc0021d7760, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0xc0017985b0, 0xc001f1af00, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001f14b40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x237ac70, 0xc001f1ae40, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001f14b40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001f14b40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001f14b40}, 0xc001d19f20, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001f1a780}, 0xc001f14ac0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001f1a780}, 0xc001f14ac0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00278a5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00278a648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad28cc0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001f1a780}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00251b5d0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00278a898, {0x235a640, 0xc000045ad0}, 0x392d353831342d38)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001f1a780}, 0xc001f1a5d0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a2adb8, 0x4c1cb4}, {0xc057031378e9340c, 0xb4d300a989}, {0xc001a2adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001f1a780}, 0xc001f1a5d0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0xc0021d75e0, 0xc0026e73c8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0x15, 0xc001f14880)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0x207690b, 0xc001f14880)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0x237bc38, 0xc001f14880)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f1a630}, {0x1f33180, 0xc001f1a5d0}, 0x1e5a940, 0xc001f14880)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a630}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0x9, 0xc001f14880)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0xc001c8f768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001c8fb60, {0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0xc0021d75e0, 0xc0026e73c8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001f1a5a0}, 0xc0017dce40, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002480900, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002480900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282766 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00242f320, {0x2392298, 0xc001ffe6c0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002465200, {0x23922d0, 0xc001ea3920}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001ea3920}, 0xc001e97880, 0x1bd58c0, 0x1e97890)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001ea38c0, 0xc001ea6140)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001ea3620}, 0xc001ea6140)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ea3620}, {0x1f353a0, 0xc001ea6140})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001ea3620}, {0x1f353a0, 0xc001ea6140}, 0x237bc38, 0xc001ea4228)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3620}, {0x1f353a0, 0xc001ea6140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001ea34a0, {0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140}, 0xc001ea27e0, 0xc001ea6180)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140}, 0x40, 0xc001ea6180)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140}, 0xc0019a59e0, 0xc001ea4228)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001ea3440}, 0xc001ffe660, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00196b9e0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00196b9e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286004 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001588f00, {0x2392228, 0xc002232840})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1319418 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 667 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000fbc3a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 128 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 116 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef12c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00076ce00, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00076ce00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00076ce00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000a0ab70)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000a0ab70)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc00002d340, {0x23860f0, 0xc000a0ab70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc0007137d0)
	/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc000a36720}, 0x0)
	/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
	/temporal/service/matching/fx.go:129 +0x9a

goroutine 1318871 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001961560)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001838c60, {0x1f736c0, 0xc00105c2d0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6c00, 0xc00151b5c0, 0xc00151b590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6c00, {0x1f736c0, 0xc00105c2d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002036cc0}, {0x2076920, 0x8}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x1c75901, {0xc00247d660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x7fb451d0c5b8, {0xc00247d660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x20, 0xc001b75230, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0xc0006ca340, {0xc001052580, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x0, 0xc001b75200, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x0, 0xc001b750e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002036cc0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x203000, 0xc001b74ff0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc00186be50, {0x2392298, 0xc002036cc0}, {0x2076920, 0x22ac7a3d0}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002036cc0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc002036cc0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 174 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00034edc0, 0x1fef77d)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 1266199 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0x0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266214 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc00177b2c0)
	/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
	/temporal/service/history/queueProcessor.go:158 +0x23b

goroutine 127 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 1318718 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00032d320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018388f0, {0x1f736c0, 0xc000fbe120}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968500, 0xc0008915c0, 0xc000891590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968500, {0x1f736c0, 0xc000fbe120})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0015bb200}, {0x2076920, 0x8}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x1c75901, {0xc00247cbc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x7fb451d0c5b8, {0xc00247cbc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x20, 0xc0022037d0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0xc0006ca340, {0xc000fb3380, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x0, 0xc0022037a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x0, 0xc002203710, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0015bb200}, {0x2076920, 0x203000}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x203000, 0xc0022036e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc00186be50, {0x2392298, 0xc0015bb200}, {0x2076920, 0x200a74410}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0015bb200}, 0xc0006ef8c0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc0015bb200})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1282661 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000645e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb46e0, {0x1f736c0, 0xc001d62900}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968600, 0xc00152d5c0, 0xc00152d590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968600, {0x1f736c0, 0xc001d62900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002282720}, {0x2076920, 0x8}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x1c75901, {0xc0008e7f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x7fb451d0c108, {0xc0008e7f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x20, 0xc001d66d50, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0xc0006ca340, {0xc001d5ecc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x0, 0xc001d66d20, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x1e0cfc0, 0xc001d66c90, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0xc051a60d0c, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002282720}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x203000, 0xc001d66c60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000fa0e50, {0x2392298, 0xc002282720}, {0x2076920, 0x1010013d8}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002282720}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc002282720})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1270840 [select, 1 minutes]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00211f200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e8f0, {0x1f736c0, 0xc0024c42d0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969000, 0xc0015195c0, 0xc001519590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969000, {0x1f736c0, 0xc0024c42d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x8}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x1c75901, {0xc0019a45a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x7fb451d0c108, {0xc0019a45a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x20, 0xc002136090, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0xc0006ca340, {0xc002446780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x0, 0xc002136060, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x1e0cfc0, 0xc002117fb0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x4b400cb94e8, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x203000, 0xc002117f80, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000cb9650, {0x2392298, 0xc002c5a0c0}, {0x2076920, 0x100cb96d0}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002c5a0c0}, 0x1fe4e76, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc002c5a0c0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 835 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0978, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000aba680, 0xc0010f6000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000aba680, {0xc0010f6000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000aba680, {0xc0010f6000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa528, {0xc0010f6000, 0x4383e0, 0xc001264ea0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0008a6ba0, {0xc000828660, 0x9, 0xc000a46d18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc0008a6ba0}, {0xc000828660, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000828660, 0x9, 0x18}, {0x23585c0, 0xc0008a6ba0})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000828620)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000ad20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1266069 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc00185fd58)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 358 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e790)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 357 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e6e0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 173 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef0b80, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000af780, 0xc0007dcef0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000af780, {0xc0007dcef0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000af780, {0xc0007dcef0, 0xc000015f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa068, {0xc0007dcef0, 0x46b213, 0x617b94f5})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa068}, {0xc0007dcef0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00034edc0, 0x7ff9f0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 359 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e840)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 117 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a9b1d0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 130 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef1490, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00076cb00, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00076cb00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00076cb00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000a0a870)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000a0a870)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc00053e880)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000352600)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 131 [sleep]:
time.Sleep(0x306dc4200)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 191 [select]:
database/sql.(*DB).connectionCleaner(0xc0007c0750, 0x1c)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 1272206 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00211c360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e210, {0x1f7b560, 0xc00204abe0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000734c00, 0xc002585d18, 0xc002585ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000734c00, {0x1f7b560, 0xc00204abe0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x20, {0xc001fa04e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x7fb42ac74c70, {0xc001fa04e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0xc0023ec2f0, 0xc001ef3f20, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000c35b00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x237ac70, 0xc001ef3e60, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000c35b00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000c35b00}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc000c35b00}, 0xc0027bcb70, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001ef3770}, 0xc000c35a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001ef3770}, 0xc000c35a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0025865a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002586648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad664f0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001a31140, {0x23922d0, 0xc001ef3770}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0010bc9e0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002586898, {0x235a640, 0xc000045ad0}, 0x612d346530342d64)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001ef3770}, 0xc001ef35c0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00180cdb8, 0x4c1cb4}, {0xc0570311b98c93a7, 0xb33268832b}, {0xc00180cdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001ef3770}, 0xc001ef35c0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0xc001fa0400, 0xc0019663c0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0x15, 0xc000c35780)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0x207690b, 0xc000c35780)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0x237bc38, 0xc000c35780)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ef3620}, {0x1f33180, 0xc001ef35c0}, 0x1e5a940, 0xc000c35780)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3620}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0x9, 0xc000c35780)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0xc0026da768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0026dab60, {0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0xc001fa0400, 0xc0019663c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001ef3590}, 0xc00283fb00, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00107cc60, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00107cc60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 356 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 201 [sleep]:
time.Sleep(0x37e11d600)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 354 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 855 [select, 13 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00075d860)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 179 [select]:
database/sql.(*DB).connectionCleaner(0xc0005796c0, 0x1fef77d)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 583 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060ebb0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 175 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef0c68, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000aec00, 0xc0007dd260, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000aec00, {0xc0007dd260, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000aec00, {0xc0007dd260, 0xc000ab1c20, 0xc001b7b230})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa060, {0xc0007dd260, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa060}, {0xc0007dd260, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00060c580, 0x1fef77d)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 176 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00060c580, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 360 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e8f0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 186 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc0007c0750, {0x2392228, 0xc0009fb680})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 192 [sleep]:
time.Sleep(0x28fa6ae00)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 190 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 851 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 848 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc002200ac0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1270842 [select, 1 minutes]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00213a240)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e9a0, {0x1f7b7c0, 0xc00204b9a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969200, 0xc00186fbf8, 0xc00186fbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969200, {0x1f7b7c0, 0xc00204b9a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x20, {0xc0019a4860, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0xc0026454a0, {0xc0019a4860, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0xc0023ec2f0, 0xc002136d50, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002446d00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x237ac70, 0xc002136c90, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002446d00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002446d00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc002446d00}, 0xc002672750, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0021365d0}, 0xc002446c80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0021365d0}, 0xc002446c80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001870480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001870528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ace38b0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0021365d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000756ae0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001870778, {0x235a640, 0xc000045ad0}, 0x382d643438342d37)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0021365d0}, 0xc002446980)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00197adb8, 0x4c1cb4}, {0xc05703118058bf28, 0xb2f934aea6}, {0xc00197adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0021365d0}, 0xc002446980)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0xc0019a4720, 0xc001b8d530)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0x15, 0xc0024469c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0x2076951, 0xc0024469c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0x237bc38, 0xc0024469c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002136480}, {0x1f33320, 0xc002446980}, 0x1e5a940, 0xc0024469c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136480}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0x23c27b0, 0xc0024469c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a04b60, {0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0xc0019a4720, 0xc001b8d530)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002136450}, 0xc002c5a300, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00211f320, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00211f320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 213 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 850 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc00112d9c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1285974 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001766360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bac60, {0x1f7c860, 0xc0022babb0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000735400, 0xc0005bf5e8, 0xc0005bf5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000735400, {0x1f7c860, 0xc0022babb0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x8}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x1c75901, {0xc0019a58a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x7fb451d0c5b8, {0xc0019a58a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x20, 0xc00256fb90, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0xc0007c0000, {0xc0021e6780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, 0xc00256fb30, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x1e0cfc0, 0xc00256fa70, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x8, 0xc00256f9e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc000fe9ce0}, {0x20768da, 0x0}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc000fe9ce0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc000fe9ce0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 671 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc001d68250)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 450 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005426e0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 136 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42ada69d8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c21700, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c21700)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000c21700)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000c7c7f8)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000c7c7f8)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc00053e560)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790200)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 137 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc000aa2270, {0x2392228, 0xc0006e6a80})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 183 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7fb42ada68f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c21a00, 0xc00004e800, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c21a00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000c21a00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000c7caf8)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000c7caf8)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000750380, {0x23860f0, 0xc000c7caf8})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0xc0008217d0)
	/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc00003a680}, 0x0)
	/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
	/temporal/service/frontend/fx.go:282 +0x9a

goroutine 184 [select, 3 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00056a6e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 1266067 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc000edac80)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266071 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc00185fb18)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 199 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 196 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42ada6468, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000afb00, 0x40d187, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0000afb00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0000afb00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000569488)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000569488)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007991c0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790400)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 262 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 198 [chan receive, 13 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0xc000497fd0)
	/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc000c73220}, 0x0)
	/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
	/temporal/service/worker/fx.go:78 +0x9a

goroutine 214 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 1289988 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0024667e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001556370, {0x1f7c860, 0xc0015562c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aae00, 0xc0005bd5e8, 0xc0005bd5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aae00, {0x1f7c860, 0xc0015562c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001da3680}, {0x20768da, 0x8}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x1c75901, {0xc000084420, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x7fb451d0c5b8, {0xc000084420, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x20, 0xc0021be900, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0xc0007c0000, {0xc0009db040, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x0, 0xc0021be8d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x1e0cfc0, 0xc0021be840, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001da3680}, {0x20768da, 0x203000}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x8, 0xc0021be810, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc001da3680}, {0x20768da, 0x0}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc001da3680}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc001da3680})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 250 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5e10, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000888a80, 0xc000757350, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000888a80, {0xc000757350, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000888a80, {0xc000757350, 0xc0007b9f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa5b8, {0xc000757350, 0x46b213, 0x617b94f5})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa5b8}, {0xc000757350, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00075eb00, 0x7202b0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 254 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0b48, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000888980, 0xc0007574f0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000888980, {0xc0007574f0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000888980, {0xc0007574f0, 0xc0007bdf50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa590, {0xc0007574f0, 0x46b213, 0x617b94f5})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa590}, {0xc0007574f0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006eedc0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 355 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 451 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000542790)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 447 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 669 [select, 1 minutes]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000df12b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 670 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc001d166e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 263 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 1266148 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc001ba74b8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 578 [select]:
github.com/temporalio/ringpop-go/swim.sendPingWithChanges(0xc0008bf040, {0xc001248bf0, 0x10}, {0x34c4b38, 0x0, 0x0}, 0x0)
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/ping_sender.go:97 +0x6f5
github.com/temporalio/ringpop-go/swim.sendPing(0xc0008bf040, {0xc001248bf0, 0x10}, 0xc000bf9f74)
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/ping_sender.go:47 +0x6b
github.com/temporalio/ringpop-go/swim.(*Node).pingNextMember(0xc0008bf040)
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/node.go:527 +0x108
github.com/temporalio/ringpop-go/swim.(*gossip).ProtocolPeriod(0xc00077b3f0)
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/gossip.go:177 +0x47
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:30 +0xb2
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 252 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5fe0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000aff80, 0xc0007574e0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000aff80, {0xc0007574e0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000aff80, {0xc0007574e0, 0xc0028a8750, 0xc000ed9460})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa5b0, {0xc0007574e0, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa5b0}, {0xc0007574e0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006eeb00, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 1282222 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006685a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022baf20, {0x1f7b560, 0xc00156e5a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7100, 0xc000882440, 0xc000882410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7100, {0x1f7b560, 0xc00156e5a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x20, {0xc0021d7ac0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0xc0020d7b80, {0xc0021d7ac0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0xc0021b2800, 0xc001d8c960, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001cdff80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x237ac70, 0xc001d8c8a0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001cdff80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001cdff80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001cdff80}, 0xc0023b2ae6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0025d9a40}, 0xc001cdff00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0025d9a40}, 0xc001cdff00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0024adaa0, {0x2392298, 0xc0025d9a40})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024adf80, {0x2392298, 0xc0025d9a40}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001082360, {0x23922d0, 0xc001c523f0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001c523f0}, 0xc001c50560, 0x1bd58c0, 0x1c50570)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001c52390, 0xc001c32680)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001c520f0}, 0xc001c32680)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001c520f0}, {0x1f353a0, 0xc001c32680})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001c520f0}, {0x1f353a0, 0xc001c32680}, 0x237bc38, 0xc0021dede0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c520f0}, {0x1f353a0, 0xc001c32680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001b75f50, {0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680}, 0xc00186d5b0, 0xc001c326c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680}, 0x40, 0xc001c326c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680}, 0xc0021d7280, 0xc0021dede0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001b75ef0}, 0xc0025d99e0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024805a0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024805a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 251 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00075eb00, 0x790000)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 253 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006eeb00, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 255 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006eedc0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 580 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 302 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0320, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889a80, 0xc000c66ad0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889a80, {0xc000c66ad0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889a80, {0xc000c66ad0, 0xc000ab1c20, 0xc001b7ad20})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aace8, {0xc000c66ad0, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aace8}, {0xc000c66ad0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef8c0, 0xa99ae0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 573 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 452 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e630)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 453 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e9a0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 448 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 449 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 1286320 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002216120, {0x2392228, 0xc001dc2440})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1318843 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006457a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001838bb0, {0x1f7b7c0, 0xc000021180}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6a00, 0xc002347bf8, 0xc002347bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6a00, {0x1f7b7c0, 0xc000021180})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x20, {0xc00247d480, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0xc0020966e0, {0xc00247d480, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0xc0017985b0, 0xc001d31320, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001047600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x237ac70, 0xc001d31200, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001047600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001047600}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001047600}, 0xc00104f140, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001d30990}, 0xc001047580, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001d30990}, 0xc001047580, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc002348480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002348528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acf1d80, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001d30990}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00289baa0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002348778, {0x235a640, 0xc000045ad0}, 0x392d653239342d32)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001d30990}, 0xc001047280)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001b9cdb8, 0x4c1cb4}, {0xc0570314bb0103eb, 0xb5ff1e6b73}, {0xc001b9cdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001d30990}, 0xc001047280)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0xc00247d380, 0xc0025baaf8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0x15, 0xc0010472c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0x2076951, 0xc0010472c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0x237bc38, 0xc0010472c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001d30840}, {0x1f33320, 0xc001047280}, 0x1e5a940, 0xc0010472c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30840}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0xc00104e940, 0xc0010472c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00186bb60, {0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0xc00247d380, 0xc0025baaf8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001d30810}, 0xc001971aa0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0017677a0, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0017677a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 849 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc00104baa0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 672 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000f7cab0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 298 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0d18, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889b80, 0xc000c66960, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889b80, {0xc000c66960, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889b80, {0xc000c66960, 0xc000898f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aad10, {0xc000c66960, 0x46b213, 0x617b94f5})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aad10}, {0xc000c66960, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef340, 0x9e6580)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 247 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0a60, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000888a00, 0xc000757340, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000888a00, {0xc000757340, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000888a00, {0xc000757340, 0xc000c65d10, 0xc001b7a610})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa598, {0xc000757340, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa598}, {0xc000757340, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00075e840, 0xab8720)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 248 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00075e840, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 1277170 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc00222b4a0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 300 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5c40, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889100, 0xc000c66ac0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889100, {0xc000c66ac0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889100, {0xc000c66ac0, 0xc000c65d10, 0xc001b7ab40})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aad08, {0xc000c66ac0, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aad08}, {0xc000c66ac0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef600, 0x5d09f0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 299 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef340, 0x1d8150)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 301 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef600, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 1318923 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000127e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9080, {0x1f736c0, 0xc001107b90}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aaf00, 0xc001c095c0, 0xc001c09590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aaf00, {0x1f736c0, 0xc001107b90})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0028c46c0}, {0x2076920, 0x8}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x1c75901, {0xc001fa0b20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x7fb451d0c108, {0xc001fa0b20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x20, 0xc001d38030, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0xc0006ca340, {0xc001146440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x0, 0xc001d38000, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x0, 0xc001d67ec0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0028c46c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x203000, 0xc001d67e90, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc0028c46c0}, {0x2076920, 0x200e03640}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0028c46c0}, 0xc000790400, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc0028c46c0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1266177 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc000de6c30)
	/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
	/temporal/service/history/queueProcessor.go:158 +0x23b

goroutine 1266116 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0790, 0xc000a0b440)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 303 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef8c0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 1318849 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00067b7a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce8fd0, {0x1f736c0, 0xc000f7abd0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aac00, 0xc00152f5c0, 0xc00152f590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aac00, {0x1f736c0, 0xc000f7abd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001cc76e0}, {0x2076920, 0x8}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x1c75901, {0xc001fa0820, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x7fb451d0c108, {0xc001fa0820, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x20, 0xc001d8d830, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0xc0006ca340, {0xc000f75ac0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x0, 0xc001d8d800, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x0, 0xc001d8d770, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001cc76e0}, {0x2076920, 0x203000}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x203000, 0xc001d8d740, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc001cc76e0}, {0x2076920, 0x266343752}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001cc76e0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc001cc76e0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1272371 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006777a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e420, {0x1f736c0, 0xc0007719e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa100, 0xc000aab5c0, 0xc000aab590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa100, {0x1f736c0, 0xc0007719e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00230d920}, {0x2076920, 0x8}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x1c75901, {0xc001fa1260, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x7fb451d0c108, {0xc001fa1260, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x20, 0xc0009d7f20, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0xc0006ca340, {0xc000a944c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x0, 0xc0009d7ef0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x1e0cfc0, 0xc0009d7e60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x4b700cb74e8, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00230d920}, {0x2076920, 0x203000}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x203000, 0xc0009d7e30, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000cb7650, {0x2392298, 0xc00230d920}, {0x2076920, 0x100000000}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc00230d920}, 0x4802f6, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc00230d920})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 668 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000e5a9a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 581 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060ea50)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 582 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060eb00)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 925 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc0007fd380)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0008dbef0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0008dbef0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1002155 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000946820, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0024594a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 579 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 403 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016ae40, {0x2392228, 0xc000edabc0})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 404 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 295 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0238, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889b00, 0xc000c66950, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889b00, {0xc000c66950, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889b00, {0xc000c66950, 0xc00089df50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aacf0, {0xc000c66950, 0x46b213, 0x617b94f5})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aacf0}, {0xc000c66950, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef080, 0x9e6250)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 296 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef080, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 1266204 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0x0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1282558 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019618c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb340, {0x1f7b560, 0xc00156ec80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7700, 0xc001611d18, 0xc001611ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7700, {0x1f7b560, 0xc00156ec80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x20, {0xc0021d7f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0xc002644f20, {0xc0021d7f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0xc0023ec2f0, 0xc001d994d0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d93580}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x237ac70, 0xc001d99410, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d93580}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d93580}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001d93580}, 0xc00003e900, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001d98d50}, 0xc001d93500, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001d98d50}, 0xc001d93500, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0016125a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001612648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad221e0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001d98d50}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0024ba260})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001612898, {0x235a640, 0xc000045ad0}, 0x382d613065342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001d98d50}, 0xc001d98ba0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc0005c6db8, 0x4c1cb4}, {0xc0570312400e0741, 0xb3abba54c0}, {0xc0005c6db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001d98d50}, 0xc001d98ba0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0xc0021d7e80, 0xc002652e28)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0x15, 0xc001d932c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0x207690b, 0xc001d932c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0x237bc38, 0xc001d932c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001d98c00}, {0x1f33180, 0xc001d98ba0}, 0x1e5a940, 0xc001d932c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98c00}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0x9, 0xc001d932c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0xc001867768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001867b60, {0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0x40, 0xc00005fc00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0xc0021d7e80, 0xc002652e28)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001d98b70}, 0xc00250bb00, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001960480, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001960480, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286340 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023cb4a0, {0x2392298, 0xc0028f9a40}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a12b40, {0x23922d0, 0xc0020fa900}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0020fa900}, 0xc001d687e0, 0x1bd58c0, 0x1d687f0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0020fa540, 0xc001d5eb40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0020fa270}, 0xc001d5eb40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020fa270}, {0x1f353a0, 0xc001d5eb40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0020fa270}, {0x1f353a0, 0xc001d5eb40}, 0x237bc38, 0xc0022e3920)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa270}, {0x1f353a0, 0xc001d5eb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40}, 0x203000, 0xc001d5eb80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40}, 0x40, 0xc001d5eb80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40}, 0xc001fa02c0, 0xc0022e3920)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0020fa090}, 0xc0028f99e0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0017f9320, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0017f9320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286313 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002339320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060ef20, {0x1f736c0, 0xc000daccf0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5f)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6e00, 0xc00158b5c0, 0xc00158b590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6e00, {0x1f736c0, 0xc000daccf0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001b67980}, {0x2076920, 0x8}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x1c75901, {0xc0008e6660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x7fb451d0c5b8, {0xc0008e6660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x20, 0xc0017fe1b0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0xc0006ca340, {0xc001d47480, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x0, 0xc0017fe180, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x1e0cfc0, 0xc0017fe0f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x4cc47c577b3, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001b67980}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x203000, 0xc0017fe0c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000e03e50, {0x2392298, 0xc001b67980}, {0x2076920, 0x14f6a69fe}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001b67980}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e57c70, {0x2392298, 0xc001b67980})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1266215 [select, 1 minutes]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001568210)
	/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
	/temporal/service/history/visibilityQueueProcessor.go:203 +0x85

goroutine 1266101 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc0000a3770)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286281 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00110fda0, {0x2392228, 0xc001d2f340})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266056 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69d90, 0xc00185fd58)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266051 [select, 1 minutes]:
go.temporal.io/server/service/history.NewLocalTimerGate.func1()
	/temporal/service/history/timerGate.go:114 +0xe5
created by go.temporal.io/server/service/history.NewLocalTimerGate
	/temporal/service/history/timerGate.go:109 +0x15e

goroutine 1318874 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000a923c0, {0x2392298, 0xc002037260}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000378b40, {0x23922d0, 0xc001db4960}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0006ccc60, {0x23922d0, 0xc001db4960}, 0xc00105e150, 0x1, 0x105e160)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001db4900, 0xc001052d00)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001db4690}, 0xc001052d00)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001db4690}, {0x1f35540, 0xc001052d00})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001db4690}, {0x1f35540, 0xc001052d00}, 0x237bc38, 0xc0025bbcc8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db4690}, {0x1f35540, 0xc001052d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00}, 0xc001052780, 0xc001052d80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00}, 0x40, 0xc001052d80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00}, 0xc00247dbe0, 0xc0025bbcc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001db44e0}, 0xc002037200, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c13c20, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c13c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1276962 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006bec60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5ce0, {0x1f7b7c0, 0xc000020e60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb7)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a800, 0xc001751bf8, 0xc001751bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a800, {0x1f7b7c0, 0xc000020e60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x20, {0xc000799540, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0xc001581760, {0xc000799540, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0xc0023ec2f0, 0xc002077e30, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00207e9c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x237ac70, 0xc002077d70, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00207e9c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00207e9c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc00207e9c0}, 0xc00074c5e0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0020776b0}, 0xc00207e940, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0020776b0}, 0xc00207e940, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001752480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001752528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad35118, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0020776b0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002061090})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001752778, {0x235a640, 0xc000045ad0}, 0x392d363266342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0020776b0}, 0xc00207e640)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001c64db8, 0x4c1cb4}, {0xc05703120130b699, 0xb371423a15}, {0xc001c64db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0020776b0}, 0xc00207e640)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0xc000799260, 0xc00205d230)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0x15, 0xc00207e680)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0x2076951, 0xc00207e680)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0x237bc38, 0xc00207e680)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002077560}, {0x1f33320, 0xc00207e640}, 0x1e5a940, 0xc00207e680)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077560}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0x23c27b0, 0xc00207e680)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a05b60, {0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0xc000799260, 0xc00205d230)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002077530}, 0xc001ccc120, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006810e0, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006810e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282217 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00230c1e0, {0x2392298, 0xc0025d9500}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00211d320, {0x23922d0, 0xc001b75770}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001b75770}, 0xc001b55e10, 0x1bd58c0, 0x1b55e20)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001b75560, 0xc001c320c0)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001b752c0}, 0xc001c320c0)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001b752c0}, {0x1f353a0, 0xc001c320c0})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001b752c0}, {0x1f353a0, 0xc001c320c0}, 0x237bc38, 0xc0021de8d0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b752c0}, {0x1f353a0, 0xc001c320c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7, {0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0}, 0xc00186d5b0, 0xc001c32100)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0}, 0x40, 0xc001c32100)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0}, 0xc0021d7120, 0xc0021de8d0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001b75110}, 0xc0025d94a0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0006765a0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0006765a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 869 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc001d90440)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 870 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc002122630)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1266142 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc00101cf00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1318720 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002466480)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018389a0, {0x1f7b7c0, 0xc0000205a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968d00, 0xc001755bf8, 0xc001755bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968d00, {0x1f7b7c0, 0xc0000205a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x20, {0xc00247cda0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0xc001581600, {0xc00247cda0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0xc0017985b0, 0xc0021ca690, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000fb3900}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x237ac70, 0xc0021ca5d0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000fb3900}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000fb3900}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc000fb3900}, 0xc001d49b40, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc002203d10}, 0xc000fb3880, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc002203d10}, 0xc000fb3880, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001756480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001756528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acc4140, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002203d10}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002104ee0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001756778, {0x235a640, 0xc000045ad0}, 0x622d626262342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc002203d10}, 0xc000fb3580)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00158cdb8, 0x4c1cb4}, {0xc0570314b85e7290, 0xb5fc7bda76}, {0xc00158cdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc002203d10}, 0xc000fb3580)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0xc00247ccc0, 0xc0025e5278)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0x15, 0xc000fb35c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0x2076951, 0xc000fb35c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0x237bc38, 0xc000fb35c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002203bc0}, {0x1f33320, 0xc000fb3580}, 0x1e5a940, 0xc000fb35c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203bc0}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0x23c27b0, 0xc000fb35c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00186bb60, {0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0xc00247ccc0, 0xc0025e5278)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002203b90}, 0xc0015bb680, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00032d440, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00032d440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266132 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc00185f770)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1174603 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002574140)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1319106 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b845a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9a20, {0x1f7b7c0, 0xc0019808c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee6800, 0xc001ca5bf8, 0xc001ca5bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee6800, {0x1f7b7c0, 0xc0019808c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x20, {0xc0016521c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0xc00199e9a0, {0xc0016521c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0xc0017985b0, 0xc001f1a960, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00117f640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x237ac70, 0xc001f1a870, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00117f640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00117f640}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc00117f640}, 0xc00203d5c0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001ea3ec0}, 0xc00117f5c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001ea3ec0}, 0xc00117f5c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001ca6480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001ca6528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad24468, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001ea3ec0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0027af7c0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001ca6778, {0x235a640, 0xc000045ad0}, 0x612d633434342d38)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001ea3ec0}, 0xc00117f2c0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00155edb8, 0x4c1cb4}, {0xc0570314c2e5511a, 0xb6029d8298}, {0xc00155edb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001ea3ec0}, 0xc00117f2c0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0xc0016520a0, 0xc001fb8cc0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0x15, 0xc00117f300)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0x2076951, 0xc00117f300)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0x237bc38, 0xc00117f300)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ea3d70}, {0x1f33320, 0xc00117f2c0}, 0x1e5a940, 0xc00117f300)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3d70}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0}, 0x23c27b0, 0xc00117f300)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002471b60, {0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0}, 0x40, 0xc00005fc00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0}, 0xc0016520a0, 0xc001fb8cc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001ea3d40}, 0xc002561740, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006bfe60, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006bfe60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1318931 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00174e180, {0x2392298, 0xc0028c59e0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0015f97a0, {0x23922d0, 0xc001e0b8f0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002410990, {0x23922d0, 0xc001e0b8f0}, 0xc001160aa0, 0x1, 0x1160ab0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001e0b860, 0xc001147940)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001e0b470}, 0xc001147940)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e0b470}, {0x1f35540, 0xc001147940})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e0b470}, {0x1f35540, 0xc001147940}, 0x237bc38, 0xc00205c0f0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0b470}, {0x1f35540, 0xc001147940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e0b320}, {0x1f35540, 0xc001147940}, 0xc001147380, 0xc0011479c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0b320}, {0x1f35540, 0xc001147940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e0b320}, {0x1f35540, 0xc001147940}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0b320}, {0x1f35540, 0xc001147940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e0b2f0}, {0x1f35540, 0xc001147940}, 0x40, 0xc0011479c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0b2f0}, {0x1f35540, 0xc001147940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e0b2f0}, {0x1f35540, 0xc001147940}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0b2f0}, {0x1f35540, 0xc001147940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e0b2f0}, {0x1f35540, 0xc001147940}, 0xc001fa1b00, 0xc00205c0f0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e0b2f0}, 0xc0028c5980, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0003785a0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0003785a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266196 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0xc0026d6380)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1002167 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c08d80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1282681 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000127440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb6b0, {0x1f7b7c0, 0xc00156ff40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7f00, 0xc0021744d0, 0xc0021744a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7f00, {0x1f7b7c0, 0xc00156ff40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e1cdb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0x20, {0xc001fa1300, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e1cdb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e1cdb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0xc000eb22c0, {0xc001fa1300, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e1cdb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0xc0021b2800, 0xc001e1cde0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e1cdb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e0f300}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0x237ac70, 0xc001e1cd20, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e0f300}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e0f300}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001e0f240}, {0x1f7b7c0, 0xc00156ff40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001e0f300}, 0xc0017f4886, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0024dc600}, 0xc001e0f240, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000799920, {0x2392298, 0xc0024dc600}, 0xc001e0f240, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0028d1ce0, {0x2392298, 0xc0024dc600})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002560120, {0x2392298, 0xc0024dc600}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0008cfd40, {0x23922d0, 0xc001e1cb10}, 0x14)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0025f81b0, {0x23922d0, 0xc001e1cb10}, 0xc001dd5b00, 0x1, 0x1dd5b10)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001e1cab0, 0xc001e0f000)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001e1c810}, 0xc001e0f000)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e1c810}, {0x1f35540, 0xc001e0f000})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e1c810}, {0x1f35540, 0xc001e0f000}, 0x237bc38, 0xc001e071a0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e1c810}, {0x1f35540, 0xc001e0f000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e1c6c0}, {0x1f35540, 0xc001e0f000}, 0xc001e0e9c0, 0xc001e0f080)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e1c6c0}, {0x1f35540, 0xc001e0f000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e1c6c0}, {0x1f35540, 0xc001e0f000}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e1c6c0}, {0x1f35540, 0xc001e0f000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e1c690}, {0x1f35540, 0xc001e0f000}, 0x40, 0xc001e0f080)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e1c690}, {0x1f35540, 0xc001e0f000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e1c690}, {0x1f35540, 0xc001e0f000}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e1c690}, {0x1f35540, 0xc001e0f000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e1c690}, {0x1f35540, 0xc001e0f000}, 0xc001fa11e0, 0xc001e071a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e1c690}, 0xc0024dc5a0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0005e5320, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0005e5320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 883 [select, 13 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a08500)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 1282510 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000669320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb080, {0x1f7c860, 0xc0022bafd0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7500, 0xc0015155e8, 0xc0015155b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7500, {0x1f7c860, 0xc0022bafd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00250a900}, {0x20768da, 0x8}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x1c75901, {0xc0021d7bc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x7fb451d0c5b8, {0xc0021d7bc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x20, 0xc001d8cea0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0xc0007c0000, {0xc001d92200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x0, 0xc001d8ce70, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x1e0cfc0, 0xc001d8cde0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00250a900}, {0x20768da, 0x203000}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x8, 0xc001d8cdb0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00250a900}, {0x20768da, 0x46}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc00250a900}, {0x20768da, 0x0}, {0x1f33180, 0xc001d8cd50}, {0x1f7c860, 0xc0022bafd0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc00250a900}, 0xc000821f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc00250a900})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1300575 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0024807e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb56b0, {0x1f7c860, 0xc001cb5550}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969700, 0xc0007d35e8, 0xc0007d35b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969700, {0x1f7c860, 0xc001cb5550})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0017dcba0}, {0x20768da, 0x8}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x1c75901, {0xc0021d7440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x7fb451d0c5b8, {0xc0021d7440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x20, 0xc001f1a0c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0xc0007c0000, {0xc001f14580, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x0, 0xc001f1a090, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x1e0cfc0, 0xc001f1a000, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x7fb42b3dafff, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0017dcba0}, {0x20768da, 0x203000}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x8, 0xc001f03fb0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0017dcba0}, {0x20768da, 0x46}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc0017dcba0}, {0x20768da, 0x0}, {0x1f33180, 0xc001f03f50}, {0x1f7c860, 0xc001cb5550}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0017dcba0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc0017dcba0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 690 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb05d8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ee2d00, 0xc0010bc5b0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ee2d00, {0xc0010bc5b0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ee2d00, {0xc0010bc5b0, 0xc00001af50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa0e8, {0xc0010bc5b0, 0x46b213, 0x617b94f5})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa0e8}, {0xc0010bc5b0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0007b7340, 0x1096540)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 840 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada6638, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ee3900, 0xc000e68000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ee3900, {0xc000e68000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ee3900, {0xc000e68000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa530, {0xc000e68000, 0xc000a48cf8, 0x44d2f4})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0008a6c60, {0xc000828740, 0x9, 0xb602a5c82c})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc0008a6c60}, {0xc000828740, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000828740, 0x9, 0xc00056ab90}, {0x23585c0, 0xc0008a6c60})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000828700)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00084fb00, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc00002d340, {0x23d1088, 0xc00084fb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 688 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb04f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ee2d80, 0xc0010bc450, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ee2d80, {0xc0010bc450, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ee2d80, {0xc0010bc450, 0xc000c65d10, 0xc001b7b120})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa108, {0xc0010bc450, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa108}, {0xc0010bc450, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006efb80, 0x10a4c40)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 689 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006efb80, 0x10a4c40)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 691 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0007b7340, 0x10a4c40)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 896 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a09810, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000c49e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1270848 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020734a0, {0x2392298, 0xc002c5b260}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019977a0, {0x23922d0, 0xc00215cc00}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002672cc0, {0x23922d0, 0xc00215cc00}, 0xc00004ae50, 0x1, 0x4ae60)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc00215c9f0, 0xc002447e00)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc00215c750}, 0xc002447e00)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00215c750}, {0x1f35540, 0xc002447e00})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc00215c750}, {0x1f35540, 0xc002447e00}, 0x237bc38, 0xc0017ee078)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215c750}, {0x1f35540, 0xc002447e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00215c600}, {0x1f35540, 0xc002447e00}, 0x203000, 0xc002447e80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215c600}, {0x1f35540, 0xc002447e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00215c600}, {0x1f35540, 0xc002447e00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215c600}, {0x1f35540, 0xc002447e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00215c5d0}, {0x1f35540, 0xc002447e00}, 0x40, 0xc002447e80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215c5d0}, {0x1f35540, 0xc002447e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00215c5d0}, {0x1f35540, 0xc002447e00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215c5d0}, {0x1f35540, 0xc002447e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00215c5d0}, {0x1f35540, 0xc002447e00}, 0xc0019a4fa0, 0xc0017ee078)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc00215c5d0}, 0xc002c5b200, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc00213b440, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc00213b440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266107 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc002372b80)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 895 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada6550, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000af680, 0xc0014c8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000af680, {0xc0014c8000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000af680, {0xc0014c8000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000ab200, {0xc0014c8000, 0x4383e0, 0xc001084ea0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000c49e00, {0xc000828820, 0x9, 0xc0014bdd18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000c49e00}, {0xc000828820, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000828820, 0x9, 0x18}, {0x23585c0, 0xc000c49e00})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008287e0)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000bc20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 899 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a09db0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000e583c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 572 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016b200, {0x2392228, 0xc0010a4980})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 665 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc0007c8120)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a4b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a4b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1266085 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568000, 0xc002372d40)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1174604 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c0a10, {0x2392228, 0xc002576600})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1082925 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0028d1c20, {0x2392228, 0xc000e8e3c0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 561 [select, 13 minutes]:
go.temporal.io/server/service/frontend.(*VersionChecker).versionCheckLoop(0xc00056a5a0)
	/temporal/service/frontend/versionChecker.go:88 +0xe5
created by go.temporal.io/server/service/frontend.(*VersionChecker).Start.func1
	/temporal/service/frontend/versionChecker.go:70 +0x5f

goroutine 560 [select, 3 minutes]:
go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).purgeProcessor(0xc0000fc000)
	/temporal/common/persistence/namespaceReplicationQueue.go:404 +0xed
created by go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).Start
	/temporal/common/persistence/namespaceReplicationQueue.go:123 +0x71

goroutine 559 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016acc0, {0x2392228, 0xc0010a4f00})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 570 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 1282680 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005e50e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb600, {0x1f7b7c0, 0xc00156fcc0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7e00, 0xc001a1bbf8, 0xc001a1bbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7e00, {0x1f7b7c0, 0xc00156fcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e1c330}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0x20, {0xc001fa10c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e1c330}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e1c330}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0xc001c4fce0, {0xc001fa10c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e1c330}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0xc0023ec2f0, 0xc001e1c360, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e1c330}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e0ed40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0x237ac70, 0xc001e1c2a0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e0ed40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e0ed40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001e0ecc0}, {0x1f7b7c0, 0xc00156fcc0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001e0ed40}, 0xc00003f248, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001e0bbc0}, 0xc001e0ecc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001e0bbc0}, 0xc001e0ecc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001a1c480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001a1c528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ace2838, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001e0bbc0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00251a530})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001a1c778, {0x235a640, 0xc000045ad0}, 0x612d343563342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001e0bbc0}, 0xc001e0e9c0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc000704db8, 0x4c1cb4}, {0xc057031241b45c1a, 0xb3ad60a9a4}, {0xc000704db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001e0bbc0}, 0xc001e0e9c0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0}, 0xc001fa0f60, 0xc001e06db0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0}, 0x15, 0xc001e0ea00)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0}, 0x2076951, 0xc001e0ea00)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0}, 0x237bc38, 0xc001e0ea00)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0bbc0}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001e0ba70}, {0x1f33320, 0xc001e0e9c0}, 0x1e5a940, 0xc001e0ea00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0ba70}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0}, 0x10000000066f114, 0xc001e0ea00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0}, 0x0, 0xc00049d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002605b60, {0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e0ba40}, {0x1f33320, 0xc001e0e9c0}, 0xc001fa0f60, 0xc001e06db0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001e0ba40}, 0xc0024dc360, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001997560, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001997560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1318930 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000378480)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9290, {0x1f7b7c0, 0xc00204b720}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab800, 0xc002249bf8, 0xc002249bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab800, {0x1f7b7c0, 0xc00204b720})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e0af90}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0x20, {0xc001fa18c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e0af90}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e0af90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0xc001632b00, {0xc001fa18c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e0af90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0xc0017985b0, 0xc001e0afc0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e0af90}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001147700}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0x237ac70, 0xc001e0af00, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001147700}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001147700}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001147680}, {0x1f7b7c0, 0xc00204b720}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001147700}, 0xc00203cd00, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001e0a6c0}, 0xc001147680, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001e0a6c0}, 0xc001147680, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00224a480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00224a528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad307b8, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001e0a6c0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0027ae450})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00224a778, {0x235a640, 0xc000045ad0}, 0x622d353935342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001e0a6c0}, 0xc001147380)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc0025a4db8, 0x4c1cb4}, {0xc0570314c0371df6, 0xb5ffef4f7f}, {0xc0025a4db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001e0a6c0}, 0xc001147380)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380}, 0xc001fa1700, 0xc0025dbcf8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380}, 0x15, 0xc0011473c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380}, 0x2076951, 0xc0011473c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380}, 0x237bc38, 0xc0011473c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a6c0}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001e0a540}, {0x1f33320, 0xc001147380}, 0x1e5a940, 0xc0011473c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a540}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380}, 0xc00203cc80, 0xc0011473c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002471b60, {0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e0a510}, {0x1f33320, 0xc001147380}, 0xc001fa1700, 0xc0025dbcf8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001e0a510}, 0xc0028c5620, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001b68a20, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001b68a20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266138 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc001f90f00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1319107 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001b8e2a0, {0x2392298, 0xc002561aa0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001b8a6c0, {0x23922d0, 0xc001f1b530}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0024115f0, {0x23922d0, 0xc001f1b530}, 0xc001234280, 0x1, 0x1234290)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001f1b4a0, 0xc00117fa80)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001f1af30}, 0xc00117fa80)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001f1af30}, {0x1f35540, 0xc00117fa80})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001f1af30}, {0x1f35540, 0xc00117fa80}, 0x237bc38, 0xc001fb90f8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1af30}, {0x1f35540, 0xc00117fa80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001f1ad50}, {0x1f35540, 0xc00117fa80}, 0xc00117f2c0, 0xc00117fb00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1ad50}, {0x1f35540, 0xc00117fa80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f1ad50}, {0x1f35540, 0xc00117fa80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1ad50}, {0x1f35540, 0xc00117fa80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f1ad20}, {0x1f35540, 0xc00117fa80}, 0x40, 0xc00117fb00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1ad20}, {0x1f35540, 0xc00117fa80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f1ad20}, {0x1f35540, 0xc00117fa80}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1ad20}, {0x1f35540, 0xc00117fa80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f1ad20}, {0x1f35540, 0xc00117fa80}, 0xc001652320, 0xc001fb90f8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001f1ad20}, 0xc002561a40, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b846c0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b846c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282533 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024adf80, {0x2392298, 0xc002636840}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001082360, {0x23922d0, 0xc001d388d0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001d388d0}, 0xc001d36820, 0x1bd58c0, 0x1d36830)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001d38870, 0xc001d2f100)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001d385d0}, 0xc001d2f100)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001d385d0}, {0x1f353a0, 0xc001d2f100})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001d385d0}, {0x1f353a0, 0xc001d2f100}, 0x237bc38, 0xc002529de8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d385d0}, {0x1f353a0, 0xc001d2f100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001d38450, {0x23922d0, 0xc001d38480}, {0x1f353a0, 0xc001d2f100}, 0xc001d31770, 0xc001d2f140)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38480}, {0x1f353a0, 0xc001d2f100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d38480}, {0x1f353a0, 0xc001d2f100}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38480}, {0x1f353a0, 0xc001d2f100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d383f0}, {0x1f353a0, 0xc001d2f100}, 0x40, 0xc001d2f140)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d383f0}, {0x1f353a0, 0xc001d2f100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d383f0}, {0x1f353a0, 0xc001d2f100}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d383f0}, {0x1f353a0, 0xc001d2f100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d383f0}, {0x1f353a0, 0xc001d2f100}, 0xc0008e7be0, 0xc002529de8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001d383f0}, 0xc0026367e0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c11200, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c11200, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1272273 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0023c1f90)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1159252 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002160e40, {0x2392228, 0xc0024dbac0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1276940 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000676360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bab00, {0x1f7b560, 0xc00156e140}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6100, 0xc00187a440, 0xc00187a410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6100, {0x1f7b560, 0xc00156e140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001b74db0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0x20, {0xc0021d6f20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001b74db0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b74db0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0xc0020d6dc0, {0xc0021d6f20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b74db0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0xc0024e7e90, 0xc001b74de0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b74db0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001a73ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0x237ac70, 0xc001b74d20, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001a73ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001a73ec0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001a73e40}, {0x1f7b560, 0xc00156e140}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001a73ec0}, 0xc0015c8de6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0000872c0}, 0xc001a73e40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0000872c0}, 0xc001a73e40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0019e9c80, {0x2392298, 0xc0000872c0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0019e9ec0, {0x2392298, 0xc0000872c0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0023385a0, {0x23922d0, 0xc00212e870}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00212e870}, 0xc002122db0, 0x1bd58c0, 0x2122dc0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc00212e810, 0xc0020fdb40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc00212e570}, 0xc0020fdb40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00212e570}, {0x1f353a0, 0xc0020fdb40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc00212e570}, {0x1f353a0, 0xc0020fdb40}, 0x237bc38, 0xc002130120)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e570}, {0x1f353a0, 0xc0020fdb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00212e3f0, {0x23922d0, 0xc00212e420}, {0x1f353a0, 0xc0020fdb40}, 0xc002115710, 0xc0020fdb80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e420}, {0x1f353a0, 0xc0020fdb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00212e420}, {0x1f353a0, 0xc0020fdb40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e420}, {0x1f353a0, 0xc0020fdb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00212e390}, {0x1f353a0, 0xc0020fdb40}, 0x40, 0xc0020fdb80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e390}, {0x1f353a0, 0xc0020fdb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00212e390}, {0x1f353a0, 0xc0020fdb40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e390}, {0x1f353a0, 0xc0020fdb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00212e390}, {0x1f353a0, 0xc0020fdb40}, 0xc0021d62e0, 0xc002130120)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc00212e390}, 0xc0000871a0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001a12900, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001a12900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1358740 [runnable]:
github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent·dwrap·3()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1
created by github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133 +0x13a

goroutine 1266184 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0dc0, 0x8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 847 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc001bb87d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 845 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc000a92240, {0x2392228, 0xc000c3fb40})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 844 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000a92240, {0x2392228, 0xc000c3fb40})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 843 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c8460, {0x2392228, 0xc000c3fa80})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1266149 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc000e30cb8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286345 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00286ab40, {0x2392298, 0xc00286aba0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0017aefc0, {0x23922d0, 0xc002077b00}, 0x2a)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0023b3320, {0x23922d0, 0xc002077b00}, 0xc001d69730, 0x1, 0x1d69740)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc0020778c0, 0xc001d721c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc002077500}, 0xc001d721c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002077500}, {0x1f35540, 0xc001d721c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002077500}, {0x1f35540, 0xc001d721c0}, 0x237bc38, 0xc0020c05e8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077500}, {0x1f35540, 0xc001d721c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0020771d0}, {0x1f35540, 0xc001d721c0}, 0xc001d5f500, 0xc001d72240)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020771d0}, {0x1f35540, 0xc001d721c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020771d0}, {0x1f35540, 0xc001d721c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020771d0}, {0x1f35540, 0xc001d721c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020771a0}, {0x1f35540, 0xc001d721c0}, 0x40, 0xc001d72240)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020771a0}, {0x1f35540, 0xc001d721c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020771a0}, {0x1f35540, 0xc001d721c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020771a0}, {0x1f35540, 0xc001d721c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020771a0}, {0x1f35540, 0xc001d721c0}, 0xc001fa0aa0, 0xc0020c05e8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0020771a0}, 0xc00286a7e0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0024919e0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0024919e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266129 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc002696dc8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1282228 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005d4a20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060f4a0, {0x1f7c860, 0xc00060f290}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084ba00, 0xc0008a55e8, 0xc0008a55b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084ba00, {0x1f7c860, 0xc00060f290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001c575c0}, {0x20768da, 0x8}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x1c75901, {0xc0008e7300, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x7fb451d0c108, {0xc0008e7300, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x20, 0xc001bbcc60, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0xc0007c0000, {0xc0019ffa80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x0, 0xc001bbcc30, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x1e0cfc0, 0xc001bbcba0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x60, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001c575c0}, {0x20768da, 0x203000}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x8, 0xc001bbcb70, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001c575c0}, {0x20768da, 0x46}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, 0x6, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc001c575c0}, {0x20768da, 0x0}, {0x1f33180, 0xc001bbcb10}, {0x1f7c860, 0xc00060f290}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc001c575c0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc001c575c0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 871 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 756 [select, 13 minutes]:
net/http.(*persistConn).writeLoop(0xc00123f200)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 755 [select, 13 minutes]:
net/http.(*persistConn).readLoop(0xc00123f200)
	/usr/local/go/src/net/http/transport.go:2207 +0xd8a
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 1170555 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00213cde0, {0x2392228, 0xc001dd8f40})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 868 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc001785950)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 867 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc0027cab40)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 866 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc002163ca0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1266182 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0dc0, 0x8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 865 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc001f07630)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 666 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125a4b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 836 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00056a9b0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0008a6c00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 842 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc00056b810)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 838 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00056ab40, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0008a6cc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 839 [select, 13 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00084fb00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1277075 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000622e00, {0x2392228, 0xc00217fac0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1174606 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002572360, {0x2392228, 0xc002576640})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 900 [select, 13 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000ad2d80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 901 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef1008, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000afc80, 0xc0014e0000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000afc80, {0xc0014e0000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000afc80, {0xc0014e0000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000ab208, {0xc0014e0000, 0xc0014becf8, 0x44d2f4})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000e58360, {0xc000828900, 0x9, 0xb428447a39})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000e58360}, {0xc000828900, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000828900, 0x9, 0xc000a09e00}, {0x23585c0, 0xc000e58360})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008288c0)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000ad2d80, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc00002d340, {0x23d1088, 0xc000ad2d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1318926 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123e900)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9130, {0x1f7b7c0, 0xc00204b180}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab000, 0xc00284dbf8, 0xc00284dbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab000, {0x1f7b7c0, 0xc00204b180})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d393b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0x20, {0xc001fa0ee0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d393b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d393b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0xc00210e6e0, {0xc001fa0ee0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d393b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0xc0017985b0, 0xc001d393e0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d393b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001146b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0x237ac70, 0xc001d39320, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001146b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001146b00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001146a80}, {0x1f7b7c0, 0xc00204b180}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001146b00}, 0xc00203cb40, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001d38cc0}, 0xc001146a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001d38cc0}, 0xc001146a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00284e480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00284e528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad307b8, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001d38cc0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0027ae210})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00284e778, {0x235a640, 0xc000045ad0}, 0x382d313435342d31)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001d38cc0}, 0xc001146780)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a44db8, 0x4c1cb4}, {0xc0570314c02d645c, 0xb5ffe595d9}, {0xc001a44db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001d38cc0}, 0xc001146780)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780}, 0xc001fa0c80, 0xc0025db3e0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780}, 0x15, 0xc0011467c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780}, 0x2076951, 0xc0011467c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780}, 0x237bc38, 0xc0011467c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38cc0}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001d38b70}, {0x1f33320, 0xc001146780}, 0x1e5a940, 0xc0011467c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38b70}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780}, 0x23c27b0, 0xc0011467c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002471b60, {0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d38b40}, {0x1f33320, 0xc001146780}, 0xc001fa0c80, 0xc0025db3e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001d38b40}, 0xc0028c4de0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000c1e000, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000c1e000, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266201 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc000de6d20)
	/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
	/temporal/service/history/queueProcessor.go:158 +0x23b

goroutine 1286306 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001082fc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060edc0, {0x1f7b7c0, 0xc00156eaa0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6d00, 0xc0025484d0, 0xc0025484a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6d00, {0x1f7b7c0, 0xc00156eaa0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002812ba0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0x20, {0xc0008e6200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002812ba0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002812ba0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0xc002096420, {0xc0008e6200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002812ba0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0xc0021b2800, 0xc002812bd0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002812ba0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d46bc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0x237ac70, 0xc002812b10, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d46bc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d46bc0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001d46b00}, {0x1f7b7c0, 0xc00156eaa0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001d46bc0}, 0xc0022d3806, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000ab1ce0, {0x2392298, 0xc001b67020}, 0xc001d46b00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000799920, {0x2392298, 0xc001b67020}, 0xc001d46b00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001b66d80, {0x2392298, 0xc001b67020})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001b66fc0, {0x2392298, 0xc001b67020}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001082900, {0x23922d0, 0xc002812000}, 0x1e)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0022d37a0, {0x23922d0, 0xc002812000}, 0xc001d36170, 0x1, 0x1d36180)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001853f80, 0xc001d46200)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001853ce0}, 0xc001d46200)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001853ce0}, {0x1f35540, 0xc001d46200})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001853ce0}, {0x1f35540, 0xc001d46200}, 0x237bc38, 0xc0022af980)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001853ce0}, {0x1f35540, 0xc001d46200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001853b90}, {0x1f35540, 0xc001d46200}, 0xc001d2fc40, 0xc001d46280)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001853b90}, {0x1f35540, 0xc001d46200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001853b90}, {0x1f35540, 0xc001d46200}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001853b90}, {0x1f35540, 0xc001d46200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001853b60}, {0x1f35540, 0xc001d46200}, 0x40, 0xc001d46280)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001853b60}, {0x1f35540, 0xc001d46200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001853b60}, {0x1f35540, 0xc001d46200}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001853b60}, {0x1f35540, 0xc001d46200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001853b60}, {0x1f35540, 0xc001d46200}, 0xc0008e60e0, 0xc0022af980)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001853b60}, 0xc001b669c0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000645560, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000645560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286338 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0028f9140, {0x2392228, 0xc001d5e780})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1319069 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017ddce0, {0x2392298, 0xc00174f800}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a127e0, {0x23922d0, 0xc001e82330}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0006cd260, {0x23922d0, 0xc001e82330}, 0xc0011be080, 0x1, 0x11be090)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001e822d0, 0xc001195480)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001e82060}, 0xc001195480)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e82060}, {0x1f35540, 0xc001195480})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e82060}, {0x1f35540, 0xc001195480}, 0x237bc38, 0xc0020ef920)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e82060}, {0x1f35540, 0xc001195480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e99ef0}, {0x1f35540, 0xc001195480}, 0xc001194f00, 0xc001195500)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99ef0}, {0x1f35540, 0xc001195480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e99ef0}, {0x1f35540, 0xc001195480}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99ef0}, {0x1f35540, 0xc001195480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e99ec0}, {0x1f35540, 0xc001195480}, 0x40, 0xc001195500)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99ec0}, {0x1f35540, 0xc001195480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e99ec0}, {0x1f35540, 0xc001195480}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99ec0}, {0x1f35540, 0xc001195480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e99ec0}, {0x1f35540, 0xc001195480}, 0xc000799f80, 0xc0020ef920)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e99ec0}, 0xc00174f7a0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0023bcb40, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0023bcb40, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286378 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0025275c0, {0x2392298, 0xc0024ad080}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001b859e0, {0x23922d0, 0xc002187740}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002187740}, 0xc001d7e8a0, 0x1bd58c0, 0x1d7e8b0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0021876e0, 0xc001dada00)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc002187440}, 0xc001dada00)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002187440}, {0x1f353a0, 0xc001dada00})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002187440}, {0x1f353a0, 0xc001dada00}, 0x237bc38, 0xc0025e4780)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002187440}, {0x1f353a0, 0xc001dada00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0021872c0, {0x23922d0, 0xc0021872f0}, {0x1f353a0, 0xc001dada00}, 0x2392298, 0xc001dada40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021872f0}, {0x1f353a0, 0xc001dada00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021872f0}, {0x1f353a0, 0xc001dada00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021872f0}, {0x1f353a0, 0xc001dada00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002187260}, {0x1f353a0, 0xc001dada00}, 0x40, 0xc001dada40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002187260}, {0x1f353a0, 0xc001dada00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002187260}, {0x1f353a0, 0xc001dada00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002187260}, {0x1f353a0, 0xc001dada00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002187260}, {0x1f353a0, 0xc001dada00}, 0xc0007988a0, 0xc0025e4780)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc002187260}, 0xc0024acfc0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b85440, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b85440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1201993 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c09200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1266205 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0x8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1285976 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0017679e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bad10, {0x1f7b560, 0xc001980c80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000735600, 0xc002359d18, 0xc002359ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000735600, {0x1f7b560, 0xc001980c80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00257b3e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0x20, {0xc0019a5ba0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00257b3e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00257b3e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0xc0019f3b80, {0xc0019a5ba0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00257b3e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0xc0023ec2f0, 0xc00257b410, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00257b3e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021e6c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0x237ac70, 0xc00257b230, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021e6c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021e6c40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0021e6bc0}, {0x1f7b560, 0xc001980c80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc0021e6c40}, 0xc002415650, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc00257a840}, 0xc0021e6bc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc00257a840}, 0xc0021e6bc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00235a5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00235a648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42aceb488, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc00257a840}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001ce6070})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00235a898, {0x235a640, 0xc000045ad0}, 0x392d303638342d31)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc00257a840}, 0xc00257a1e0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001662db8, 0x4c1cb4}, {0xc057031278b9c619, 0xb3e466139c}, {0xc001662db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc00257a840}, 0xc00257a1e0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0}, 0xc0019a5a80, 0xc002590e40)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0}, 0x15, 0xc0021e6980)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0}, 0x207690b, 0xc0021e6980)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0}, 0x237bc38, 0xc0021e6980)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a840}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00257a240}, {0x1f33180, 0xc00257a1e0}, 0x1e5a940, 0xc0021e6980)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a240}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0}, 0x9, 0xc0021e6980)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0}, 0xc0025cc768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0025ccb60, {0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00257a1b0}, {0x1f33180, 0xc00257a1e0}, 0xc0019a5a80, 0xc002590e40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc00257a1b0}, 0xc000fe9f20, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001766480, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001766480, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266079 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568000, 0xc00185f8d0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266118 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0790, 0xc000a0b440)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286353 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008cfc20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f130, {0x1f7b560, 0xc001a55e00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa500, 0xc0019ddd18, 0xc0019ddce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa500, {0x1f7b560, 0xc001a55e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002115f80}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0x20, {0xc001fa1000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002115f80}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002115f80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0xc000a2f4a0, {0xc001fa1000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002115f80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0xc0017985b0, 0xc002115fb0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002115f80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d73280}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0x237ac70, 0xc002115ec0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d73280}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d73280}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001d73200}, {0x1f7b560, 0xc001a55e00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001d73280}, 0xc0023b3740, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc002115590}, 0xc001d73200, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc002115590}, 0xc001d73200, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0019de5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0019de648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acdfb58, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc002115590}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001798ce0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0019de898, {0x235a640, 0xc000045ad0}, 0x612d626466342d38)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc002115590}, 0xc0021152c0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00263edb8, 0x4c1cb4}, {0xc057031283de5270, 0xb3eb2569ed}, {0xc00263edb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc002115590}, 0xc0021152c0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0}, 0xc001fa0ee0, 0xc0020c0ea0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0}, 0x15, 0xc001d72fc0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0}, 0x207690b, 0xc001d72fc0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0}, 0x237bc38, 0xc001d72fc0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115590}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002115410}, {0x1f33180, 0xc0021152c0}, 0x1e5a940, 0xc001d72fc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115410}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0}, 0x9, 0xc001d72fc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0}, 0xc00260c768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00260cb60, {0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002115290}, {0x1f33180, 0xc0021152c0}, 0xc001fa0ee0, 0xc0020c0ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002115290}, 0xc00286b6e0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0008ce120, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0008ce120, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266146 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc00177a0f0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1276936 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b699e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205fb80, {0x1f7c860, 0xc00205fad0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab700, 0xc00151f5e8, 0xc00151f5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab700, {0x1f7c860, 0xc00205fad0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0024d1b00}, {0x20768da, 0x8}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x1c75901, {0xc0021d6040, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x7fb451d0c108, {0xc0021d6040, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x20, 0xc002115260, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0xc0007c0000, {0xc0020fd400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x0, 0xc002115230, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x1e0cfc0, 0xc0021151a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0024d1b00}, {0x20768da, 0x203000}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x8, 0xc002115170, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0024d1b00}, {0x20768da, 0x46}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, 0x2, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc0024d1b00}, {0x20768da, 0x0}, {0x1f33180, 0xc002115110}, {0x1f7c860, 0xc00205fad0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0024d1b00}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc0024d1b00})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 926 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0008dbef0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 927 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e57c70, 0xc000f53280)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e57c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 928 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e57c70, 0xc001d375b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e57c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 945 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e57c70, 0xc00100d450)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e57c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 946 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e57c70, 0xc001dd5340)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e57c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 947 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0008dbe00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1272276 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00230c000, {0x2392228, 0xc0023d4380})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1282654 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000622af0, {0x2392228, 0xc001e0e000})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1318796 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005e5c20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce8d10, {0x1f736c0, 0xc000f4f4d0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aaa00, 0xc0015595c0, 0xc001559590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aaa00, {0x1f736c0, 0xc000f4f4d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001cc6600}, {0x2076920, 0x8}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x1c75901, {0xc0021d7e00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x7fb451d0c108, {0xc0021d7e00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x20, 0xc001c52630, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0xc0006ca340, {0xc000f74040, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x0, 0xc001c52600, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x41774b, 0xc001c52570, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x4175cd, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001cc6600}, {0x2076920, 0x203000}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x203000, 0xc001c52540, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001cc6600}, {0x2076920, 0x46}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc001cc6600}, {0x2076920, 0x200dd96d0}, {0x1f33320, 0xc000f74000}, {0x1f736c0, 0xc000f4f4d0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001cc6600}, 0x1fe4e76, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e57c70, {0x2392298, 0xc001cc6600})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1011 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c9180, {0x2392228, 0xc0008d9a80})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1012 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0015ba1e0, {0x2392228, 0xc0008d9ac0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1013 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0015ba1e0, {0x2392228, 0xc0008d9ac0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1290277 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0024cb200, {0x2392228, 0xc00101dd80})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1282556 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001960360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb290, {0x1f7c860, 0xc0022bb1e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7600, 0xc0015d95e8, 0xc0015d95b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7600, {0x1f7c860, 0xc0022bb1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00250b860}, {0x20768da, 0x8}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x1c75901, {0xc0021d7d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x7fb451d0c5b8, {0xc0021d7d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x20, 0xc001d987e0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0xc0007c0000, {0xc001d930c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x0, 0xc001d987b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x1e0cfc0, 0xc001d98720, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x60, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00250b860}, {0x20768da, 0x203000}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x8, 0xc001d986f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00250b860}, {0x20768da, 0x46}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc00250b860}, {0x20768da, 0x0}, {0x1f33180, 0xc001d98690}, {0x1f7c860, 0xc0022bb1e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc00250b860}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00076c300, {0x2392298, 0xc00250b860})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1062 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1061 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001615ad0, 0xc001160110)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001615ad0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1060 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001615ad0, 0xc00206b1c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001615ad0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1059 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001615ad0, 0xc001161650)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001615ad0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1058 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001615ad0, 0xc001d17950)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001615ad0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125ae10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1057 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125af00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1266211 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0xc002422180)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1318927 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123fb00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce91e0, {0x1f736c0, 0xc00115e120}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab600, 0xc0015f15c0, 0xc0015f1590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab600, {0x1f736c0, 0xc00115e120})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0028c5140}, {0x2076920, 0x8}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x1c75901, {0xc001fa13c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x7fb451d0c108, {0xc001fa13c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x20, 0xc001d399e0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0xc0006ca340, {0xc001146e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x0, 0xc001d399b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x0, 0xc001d39920, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0028c5140}, {0x2076920, 0x203000}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x203000, 0xc001d398f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0028c5140}, {0x2076920, 0x46}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc0028c5140}, {0x2076920, 0x200000000}, {0x1f33320, 0xc001146e00}, {0x1f736c0, 0xc00115e120}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0028c5140}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001615ad0, {0x2392298, 0xc0028c5140})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1201990 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c09080)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1318772 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000402c60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001838a50, {0x1f736c0, 0xc00101ab40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6200, 0xc0015915c0, 0xc001591590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6200, {0x1f736c0, 0xc00101ab40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001cc4c60}, {0x2076920, 0x8}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x1c75901, {0xc00247cfa0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x7fb451d0c5b8, {0xc00247cfa0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x20, 0xc00223f170, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0xc0006ca340, {0xc001034080, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x0, 0xc00223f140, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x0, 0xc00223f0b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001cc4c60}, {0x2076920, 0x203000}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x203000, 0xc00223f080, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001cc4c60}, {0x2076920, 0x46}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc00186be50, {0x2392298, 0xc001cc4c60}, {0x2076920, 0x201001018}, {0x1f33320, 0xc001034040}, {0x1f736c0, 0xc00101ab40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001cc4c60}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e57c70, {0x2392298, 0xc001cc4c60})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1282748 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023cb4a0, {0x2392298, 0xc001c5a660}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a12b40, {0x23922d0, 0xc001e3eff0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001e3eff0}, 0xc001e3aa00, 0x1bd58c0, 0x1e3aa10)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001e3ede0, 0xc001e2bac0)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001e3eb40}, 0xc001e2bac0)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e3eb40}, {0x1f353a0, 0xc001e2bac0})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e3eb40}, {0x1f353a0, 0xc001e2bac0}, 0x237bc38, 0xc001e30d08)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3eb40}, {0x1f353a0, 0xc001e2bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e3e9f0}, {0x1f353a0, 0xc001e2bac0}, 0xc001e0e9c0, 0xc001e2bb00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e9f0}, {0x1f353a0, 0xc001e2bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e3e9f0}, {0x1f353a0, 0xc001e2bac0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e9f0}, {0x1f353a0, 0xc001e2bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e3e990}, {0x1f353a0, 0xc001e2bac0}, 0x40, 0xc001e2bb00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e990}, {0x1f353a0, 0xc001e2bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e3e990}, {0x1f353a0, 0xc001e2bac0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e990}, {0x1f353a0, 0xc001e2bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e3e990}, {0x1f353a0, 0xc001e2bac0}, 0xc001fa1c20, 0xc001e30d08)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e3e990}, 0xc001c5a600, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0024907e0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0024907e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1010 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0015b8050)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1277077 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001ba47e0, {0x2392228, 0xc00217fb00})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286278 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00110ff80, {0x2392298, 0xc001b66000}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001960000, {0x23922d0, 0xc001601620}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0022d3110, {0x23922d0, 0xc001601620}, 0xc001d16df0, 0x1, 0x1d16e00)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc0016015c0, 0xc001d2e500)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001601320}, 0xc001d2e500)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001601320}, {0x1f35540, 0xc001d2e500})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001601320}, {0x1f35540, 0xc001d2e500}, 0x237bc38, 0xc0022ae7e0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001601320}, {0x1f35540, 0xc001d2e500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0016011d0}, {0x1f35540, 0xc001d2e500}, 0xc001d93f80, 0xc001d2e580)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016011d0}, {0x1f35540, 0xc001d2e500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0016011d0}, {0x1f35540, 0xc001d2e500}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016011d0}, {0x1f35540, 0xc001d2e500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0016011a0}, {0x1f35540, 0xc001d2e500}, 0x40, 0xc001d2e580)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016011a0}, {0x1f35540, 0xc001d2e500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0016011a0}, {0x1f35540, 0xc001d2e500}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016011a0}, {0x1f35540, 0xc001d2e500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0016011a0}, {0x1f35540, 0xc001d2e500}, 0xc0021d7900, 0xc0022ae7e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0016011a0}, 0xc00110fbc0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000668a20, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000668a20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286344 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0024918c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205ee70, {0x1f7b7c0, 0xc001a557c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa000, 0xc00298c4d0, 0xc00298c4a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa000, {0x1f7b7c0, 0xc001a557c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002076db0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0x20, {0xc001fa0980, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002076db0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002076db0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0xc000eb26e0, {0xc001fa0980, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002076db0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0xc0021b2800, 0xc002076de0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002076db0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d5ffc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0x237ac70, 0xc002076d20, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d5ffc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d5ffc0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001d5ff00}, {0x1f7b7c0, 0xc001a557c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001d5ffc0}, 0xc002304546, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000ab1ce0, {0x2392298, 0xc00286a480}, 0xc001d5ff00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000799920, {0x2392298, 0xc00286a480}, 0xc001d5ff00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002092ae0, {0x2392298, 0xc00286a480})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002092d80, {0x2392298, 0xc00286a480}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001b69440, {0x23922d0, 0xc002076b10}, 0x2a)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0023b31a0, {0x23922d0, 0xc002076b10}, 0xc001d691f0, 0x1, 0x1d69200)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc002076ab0, 0xc001d5fcc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc002076810}, 0xc001d5fcc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002076810}, {0x1f35540, 0xc001d5fcc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002076810}, {0x1f35540, 0xc001d5fcc0}, 0x237bc38, 0xc0020c02b8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002076810}, {0x1f35540, 0xc001d5fcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0020766c0}, {0x1f35540, 0xc001d5fcc0}, 0xc001d5f500, 0xc001d5fd40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020766c0}, {0x1f35540, 0xc001d5fcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020766c0}, {0x1f35540, 0xc001d5fcc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020766c0}, {0x1f35540, 0xc001d5fcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002076690}, {0x1f35540, 0xc001d5fcc0}, 0x40, 0xc001d5fd40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002076690}, {0x1f35540, 0xc001d5fcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002076690}, {0x1f35540, 0xc001d5fcc0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002076690}, {0x1f35540, 0xc001d5fcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002076690}, {0x1f35540, 0xc001d5fcc0}, 0xc001fa08e0, 0xc0020c02b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc002076690}, 0xc00286a420, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024905a0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024905a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1164328 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002092a20, {0x2392228, 0xc000ea0bc0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266053 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69d90, 0xc000dd3cb8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286346 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000df9a40)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1286287 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019610e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000542840, {0x1f736c0, 0xc000dac6c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x69)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084bf00, 0xc001c0f5c0, 0xc001c0f590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084bf00, {0x1f736c0, 0xc000dac6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001b66540}, {0x2076920, 0x8}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x1c75901, {0xc0021d7c80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x7fb451d0c5b8, {0xc0021d7c80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x20, 0xc001852ba0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0xc0006ca340, {0xc001d2fa40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x0, 0xc001852b70, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x1e0cfc0, 0xc001852ae0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0xc001d85488, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001b66540}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x203000, 0xc001852ab0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001b66540}, {0x2076920, 0x46}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001d85650, {0x2392298, 0xc001b66540}, {0x2076920, 0x101d85658}, {0x1f33320, 0xc001d2fa00}, {0x1f736c0, 0xc000dac6c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001b66540}, 0xc001d2f2c0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001615ad0, {0x2392298, 0xc001b66540})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1286276 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000c10480)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bbef0, {0x1f736c0, 0xc000d89ef0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084bc00, 0xc0019795c0, 0xc001979590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084bc00, {0x1f736c0, 0xc000d89ef0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00110f740}, {0x2076920, 0x8}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x1c75901, {0xc0021d6e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x7fb451d0c5b8, {0xc0021d6e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x20, 0xc00185bd10, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0xc0006ca340, {0xc001d93b40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x0, 0xc00185bce0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x1e0cfc0, 0xc00185bc50, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0xc000ec7c88, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00110f740}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x203000, 0xc00185bc20, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00110f740}, {0x2076920, 0x46}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000ec7e50, {0x2392298, 0xc00110f740}, {0x2076920, 0x100ec7e58}, {0x1f33320, 0xc001d93a80}, {0x1f736c0, 0xc000d89ef0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc00110f740}, 0xc002232780, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc00110f740})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1286329 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002217320, {0x2392228, 0xc001e0e600})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266105 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc00185eb28)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266212 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0x0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266088 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1276927 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002491440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f970, {0x1f7b560, 0xc001981540}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab500, 0xc00292a440, 0xc00292a410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab500, {0x1f7b560, 0xc001981540})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0020fb410}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0x20, {0xc00247dc60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0020fb410}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fb410}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0xc001633a20, {0xc00247dc60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fb410}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0xc0024e7e90, 0xc0020fb440, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fb410}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0020fc200}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0x237ac70, 0xc0020fb380, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0020fc200}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0020fc200}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0020fc180}, {0x1f7b560, 0xc001981540}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc0020fc200}, 0xc0007d8606, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0024d0ba0}, 0xc0020fc180, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0024d0ba0}, 0xc0020fc180, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0017bd8c0, {0x2392298, 0xc0024d0ba0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017bdc20, {0x2392298, 0xc0024d0ba0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005d5560, {0x23922d0, 0xc0020fb080}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0020fb080}, 0xc0020f1440, 0x1bd58c0, 0x20f1450)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0020fb020, 0xc0020e1f80)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0020fad80}, 0xc0020e1f80)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020fad80}, {0x1f353a0, 0xc0020e1f80})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0020fad80}, {0x1f353a0, 0xc0020e1f80}, 0x237bc38, 0xc0020eef30)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fad80}, {0x1f353a0, 0xc0020e1f80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0020fac00, {0x23922d0, 0xc0020fac30}, {0x1f353a0, 0xc0020e1f80}, 0xc0020eddd0, 0xc0020e1fc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fac30}, {0x1f353a0, 0xc0020e1f80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fac30}, {0x1f353a0, 0xc0020e1f80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fac30}, {0x1f353a0, 0xc0020e1f80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020faba0}, {0x1f353a0, 0xc0020e1f80}, 0x40, 0xc0020e1fc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020faba0}, {0x1f353a0, 0xc0020e1f80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020faba0}, {0x1f353a0, 0xc0020e1f80}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020faba0}, {0x1f353a0, 0xc0020e1f80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020faba0}, {0x1f353a0, 0xc0020e1f80}, 0xc00247db80, 0xc0020eef30)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0020faba0}, 0xc0024d0a20, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc002490360, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc002490360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1002168 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada6ac0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0008f7300, 0xc00098a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0008f7300, {0xc00098a000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0008f7300, {0xc00098a000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00243bc40, {0xc00098a000, 0xc001b4ccf8, 0x44d2f4})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002391320, {0xc0003284a0, 0x9, 0xb602a45275})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc002391320}, {0xc0003284a0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0003284a0, 0x9, 0xc0009a2050}, {0x23585c0, 0xc002391320})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328460)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000c08d80, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000750380, {0x23d1088, 0xc000c08d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1286002 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006227e0, {0x2392228, 0xc002232780})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1276931 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0024d1020, {0x2392228, 0xc0020fcc40})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286337 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006235e0, {0x2392228, 0xc001d5e740})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1159249 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000d294a0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1266144 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc00101cf00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266074 [select]:
go.temporal.io/server/service/history.(*timerQueueProcessorBase).internalProcessor(0xc00127af00)
	/temporal/service/history/timerQueueProcessorBase.go:271 +0x37f
go.temporal.io/server/service/history.(*timerQueueProcessorBase).processorPump(0xc00127af00)
	/temporal/service/history/timerQueueProcessorBase.go:196 +0x9e
created by go.temporal.io/server/service/history.(*timerQueueProcessorBase).Start
	/temporal/service/history/timerQueueProcessorBase.go:163 +0xaf

goroutine 1028 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00076c300, 0xc0020f0ad0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00076c300)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1029 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00076c300, 0xc001da4f00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00076c300)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1030 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00076c300, 0xc001d91440)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00076c300)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1031 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00076c300, 0xc001d7f130)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00076c300)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1032 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00076b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1276922 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000127680)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f810, {0x1f7c860, 0xc00205f760}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab300, 0xc00161f5e8, 0xc00161f5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab300, {0x1f7c860, 0xc00205f760})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0028a7da0}, {0x20768da, 0x8}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x1c75901, {0xc00247d800, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x7fb451d0c108, {0xc00247d800, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x20, 0xc0020ed8c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0xc0007c0000, {0xc0020e16c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x0, 0xc0020ed890, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x1e0cfc0, 0xc0020ed800, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0028a7da0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x8, 0xc0020ed7d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0028a7da0}, {0x20768da, 0x46}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc0028a7da0}, {0x20768da, 0x0}, {0x1f33180, 0xc0020ed770}, {0x1f7c860, 0xc00205f760}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0028a7da0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00076c300, {0x2392298, 0xc0028a7da0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1266083 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568000, 0xc002372d40)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1056 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc0008e78e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125af00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125af00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1286358 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00286bb00, {0x2392228, 0xc001d73e40})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266140 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc001f90f00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1282230 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001668fc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060f810, {0x1f7b560, 0xc001a545a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968300, 0xc001551d18, 0xc001551ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968300, {0x1f7b560, 0xc001a545a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001bbd920}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0x20, {0xc0008e7580, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001bbd920}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001bbd920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0xc002096000, {0xc0008e7580, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001bbd920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0xc0023ec2f0, 0xc001bbd950, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001bbd920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0019fff40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0x237ac70, 0xc001bbd890, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0019fff40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0019fff40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0019ffec0}, {0x1f7b560, 0xc001a545a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc0019fff40}, 0xc001eaab70, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001bbd1d0}, 0xc0019ffec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001bbd1d0}, 0xc0019ffec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0015525a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001552648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acf0400, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001bbd1d0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0023605d0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001552898, {0x235a640, 0xc000045ad0}, 0x382d353265342d34)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001bbd1d0}, 0xc001bbd020)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00197adb8, 0x4c1cb4}, {0xc0570312388dbc13, 0xb3a89f3f92}, {0xc00197adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001bbd1d0}, 0xc001bbd020)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020}, 0xc0008e7480, 0xc002213b18)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020}, 0x15, 0xc0019ffc80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020}, 0x207690b, 0xc0019ffc80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020}, 0x237bc38, 0xc0019ffc80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbd1d0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001bbd080}, {0x1f33180, 0xc001bbd020}, 0x1e5a940, 0xc0019ffc80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbd080}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020}, 0x9, 0xc0019ffc80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020}, 0xc002603768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002603b60, {0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001bbcff0}, {0x1f33180, 0xc001bbd020}, 0xc0008e7480, 0xc002213b18)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001bbcff0}, 0xc001c57860, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0005d4ea0, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0005d4ea0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286328 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0006233b0, {0x2392228, 0xc001e0e5c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1286348 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00286a9c0, {0x2392228, 0xc001d729c0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286315 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000c12fc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060f130, {0x1f7b7c0, 0xc00156f180}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb5)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee6100, 0xc002589bf8, 0xc002589bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee6100, {0x1f7b7c0, 0xc00156f180})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0017fee10}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0x20, {0xc0008e68a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0017fee10}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0017fee10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0xc00196f1e0, {0xc0008e68a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0017fee10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0xc0023ec2f0, 0xc0017fee40, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0017fee10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d47a00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0x237ac70, 0xc0017fed80, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d47a00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d47a00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001d47980}, {0x1f7b7c0, 0xc00156f180}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001d47a00}, 0xc00003f398, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0017fe6c0}, 0xc001d47980, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0017fe6c0}, 0xc001d47980, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00258a480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00258a528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acf1830, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0017fe6c0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001da9cb0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00258a778, {0x235a640, 0xc000045ad0}, 0x392d306638342d66)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0017fe6c0}, 0xc001d47680)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001762db8, 0x4c1cb4}, {0xc05703128195a713, 0xb3e8dcbe90}, {0xc001762db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0017fe6c0}, 0xc001d47680)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680}, 0xc0008e67a0, 0xc00268ca68)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680}, 0x15, 0xc001d476c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680}, 0x2076951, 0xc001d476c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680}, 0x237bc38, 0xc001d476c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe6c0}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0017fe570}, {0x1f33320, 0xc001d47680}, 0x1e5a940, 0xc001d476c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe570}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680}, 0x23c27b0, 0xc001d476c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00260fb60, {0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0017fe540}, {0x1f33320, 0xc001d47680}, 0xc0008e67a0, 0xc00268ca68)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0017fe540}, 0xc001b67c20, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002339440, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002339440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1069 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0015f4a50)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1070 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c93b0, {0x2392228, 0xc0015e7cc0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1071 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00174e000, {0x2392228, 0xc0015e7d00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1072 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00174e000, {0x2392228, 0xc0015e7d00})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1276928 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024d1380, {0x2392298, 0xc0024d15c0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001b68b40, {0x23922d0, 0xc0020fbdd0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0020fbdd0}, 0xc0020f1a70, 0x1bd58c0, 0x20f1a80)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0020fbbc0, 0xc0020fc480)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0020fb920}, 0xc0020fc480)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020fb920}, {0x1f353a0, 0xc0020fc480})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0020fb920}, {0x1f353a0, 0xc0020fc480}, 0x237bc38, 0xc0020ef290)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fb920}, {0x1f353a0, 0xc0020fc480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0020fac00, {0x23922d0, 0xc0020fb7d0}, {0x1f353a0, 0xc0020fc480}, 0xc0020eddd0, 0xc0020fc4c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fb7d0}, {0x1f353a0, 0xc0020fc480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fb7d0}, {0x1f353a0, 0xc0020fc480}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fb7d0}, {0x1f353a0, 0xc0020fc480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fb770}, {0x1f353a0, 0xc0020fc480}, 0x40, 0xc0020fc4c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fb770}, {0x1f353a0, 0xc0020fc480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fb770}, {0x1f353a0, 0xc0020fc480}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fb770}, {0x1f353a0, 0xc0020fc480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fb770}, {0x1f353a0, 0xc0020fc480}, 0xc00247dd40, 0xc0020ef290)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0020fb770}, 0xc0024d0de0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc002491560, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc002491560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266104 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc00185eb28)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266103 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc00185eb28)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1279629 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1201977 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0025f0870, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001c234a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1358742 [runnable]:
github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent·dwrap·3()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1
created by github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133 +0x13a

goroutine 1282733 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00123efc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb4b00, {0x1f7c860, 0xc001cb4a50}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968b00, 0xc00175f5e8, 0xc00175f5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968b00, {0x1f7c860, 0xc001cb4a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0008a87e0}, {0x20768da, 0x8}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x1c75901, {0xc0019a4400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x7fb451d0c108, {0xc0019a4400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x20, 0xc001e835c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0xc0007c0000, {0xc001d73a80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x0, 0xc001e83590, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x1e0cfc0, 0xc001e83500, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0008a87e0}, {0x20768da, 0x203000}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x8, 0xc001e834d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0008a87e0}, {0x20768da, 0x46}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc0008a87e0}, {0x20768da, 0x0}, {0x1f33180, 0xc001e83470}, {0x1f7c860, 0xc001cb4a50}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0008a87e0}, 0xc000dd5f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00076c300, {0x2392298, 0xc0008a87e0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1318774 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0017665a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001838b00, {0x1f7b7c0, 0xc000020c80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6800, 0xc00163dbf8, 0xc00163dbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6800, {0x1f7b7c0, 0xc000020c80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00223fd70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0x20, {0xc00247d180, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00223fd70}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00223fd70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0xc0019f3600, {0xc00247d180, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00223fd70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0xc0017985b0, 0xc00223fda0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00223fd70}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001034640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0x237ac70, 0xc00223fce0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001034640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001034640}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0010345c0}, {0x1f7b7c0, 0xc000020c80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001034640}, 0xc00104e800, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc00223f680}, 0xc0010345c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc00223f680}, 0xc0010345c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00163e480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00163e528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ada4f40, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00223f680}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00289a3d0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00163e778, {0x235a640, 0xc000045ad0}, 0x382d303463342d63)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc00223f680}, 0xc0010342c0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001b9adb8, 0x4c1cb4}, {0xc0570314ba4484e1, 0xb5fe61ec5e}, {0xc001b9adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc00223f680}, 0xc0010342c0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0}, 0xc00247d0a0, 0xc002568c48)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0}, 0x15, 0xc001034300)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0}, 0x2076951, 0xc001034300)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0}, 0x237bc38, 0xc001034300)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f680}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00223f530}, {0x1f33320, 0xc0010342c0}, 0x1e5a940, 0xc001034300)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f530}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0}, 0x23c27b0, 0xc001034300)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00186bb60, {0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00223f500}, {0x1f33320, 0xc0010342c0}, 0xc00247d0a0, 0xc002568c48)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc00223f500}, 0xc001cc4ea0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000676120, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000676120, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266162 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc00101cdc0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1358746 [runnable]:
github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent·dwrap·3()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1
created by github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133 +0x13a

goroutine 1266141 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc00101cf00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1318776 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0015ba360, {0x2392298, 0xc001cc51a0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0006f2240, {0x23922d0, 0xc001bbc690}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0000450e0, {0x23922d0, 0xc001bbc690}, 0xc00100dc30, 0x1, 0x100dc40)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001bbc630, 0xc001034980)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001bbc3c0}, 0xc001034980)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001bbc3c0}, {0x1f35540, 0xc001034980})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001bbc3c0}, {0x1f35540, 0xc001034980}, 0x237bc38, 0xc002569158)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbc3c0}, {0x1f35540, 0xc001034980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001bbc270}, {0x1f35540, 0xc001034980}, 0xc0010342c0, 0xc001034a00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbc270}, {0x1f35540, 0xc001034980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001bbc270}, {0x1f35540, 0xc001034980}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbc270}, {0x1f35540, 0xc001034980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001bbc240}, {0x1f35540, 0xc001034980}, 0x40, 0xc001034a00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbc240}, {0x1f35540, 0xc001034980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001bbc240}, {0x1f35540, 0xc001034980}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001bbc240}, {0x1f35540, 0xc001034980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001bbc240}, {0x1f35540, 0xc001034980}, 0xc00247d260, 0xc002569158)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001bbc240}, 0xc001cc5140, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0017666c0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0017666c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1318844 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000a923c0, {0x2392298, 0xc001971e60}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000378b40, {0x23922d0, 0xc001d31da0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0006cc630, {0x23922d0, 0xc001d31da0}, 0xc00104ada0, 0x1, 0x104adb0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001d31d40, 0xc001047840)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001d31aa0}, 0xc001047840)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001d31aa0}, {0x1f35540, 0xc001047840})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001d31aa0}, {0x1f35540, 0xc001047840}, 0x237bc38, 0xc0025baeb8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d31aa0}, {0x1f35540, 0xc001047840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001d31890}, {0x1f35540, 0xc001047840}, 0xc001047280, 0xc0010478c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d31890}, {0x1f35540, 0xc001047840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d31890}, {0x1f35540, 0xc001047840}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d31890}, {0x1f35540, 0xc001047840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d31860}, {0x1f35540, 0xc001047840}, 0x40, 0xc0010478c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d31860}, {0x1f35540, 0xc001047840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d31860}, {0x1f35540, 0xc001047840}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d31860}, {0x1f35540, 0xc001047840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d31860}, {0x1f35540, 0xc001047840}, 0xc00247d5c0, 0xc0025baeb8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001d31860}, 0xc001971e00, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0006459e0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0006459e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1319040 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0017af440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce98c0, {0x1f7b7c0, 0xc00204bea0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010abd00, 0xc0019b1bf8, 0xc0019b1bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010abd00, {0x1f7b7c0, 0xc00204bea0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e3ecf0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0x20, {0xc00053ec20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e3ecf0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e3ecf0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0xc001b87ce0, {0xc00053ec20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e3ecf0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0xc0017985b0, 0xc001e3ed20, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e3ecf0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001165a80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0x237ac70, 0xc001e3ec30, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001165a80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001165a80}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001165a00}, {0x1f7b7c0, 0xc00204bea0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001165a80}, 0xc00203d1c0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001e3e240}, 0xc001165a00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001e3e240}, 0xc001165a00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0019b2480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0019b2528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ac8b858, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001e3e240}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0027aed10})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0019b2778, {0x235a640, 0xc000045ad0}, 0x392d623237342d38)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001e3e240}, 0xc001165700)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001976db8, 0x4c1cb4}, {0xc0570314c202e284, 0xb601bb1401}, {0xc001976db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001e3e240}, 0xc001165700)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700}, 0xc00053e980, 0xc00205d968)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700}, 0x15, 0xc001165740)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700}, 0x2076951, 0xc001165740)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700}, 0x237bc38, 0xc001165740)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e240}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001e3e0f0}, {0x1f33320, 0xc001165700}, 0x1e5a940, 0xc001165740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e0f0}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700}, 0x23c27b0, 0xc001165740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002471b60, {0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e3e0c0}, {0x1f33320, 0xc001165700}, 0xc00053e980, 0xc00205d968)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001e3e0c0}, 0xc002560600, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002491c20, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002491c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1319034 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000839b00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce96b0, {0x1f736c0, 0xc00115eea0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab900, 0xc00180b5c0, 0xc00180b590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab900, {0x1f736c0, 0xc00115eea0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0024d1140}, {0x2076920, 0x8}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x1c75901, {0xc001fa1fe0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x7fb451d0c108, {0xc001fa1fe0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x20, 0xc001d6fd70, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0xc0006ca340, {0xc001164900, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x0, 0xc001d6fd40, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x0, 0xc001d6fcb0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0024d1140}, {0x2076920, 0x203000}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x203000, 0xc001d6fc80, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0024d1140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc0024d1140}, {0x2076920, 0x200cbbed0}, {0x1f33320, 0xc0011648c0}, {0x1f736c0, 0xc00115eea0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0024d1140}, 0x1fe4e76, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001615ad0, {0x2392298, 0xc0024d1140})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1289993 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001589200, {0x2392298, 0xc001da3c20}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002481c20, {0x23922d0, 0xc0023ba180}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0023ba180}, 0xc000ff0470, 0x1bd58c0, 0xff0480)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0023ba060, 0xc0009dbe40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0021bfda0}, 0xc0009dbe40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0021bfda0}, {0x1f353a0, 0xc0009dbe40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0021bfda0}, {0x1f353a0, 0xc0009dbe40}, 0x237bc38, 0xc001e79680)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021bfda0}, {0x1f353a0, 0xc0009dbe40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0021bfc20, {0x23922d0, 0xc0021bfc50}, {0x1f353a0, 0xc0009dbe40}, 0xc0021bee10, 0xc0009dbe80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021bfc50}, {0x1f353a0, 0xc0009dbe40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021bfc50}, {0x1f353a0, 0xc0009dbe40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021bfc50}, {0x1f353a0, 0xc0009dbe40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021bfbc0}, {0x1f353a0, 0xc0009dbe40}, 0x40, 0xc0009dbe80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021bfbc0}, {0x1f353a0, 0xc0009dbe40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021bfbc0}, {0x1f353a0, 0xc0009dbe40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021bfbc0}, {0x1f353a0, 0xc0009dbe40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021bfbc0}, {0x1f353a0, 0xc0009dbe40}, 0xc000084f00, 0xc001e79680)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0021bfbc0}, 0xc001da3bc0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc002467b00, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc002467b00, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1316578 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1277074 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001a71130)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1272374 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002481200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e580, {0x1f7b7c0, 0xc00204b680}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa300, 0xc001c764d0, 0xc001c764a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa300, {0x1f7b7c0, 0xc00204b680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000c57050}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0x20, {0xc001fa1b40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000c57050}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c57050}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0xc00196eb00, {0xc001fa1b40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c57050}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0xc0024e7e90, 0xc000c57080, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c57050}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0006b6680}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0x237ac70, 0xc000c56fc0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0006b6680}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0006b6680}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0009d9f40}, {0x1f7b7c0, 0xc00204b680}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000ab0f8, {0x2392228, 0xc0006b6680}, 0xc0027bd656, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0024ca3c0}, 0xc0009d9f40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000799920, {0x2392298, 0xc0024ca3c0}, 0xc0009d9f40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0024ca0c0, {0x2392298, 0xc0024ca3c0})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024ca360, {0x2392298, 0xc0024ca3c0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002480c60, {0x23922d0, 0xc0007e7050}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0027bd5f0, {0x23922d0, 0xc0007e7050}, 0xc000e5b690, 0x1, 0xe5b6a0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc0007e6fc0, 0xc0009d8480)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc0007e6d20}, 0xc0009d8480)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0007e6d20}, {0x1f35540, 0xc0009d8480})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0007e6d20}, {0x1f35540, 0xc0009d8480}, 0x237bc38, 0xc001c96450)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007e6d20}, {0x1f35540, 0xc0009d8480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0007e6ab0}, {0x1f35540, 0xc0009d8480}, 0xc000a94700, 0xc0009d8600)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007e6ab0}, {0x1f35540, 0xc0009d8480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0007e6ab0}, {0x1f35540, 0xc0009d8480}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007e6ab0}, {0x1f35540, 0xc0009d8480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0007e6a80}, {0x1f35540, 0xc0009d8480}, 0x40, 0xc0009d8600)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007e6a80}, {0x1f35540, 0xc0009d8480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0007e6a80}, {0x1f35540, 0xc0009d8480}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007e6a80}, {0x1f35540, 0xc0009d8480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0007e6a80}, {0x1f35540, 0xc0009d8480}, 0xc001fa1960, 0xc001c96450)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0007e6a80}, 0xc00230de00, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc002480120, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc002480120, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1285983 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002480ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bbb80, {0x1f7b560, 0xc001981400}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a200, 0xc001597d18, 0xc001597ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a200, {0x1f7b560, 0xc001981400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001fe3bc0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0x20, {0xc0021d63c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001fe3bc0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fe3bc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0xc001581ce0, {0xc0021d63c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fe3bc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0xc0023ec2f0, 0xc001fe3bf0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fe3bc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0021e7bc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0x237ac70, 0xc001fe3b30, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0021e7bc0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0021e7bc0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0021e7b40}, {0x1f7b560, 0xc001981400}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc0021e7bc0}, 0xc002415b90, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001fe3440}, 0xc0021e7b40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001fe3440}, 0xc0021e7b40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0015985a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001598648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42aceb488, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001fe3440}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001ce64a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001598898, {0x235a640, 0xc000045ad0}, 0x382d376339342d64)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001fe3440}, 0xc001fe3170)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001b46db8, 0x4c1cb4}, {0xc057031278e51125, 0xb3e4915eb0}, {0xc001b46db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001fe3440}, 0xc001fe3170)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170}, 0xc0021d62e0, 0xc002591938)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170}, 0x15, 0xc0021e7700)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170}, 0x207690b, 0xc0021e7700)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170}, 0x237bc38, 0xc0021e7700)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe3440}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001fe3200}, {0x1f33180, 0xc001fe3170}, 0x1e5a940, 0xc0021e7700)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe3200}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170}, 0x9, 0xc0021e7700)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170}, 0xc0025cc768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0025ccb60, {0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001fe30e0}, {0x1f33180, 0xc001fe3170}, 0xc0021d62e0, 0xc002591938)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001fe30e0}, 0xc001588ae0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0005d45a0, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0005d45a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1276961 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000680ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5600, {0x1f736c0, 0xc002070ab0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x69)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a700, 0xc00180f5c0, 0xc00180f590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a700, {0x1f736c0, 0xc002070ab0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002527b00}, {0x2076920, 0x8}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x1c75901, {0xc000798c00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x7fb451d0c5b8, {0xc000798c00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x20, 0xc0020771a0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0xc0006ca340, {0xc00207e440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x0, 0xc002077170, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x1e0cfc0, 0xc0020770e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0xc07be7483a, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002527b00}, {0x2076920, 0x203000}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x203000, 0xc0020770b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002527b00}, {0x2076920, 0x46}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000ec1650, {0x2392298, 0xc002527b00}, {0x2076920, 0x176789f5e}, {0x1f33320, 0xc00207e400}, {0x1f736c0, 0xc002070ab0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002527b00}, 0x4802f6, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001615ad0, {0x2392298, 0xc002527b00})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1270847 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002c5aae0, {0x2392228, 0xc002447740})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1319073 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017ddce0, {0x2392298, 0xc0025608a0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a127e0, {0x23922d0, 0xc001e3f590}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002411020, {0x23922d0, 0xc001e3f590}, 0xc001176660, 0x1, 0x1176670)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001e3f530, 0xc001165cc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001e3f2c0}, 0xc001165cc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e3f2c0}, {0x1f35540, 0xc001165cc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e3f2c0}, {0x1f35540, 0xc001165cc0}, 0x237bc38, 0xc00205dd58)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3f2c0}, {0x1f35540, 0xc001165cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e3f170}, {0x1f35540, 0xc001165cc0}, 0xc001165700, 0xc001165d40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3f170}, {0x1f35540, 0xc001165cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e3f170}, {0x1f35540, 0xc001165cc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3f170}, {0x1f35540, 0xc001165cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e3f140}, {0x1f35540, 0xc001165cc0}, 0x40, 0xc001165d40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3f140}, {0x1f35540, 0xc001165cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e3f140}, {0x1f35540, 0xc001165cc0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e3f140}, {0x1f35540, 0xc001165cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e3f140}, {0x1f35540, 0xc001165cc0}, 0xc00053efe0, 0xc00205dd58)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e3f140}, 0xc002560840, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0017af560, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0017af560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282560 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002338b40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb3f0, {0x1f7b560, 0xc00156edc0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7800, 0xc00179e440, 0xc00179e410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7800, {0x1f7b560, 0xc00156edc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d9e1e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0x20, {0xc001fa00c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d9e1e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d9e1e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0xc0003bfce0, {0xc001fa00c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d9e1e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0xc0021b2800, 0xc001d9e210, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d9e1e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d93b00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0x237ac70, 0xc001d9e150, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d93b00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d93b00}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001d93a80}, {0x1f7b560, 0xc00156edc0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001d93b00}, 0xc0007d8406, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc002572960}, 0xc001d93a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc002572960}, 0xc001d93a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00213cea0, {0x2392298, 0xc002572960})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00213d440, {0x2392298, 0xc002572960}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00211d7a0, {0x23922d0, 0xc001d99e30}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001d99e30}, 0xc001d91c90, 0x1bd58c0, 0x1d91ca0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001d99dd0, 0xc001d93880)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001d99b30}, 0xc001d93880)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001d99b30}, {0x1f353a0, 0xc001d93880})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001d99b30}, {0x1f353a0, 0xc001d93880}, 0x237bc38, 0xc002653290)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d99b30}, {0x1f353a0, 0xc001d93880})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001d999b0, {0x23922d0, 0xc001d999e0}, {0x1f353a0, 0xc001d93880}, 0xc001d98ba0, 0xc001d938c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d999e0}, {0x1f353a0, 0xc001d93880})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d999e0}, {0x1f353a0, 0xc001d93880}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d999e0}, {0x1f353a0, 0xc001d93880})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d99950}, {0x1f353a0, 0xc001d93880}, 0x40, 0xc001d938c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d99950}, {0x1f353a0, 0xc001d93880})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d99950}, {0x1f353a0, 0xc001d93880}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d99950}, {0x1f353a0, 0xc001d93880})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d99950}, {0x1f353a0, 0xc001d93880}, 0xc001fa0020, 0xc002653290)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001d99950}, 0xc0025728a0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001961c20, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001961c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266152 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc0018493c0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1129 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000889900, 0xc001dd4b50)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000889900)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1130 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000889900, 0xc001784a60)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000889900)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1131 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000889900, 0xc001e97160)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000889900)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1132 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000889900, 0xc001359620)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000889900)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1133 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125aff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1266176 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0d10, 0xc001f90c68)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286333 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b68ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205eb00, {0x1f7c860, 0xc00205e630}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee7200, 0xc0005c95e8, 0xc0005c95b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee7200, {0x1f7c860, 0xc00205e630})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0028f8780}, {0x20768da, 0x8}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x1c75901, {0xc0008e79e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x7fb451d0c5b8, {0xc0008e79e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x20, 0xc0021175c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0xc0007c0000, {0xc001e0f8c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x0, 0xc002117590, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x1e0cfc0, 0xc002117500, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0028f8780}, {0x20768da, 0x203000}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x8, 0xc0021174d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0028f8780}, {0x20768da, 0x46}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, 0x1cab380, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc0028f8780}, {0x20768da, 0x0}, {0x1f33180, 0xc002117470}, {0x1f7c860, 0xc00205e630}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0028f8780}, 0x10, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000889900, {0x2392298, 0xc0028f8780})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1201976 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5ef8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001f18e00, 0xc000806000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f18e00, {0xc000806000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001f18e00, {0xc000806000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00243abb8, {0xc000806000, 0x466a4e, 0x4383a7})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c23440, {0xc0003283c0, 0x9, 0xc0026dcd18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c23440}, {0xc0003283c0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0003283c0, 0x9, 0xc0026dcd6f}, {0x23585c0, 0xc001c23440})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328380)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001f20000)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1286001 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000d17770)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1266202 [select, 1 minutes]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001be0e70)
	/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
	/temporal/service/history/visibilityQueueProcessor.go:203 +0x85

goroutine 1277021 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020936e0, {0x2392298, 0xc0026684e0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00196a7e0, {0x23922d0, 0xc0021bb380}, 0x2a)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0021bc090, {0x23922d0, 0xc0021bb380}, 0xc0021b8d70, 0x1, 0x21b8d80)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc0021bb320, 0xc002197d80)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc0021bb080}, 0xc002197d80)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021bb080}, {0x1f35540, 0xc002197d80})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0021bb080}, {0x1f35540, 0xc002197d80}, 0x237bc38, 0xc00219b1e8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021bb080}, {0x1f35540, 0xc002197d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0021baf30}, {0x1f35540, 0xc002197d80}, 0xc0021977c0, 0xc002197e00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021baf30}, {0x1f35540, 0xc002197d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021baf30}, {0x1f35540, 0xc002197d80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021baf30}, {0x1f35540, 0xc002197d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021baf00}, {0x1f35540, 0xc002197d80}, 0x40, 0xc002197e00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021baf00}, {0x1f35540, 0xc002197d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021baf00}, {0x1f35540, 0xc002197d80}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021baf00}, {0x1f35540, 0xc002197d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021baf00}, {0x1f35540, 0xc002197d80}, 0xc000799f20, 0xc00219b1e8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0021baf00}, 0xc002668480, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0023bc6c0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0023bc6c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1279 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0017f0d20)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1280 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003672d0, {0x2392228, 0xc001a014c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1170553 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003d22a0, {0x2392228, 0xc001dd8f00})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1276925 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002490240)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f8c0, {0x1f7b560, 0xc001980fa0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab400, 0xc00224dd18, 0xc00224dce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab400, {0x1f7b560, 0xc001980fa0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0020fa6f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0x20, {0xc00247d9c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0020fa6f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fa6f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0xc0016331e0, {0xc00247d9c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fa6f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0xc0023ec2f0, 0xc0020fa720, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020fa6f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0020e1c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0x237ac70, 0xc0020fa660, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0020e1c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0020e1c40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0020e1bc0}, {0x1f7b560, 0xc001980fa0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc0020e1c40}, 0xc0008e49c0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc0020edf80}, 0xc0020e1bc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc0020edf80}, 0xc0020e1bc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00224e5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00224e648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acf3128, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001a31140, {0x23922d0, 0xc0020edf80}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0020cb970})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00224e898, {0x235a640, 0xc000045ad0}, 0x622d373265342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc0020edf80}, 0xc0020eddd0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a48db8, 0x4c1cb4}, {0xc057031200c4bf52, 0xb370d642d0}, {0xc001a48db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc0020edf80}, 0xc0020eddd0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0}, 0xc00247d900, 0xc0020eeac8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0}, 0x15, 0xc0020e1980)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0}, 0x207690b, 0xc0020e1980)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0}, 0x237bc38, 0xc0020e1980)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edf80}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0020ede30}, {0x1f33180, 0xc0020eddd0}, 0x1e5a940, 0xc0020e1980)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020ede30}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0}, 0x9, 0xc0020e1980)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0}, 0xc002604768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002604b60, {0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020edda0}, {0x1f33180, 0xc0020eddd0}, 0xc00247d900, 0xc0020eeac8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0020edda0}, 0xc0024d0660, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0001277a0, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0001277a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266040 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1272373 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002480000)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e4d0, {0x1f7b7c0, 0xc00204b2c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa200, 0xc00187dbf8, 0xc00187dbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa200, {0x1f7b7c0, 0xc00204b2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0007e66c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0x20, {0xc001fa1720, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0007e66c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0007e66c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0xc00196e6e0, {0xc001fa1720, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0007e66c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0xc0023ec2f0, 0xc0007e66f0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0007e66c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000a94b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0x237ac70, 0xc0007e6630, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000a94b00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000a94b00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000a94a80}, {0x1f7b7c0, 0xc00204b2c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc000a94b00}, 0xc0027bd3e0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0009d31d0}, 0xc000a94a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0009d31d0}, 0xc000a94a80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00187e480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00187e528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad567d0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0009d31d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0010bd750})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00187e778, {0x235a640, 0xc000045ad0}, 0x392d373737342d65)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0009d31d0}, 0xc000a94700)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001c40db8, 0x4c1cb4}, {0xc0570311bafeea4d, 0xb333dad9ca}, {0xc001c40db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0009d31d0}, 0xc000a94700)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700}, 0xc001fa14e0, 0xc001c96060)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700}, 0x15, 0xc000a94740)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700}, 0x2076951, 0xc000a94740)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700}, 0x237bc38, 0xc000a94740)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d31d0}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0009d2e70}, {0x1f33320, 0xc000a94700}, 0x1e5a940, 0xc000a94740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d2e70}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700}, 0x23c27b0, 0xc000a94740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0026dab60, {0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0009d2e40}, {0x1f33320, 0xc000a94700}, 0xc001fa14e0, 0xc001c96060)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0009d2e40}, 0xc00230dbc0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006779e0, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006779e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1277066 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0023bde60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00077ac60, {0x1f7c860, 0xc00077ab00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084ac00, 0xc001a255e8, 0xc001a255b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084ac00, {0x1f7c860, 0xc00077ab00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002668f60}, {0x20768da, 0x8}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x1c75901, {0xc0008e6340, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x7fb451d0c5b8, {0xc0008e6340, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x20, 0xc0021ca9c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0xc0007c0000, {0xc0021c2e00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x0, 0xc0021ca990, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x1e0cfc0, 0xc0021ca900, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x60, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002668f60}, {0x20768da, 0x203000}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x8, 0xc0021ca8d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002668f60}, {0x20768da, 0x46}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc002668f60}, {0x20768da, 0x0}, {0x1f33180, 0xc0021ca870}, {0x1f7c860, 0xc00077ab00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc002668f60}, 0xc000e06720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00181c400, {0x2392298, 0xc002668f60})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1272427 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001669440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e790, {0x1f7c860, 0xc00205e6e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aad00, 0xc0008a15e8, 0xc0008a15b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aad00, {0x1f7c860, 0xc00205e6e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0024cb680}, {0x20768da, 0x8}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x1c75901, {0xc00247c0a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x7fb451d0c108, {0xc00247c0a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x20, 0xc000c64c90, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0xc0007c0000, {0xc0023e8300, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x0, 0xc000c64c60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x1e0cfc0, 0xc000c64bd0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x60, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0024cb680}, {0x20768da, 0x203000}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x8, 0xc000c64ba0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0024cb680}, {0x20768da, 0x46}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, 0x1a00, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc0024cb680}, {0x20768da, 0x0}, {0x1f33180, 0xc000c64b40}, {0x1f7c860, 0xc00205e6e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0024cb680}, 0xc000cb7f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000889900, {0x2392298, 0xc0024cb680})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1286317 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000de1f90)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1282653 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001e08140)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1266137 [select, 1 minutes]:
go.temporal.io/server/service/history.(*timerQueueProcessorImpl).completeTimersLoop(0xc0008280e0)
	/temporal/service/history/timerQueueProcessor.go:274 +0x14f
created by go.temporal.io/server/service/history.(*timerQueueProcessorImpl).Start
	/temporal/service/history/timerQueueProcessor.go:166 +0x10f

goroutine 1266136 [select]:
go.temporal.io/server/service/history.(*timerQueueProcessorBase).internalProcessor(0xc001182d80)
	/temporal/service/history/timerQueueProcessorBase.go:271 +0x37f
go.temporal.io/server/service/history.(*timerQueueProcessorBase).processorPump(0xc001182d80)
	/temporal/service/history/timerQueueProcessorBase.go:196 +0x9e
created by go.temporal.io/server/service/history.(*timerQueueProcessorBase).Start
	/temporal/service/history/timerQueueProcessorBase.go:163 +0xaf

goroutine 1276932 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0024d1020, {0x2392228, 0xc0020fcc40})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286325 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000c1f200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e2c0, {0x1f7b560, 0xc00156fb80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee6600, 0xc000885d18, 0xc000885ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee6600, {0x1f7b560, 0xc00156fb80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002136690}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0x20, {0xc0008e7200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002136690}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002136690}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0xc00210e420, {0xc0008e7200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002136690}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0xc0023ec2f0, 0xc0021366c0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002136690}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001dc3300}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0x237ac70, 0xc002136570, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001dc3300}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001dc3300}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001dc3280}, {0x1f7b560, 0xc00156fb80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001dc3300}, 0xc00003f8d8, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc0015cbc20}, 0xc001dc3280, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc0015cbc20}, 0xc001dc3280, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0008865a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000886648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad1ddf0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc0015cbc20}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0017981a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc000886898, {0x235a640, 0xc000045ad0}, 0x622d353465342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc0015cbc20}, 0xc0015cba70)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a58db8, 0x4c1cb4}, {0xc057031281b21477, 0xb3e8f92c02}, {0xc001a58db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc0015cbc20}, 0xc0015cba70)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70}, 0xc0008e7100, 0xc00268dd10)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70}, 0x15, 0xc001dc3040)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70}, 0x207690b, 0xc001dc3040)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70}, 0x237bc38, 0xc001dc3040)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cbc20}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0015cbad0}, {0x1f33180, 0xc0015cba70}, 0x1e5a940, 0xc001dc3040)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cbad0}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70}, 0x9, 0xc001dc3040)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70}, 0xc00260f768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00260fb60, {0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015cba40}, {0x1f33180, 0xc0015cba70}, 0xc0008e7100, 0xc00268dd10)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0015cba40}, 0xc002216f00, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000126480, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000126480, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1201989 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0025f10e0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001c23b60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 1277020 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0023bc5a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00077a000, {0x1f7b7c0, 0xc0000210e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc3)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084ab00, 0xc00270dbf8, 0xc00270dbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084ab00, {0x1f7b7c0, 0xc0000210e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021baba0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0x20, {0xc000799d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021baba0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021baba0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0xc0020e31e0, {0xc000799d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021baba0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0xc0023ec2f0, 0xc0021babd0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021baba0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002197b40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0x237ac70, 0xc0021bab10, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002197b40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002197b40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc002197ac0}, {0x1f7b7c0, 0xc0000210e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc002197b40}, 0xc002341e90, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0021ba450}, 0xc002197ac0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0021ba450}, 0xc002197ac0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00270e480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00270e528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42abecd98, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0021ba450}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0021b2730})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00270e778, {0x235a640, 0xc000045ad0}, 0x392d356533342d66)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0021ba450}, 0xc0021977c0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a46db8, 0x4c1cb4}, {0xc057031203120384, 0xb373238701}, {0xc001a46db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0021ba450}, 0xc0021977c0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0}, 0xc000799b40, 0xc00219adf8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0}, 0x15, 0xc002197800)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0}, 0x2076951, 0xc002197800)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0}, 0x237bc38, 0xc002197800)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba450}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021ba300}, {0x1f33320, 0xc0021977c0}, 0x1e5a940, 0xc002197800)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba300}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0}, 0x23c27b0, 0xc002197800)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a05b60, {0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021ba2d0}, {0x1f33320, 0xc0021977c0}, 0xc000799b40, 0xc00219adf8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0021ba2d0}, 0xc0026681e0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00196b0e0, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00196b0e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270843 [select, 1 minutes]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00213b320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205ea50, {0x1f7b7c0, 0xc00204bb80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969500, 0xc0017a24d0, 0xc0017a24a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969500, {0x1f7b7c0, 0xc00204bb80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00215c120}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0x20, {0xc0019a4dc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00215c120}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00215c120}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0xc002645b80, {0xc0019a4dc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00215c120}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0xc0024e7e90, 0xc00215c150, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00215c120}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0024478c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0x237ac70, 0xc00215c090, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0024478c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0024478c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc002447800}, {0x1f7b7c0, 0xc00204bb80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000ab0f8, {0x2392228, 0xc0024478c0}, 0xc0026729f6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000ab1ce0, {0x2392298, 0xc002c5ae40}, 0xc002447800, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000799920, {0x2392298, 0xc002c5ae40}, 0xc002447800, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002c5aba0, {0x2392298, 0xc002c5ae40})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002c5ade0, {0x2392298, 0xc002c5ae40}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00213ad80, {0x23922d0, 0xc002137560}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002672990, {0x23922d0, 0xc002137560}, 0xc000df1a70, 0x1, 0xdf1a80)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc002137500, 0xc002446f40)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc002137260}, 0xc002446f40)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002137260}, {0x1f35540, 0xc002446f40})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002137260}, {0x1f35540, 0xc002446f40}, 0x237bc38, 0xc001b8d950)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002137260}, {0x1f35540, 0xc002446f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc002137110}, {0x1f35540, 0xc002446f40}, 0xc002446980, 0xc002446fc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002137110}, {0x1f35540, 0xc002446f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002137110}, {0x1f35540, 0xc002446f40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002137110}, {0x1f35540, 0xc002446f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021370e0}, {0x1f35540, 0xc002446f40}, 0x40, 0xc002446fc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021370e0}, {0x1f35540, 0xc002446f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021370e0}, {0x1f35540, 0xc002446f40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021370e0}, {0x1f35540, 0xc002446f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021370e0}, {0x1f35540, 0xc002446f40}, 0xc0019a49e0, 0xc001b8d950)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0021370e0}, 0xc002c5a900, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00213a360, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00213a360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266087 [select, 1 minutes]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001f69ef0)
	/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
	/temporal/service/history/visibilityQueueProcessor.go:203 +0x85

goroutine 1159250 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000367110, {0x2392228, 0xc0024dba80})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1282 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0017ddb60, {0x2392228, 0xc001a01500})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1281 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0017ddb60, {0x2392228, 0xc001a01500})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266178 [select, 1 minutes]:
go.temporal.io/server/service/history.(*transferQueueProcessorImpl).completeTransferLoop(0xc001187140)
	/temporal/service/history/transferQueueProcessor.go:270 +0x125
created by go.temporal.io/server/service/history.(*transferQueueProcessorImpl).Start
	/temporal/service/history/transferQueueProcessor.go:166 +0xfb

goroutine 1270846 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002c5aae0, {0x2392228, 0xc002447740})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286319 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002216120, {0x2392228, 0xc001dc2440})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1285978 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024adf80, {0x2392298, 0xc001588360}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001082360, {0x23922d0, 0xc001fe2090}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fe2090}, 0xc0022004f0, 0x1bd58c0, 0x2200500)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001fe2030, 0xc0021e6f40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc00257bd40}, 0xc0021e6f40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00257bd40}, {0x1f353a0, 0xc0021e6f40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc00257bd40}, {0x1f353a0, 0xc0021e6f40}, 0x237bc38, 0xc0025912a8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257bd40}, {0x1f353a0, 0xc0021e6f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00257bbc0, {0x23922d0, 0xc00257bbf0}, {0x1f353a0, 0xc0021e6f40}, 0xc00257a1e0, 0xc0021e6f80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257bbf0}, {0x1f353a0, 0xc0021e6f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00257bbf0}, {0x1f353a0, 0xc0021e6f40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257bbf0}, {0x1f353a0, 0xc0021e6f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00257bb60}, {0x1f353a0, 0xc0021e6f40}, 0x40, 0xc0021e6f80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257bb60}, {0x1f353a0, 0xc0021e6f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00257bb60}, {0x1f353a0, 0xc0021e6f40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00257bb60}, {0x1f353a0, 0xc0021e6f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00257bb60}, {0x1f353a0, 0xc0021e6f40}, 0xc0019a5f20, 0xc0025912a8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc00257bb60}, 0xc001588300, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001767b00, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001767b00, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1290275 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc00290e690)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1272429 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000c17560)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e840, {0x1f7b560, 0xc00204be00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010ab100, 0xc001791d18, 0xc001791ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010ab100, {0x1f7b560, 0xc00204be00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0007b0a20}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0x20, {0xc00247c320, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0007b0a20}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0007b0a20}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0xc001c4e420, {0xc00247c320, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0007b0a20}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0xc0023ec2f0, 0xc0007b0a50, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0007b0a20}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0023e8840}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0x237ac70, 0xc0007b0990, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0023e8840}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0023e8840}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0023e87c0}, {0x1f7b560, 0xc00204be00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc0023e8840}, 0xc00218a980, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc000c65980}, 0xc0023e87c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc000c65980}, 0xc0023e87c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017925a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001792648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acd9ff8, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001a31140, {0x23922d0, 0xc000c65980}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000376470})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001792898, {0x235a640, 0xc000045ad0}, 0x612d323534342d62)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc000c65980}, 0xc000c655f0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc0007ccdb8, 0x4c1cb4}, {0xc0570311c20bb713, 0xb336827098}, {0xc0007ccdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc000c65980}, 0xc000c655f0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0}, 0xc00247c240, 0xc001c97fe0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0}, 0x15, 0xc0023e8500)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0}, 0x207690b, 0xc0023e8500)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0}, 0x237bc38, 0xc0023e8500)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65980}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000c65650}, {0x1f33180, 0xc000c655f0}, 0x1e5a940, 0xc0023e8500)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c65650}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0}, 0x9, 0xc0023e8500)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0}, 0xc001c8a768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001c8ab60, {0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c655c0}, {0x1f33180, 0xc000c655f0}, 0xc00247c240, 0xc001c97fe0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc000c655c0}, 0xc0024cb8c0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001669560, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001669560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282753 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00213d440, {0x2392298, 0xc0008a9020}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00211d7a0, {0x23922d0, 0xc001e8ec30}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001e8ec30}, 0xc001d7f960, 0x1bd58c0, 0x1d7f970)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001e8ebd0, 0xc001e90200)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001e8e930}, 0xc001e90200)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e8e930}, {0x1f353a0, 0xc001e90200})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e8e930}, {0x1f353a0, 0xc001e90200}, 0x237bc38, 0xc001e88810)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8e930}, {0x1f353a0, 0xc001e90200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001e8e7b0, {0x23922d0, 0xc001e8e7e0}, {0x1f353a0, 0xc001e90200}, 0xc001e83980, 0xc001e90240)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8e7e0}, {0x1f353a0, 0xc001e90200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e8e7e0}, {0x1f353a0, 0xc001e90200}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8e7e0}, {0x1f353a0, 0xc001e90200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e8e750}, {0x1f353a0, 0xc001e90200}, 0x40, 0xc001e90240)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8e750}, {0x1f353a0, 0xc001e90200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e8e750}, {0x1f353a0, 0xc001e90200}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8e750}, {0x1f353a0, 0xc001e90200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e8e750}, {0x1f353a0, 0xc001e90200}, 0xc0019a47e0, 0xc001e88810)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e8e750}, 0xc0008a8fc0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b687e0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b687e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1178413 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00164a3c0, {0x2392228, 0xc001061c80})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1276930 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000622cb0, {0x2392228, 0xc0020fcc00})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1266147 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc001f69ef0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1159251 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002160e40, {0x2392228, 0xc0024dbac0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286351 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008ce000)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f080, {0x1f7c860, 0xc00205efd0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa400, 0xc001c135e8, 0xc001c135b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa400, {0x1f7c860, 0xc00205efd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00286b4a0}, {0x20768da, 0x8}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x1c75901, {0xc001fa0d60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x7fb451d0c5b8, {0xc001fa0d60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x20, 0xc002114e70, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0xc0007c0000, {0xc001d72dc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x0, 0xc002114e40, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x1e0cfc0, 0xc002114db0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x2, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00286b4a0}, {0x20768da, 0x203000}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0x8, 0xc002114d80, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00286b4a0}, {0x20768da, 0x46}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, 0xc000cbd600, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc00286b4a0}, {0x20768da, 0x0}, {0x1f33180, 0xc002114d20}, {0x1f7c860, 0xc00205efd0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc00286b4a0}, 0xc000cbd720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00181c400, {0x2392298, 0xc00286b4a0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1319067 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008ce900)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001839130, {0x1f736c0, 0xc00105dc20}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7900, 0xc0016615c0, 0xc001661590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7900, {0x1f736c0, 0xc00105dc20})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00174f320}, {0x2076920, 0x8}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x1c75901, {0xc0007994c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x7fb451d0c5b8, {0xc0007994c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x20, 0xc001e98810, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0xc0006ca340, {0xc001194d00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x0, 0xc001e987e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x0, 0xc001e98630, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00174f320}, {0x2076920, 0x203000}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x203000, 0xc001e98600, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00174f320}, {0x2076920, 0x46}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0026dae50, {0x2392298, 0xc00174f320}, {0x2076920, 0x200579040}, {0x1f33320, 0xc001194cc0}, {0x1f736c0, 0xc00105dc20}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc00174f320}, 0xc00075e840, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001821ba0, {0x2392298, 0xc00174f320})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1272377 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0024ca000, {0x2392228, 0xc0009d9c00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1318721 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000a923c0, {0x2392298, 0xc0015bbaa0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000378b40, {0x23922d0, 0xc0021cb140}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0027bd710, {0x23922d0, 0xc0021cb140}, 0xc000fbcbd0, 0x1, 0xfbcbe0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc0021cb020, 0xc000fb3b80)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc0021cad50}, 0xc000fb3b80)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021cad50}, {0x1f35540, 0xc000fb3b80})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0021cad50}, {0x1f35540, 0xc000fb3b80}, 0x237bc38, 0xc0025e5668)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021cad50}, {0x1f35540, 0xc000fb3b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0021cac00}, {0x1f35540, 0xc000fb3b80}, 0xc000fb3580, 0xc000fb3c00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021cac00}, {0x1f35540, 0xc000fb3b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021cac00}, {0x1f35540, 0xc000fb3b80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021cac00}, {0x1f35540, 0xc000fb3b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021cabd0}, {0x1f35540, 0xc000fb3b80}, 0x40, 0xc000fb3c00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021cabd0}, {0x1f35540, 0xc000fb3b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021cabd0}, {0x1f35540, 0xc000fb3b80}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021cabd0}, {0x1f35540, 0xc000fb3b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021cabd0}, {0x1f35540, 0xc000fb3b80}, 0xc00247ce80, 0xc0025e5668)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0021cabd0}, 0xc0015bba40, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024665a0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024665a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282763 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00196a120)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5340, {0x1f7c860, 0xc001cb5290}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969300, 0xc0008955e8, 0xc0008955b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969300, {0x1f7c860, 0xc001cb5290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001ffe180}, {0x20768da, 0x8}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x1c75901, {0xc0019a5720, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x7fb451d0c108, {0xc0019a5720, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x20, 0xc001ea2420, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0xc0007c0000, {0xc001e91a80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x0, 0xc001ea23f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x1e0cfc0, 0xc001ea2360, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x2, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001ffe180}, {0x20768da, 0x203000}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0x8, 0xc001ea2330, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001ffe180}, {0x20768da, 0x46}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, 0xc001b5d650, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc001ffe180}, {0x20768da, 0x0}, {0x1f33180, 0xc001ea22d0}, {0x1f7c860, 0xc001cb5290}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc001ffe180}, 0xc001b5d720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000889900, {0x2392298, 0xc001ffe180})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1282765 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00196b8c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb53f0, {0x1f7b560, 0xc001980280}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969400, 0xc00216fd18, 0xc00216fce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969400, {0x1f7b560, 0xc001980280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ea30e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0x20, {0xc0019a5900, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ea30e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ea30e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0xc0019f2420, {0xc0019a5900, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ea30e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0xc0023ec2f0, 0xc001ea3110, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ea30e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e91f40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0x33, 0xc001ea3050, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e91f40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e91f40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e91ec0}, {0x1f7b560, 0xc001980280}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001e91f40}, 0xc00074d1e0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001ea2990}, 0xc001e91ec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001ea2990}, 0xc001e91ec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0021705a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002170648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42abf0340, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001ea2990}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00209d370})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002170898, {0x235a640, 0xc000045ad0}, 0x392d646236342d38)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001ea2990}, 0xc001ea27e0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001620db8, 0x4c1cb4}, {0xc057031242f6c718, 0xb3aea314a2}, {0xc001620db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001ea2990}, 0xc001ea27e0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0}, 0xc0019a5820, 0xc001e89e60)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0}, 0x15, 0xc001e91c80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0}, 0x207690b, 0xc001e91c80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0}, 0x237bc38, 0xc001e91c80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea2990}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ea2840}, {0x1f33180, 0xc001ea27e0}, 0x1e5a940, 0xc001e91c80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea2840}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0}, 0x9, 0xc001e91c80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0}, 0xc000a07768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a07b60, {0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ea27b0}, {0x1f33180, 0xc001ea27e0}, 0xc0019a5820, 0xc001e89e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001ea27b0}, 0xc001ffe420, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00196a360, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00196a360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1180 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc001810920)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a2d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a2d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1181 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00177a2d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1182 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001821ba0, 0xc00105f9d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001821ba0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1183 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001821ba0, 0xc0021b8650)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001821ba0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1184 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001821ba0, 0xc001161fa0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001821ba0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1233 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001821ba0, 0xc001d689b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001821ba0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1234 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00177a1e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1272449 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000668120)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060f6b0, {0x1f7c860, 0xc00060f550}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a500, 0xc0017e35e8, 0xc0017e35b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a500, {0x1f7c860, 0xc00060f550})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00275f8c0}, {0x20768da, 0x8}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x1c75901, {0xc0019a49c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x7fb451d0c5b8, {0xc0019a49c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x20, 0xc0010a35f0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0xc0007c0000, {0xc000772e00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x0, 0xc0010a35c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x1e0cfc0, 0xc0010a3530, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00275f8c0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x8, 0xc0010a3500, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00275f8c0}, {0x20768da, 0x46}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, 0x1cab380, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc00275f8c0}, {0x20768da, 0x0}, {0x1f33180, 0xc0010a3470}, {0x1f7c860, 0xc00060f550}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc00275f8c0}, 0x4, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000889900, {0x2392298, 0xc00275f8c0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1319068 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0023bca20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018391e0, {0x1f7b7c0, 0xc000021cc0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7a00, 0xc002709bf8, 0xc002709bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7a00, {0x1f7b7c0, 0xc000021cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e99b30}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0x20, {0xc000799be0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e99b30}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e99b30}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0xc000eb29a0, {0xc000799be0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e99b30}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0xc0017985b0, 0xc001e99b90, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e99b30}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001195280}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0x237ac70, 0xc001e999e0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001195280}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001195280}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001195200}, {0x1f7b7c0, 0xc000021cc0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001195280}, 0xc00104fa80, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001e98f90}, 0xc001195200, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001e98f90}, 0xc001195200, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00270a480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00270a528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ac85060, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001e98f90}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000f85050})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00270a778, {0x235a640, 0xc000045ad0}, 0x622d636635342d30)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001e98f90}, 0xc001194f00)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc000ac0db8, 0x4c1cb4}, {0xc0570314c2507589, 0xb60208a70e}, {0xc000ac0db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001e98f90}, 0xc001194f00)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00}, 0xc0007996e0, 0xc0020ef530)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00}, 0x15, 0xc001194f40)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00}, 0x2076951, 0xc001194f40)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00}, 0x237bc38, 0xc001194f40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98f90}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001e98e10}, {0x1f33320, 0xc001194f00}, 0x1e5a940, 0xc001194f40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98e10}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00}, 0x23c27b0, 0xc001194f40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00}, 0x5, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0026dab60, {0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00}, 0x40, 0xc0003e1400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e98de0}, {0x1f33320, 0xc001194f00}, 0xc0007996e0, 0xc0020ef530)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001e98de0}, 0xc00174f560, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0008cea20, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0008cea20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1285981 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005d4360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb130, {0x1f7c860, 0xc0022bae70}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a100, 0xc001c155e8, 0xc001c155b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a100, {0x1f7c860, 0xc0022bae70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001588720}, {0x20768da, 0x8}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x1c75901, {0xc0021d60c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x7fb451d0c5b8, {0xc0021d60c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x20, 0xc001fe2c30, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0xc0007c0000, {0xc0021e74c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x0, 0xc001fe2c00, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x1e0cfc0, 0xc001fe2b70, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001588720}, {0x20768da, 0x203000}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x8, 0xc001fe2b40, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001588720}, {0x20768da, 0x46}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc001588720}, {0x20768da, 0x0}, {0x1f33180, 0xc001fe2ae0}, {0x1f7c860, 0xc0022bae70}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc001588720}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc001588720})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1313 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00181c400, 0xc0021b9c10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00181c400)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1319426 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1266128 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc002696dc8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1277018 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00196aea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5e40, {0x1f736c0, 0xc002195a70}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x75)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a900, 0xc00197d5c0, 0xc00197d590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a900, {0x1f736c0, 0xc002195a70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001ccdd40}, {0x2076920, 0x8}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x1c75901, {0xc000799860, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x7fb451d0c5b8, {0xc000799860, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x20, 0xc00219ff20, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0xc0006ca340, {0xc0021975c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x0, 0xc00219fef0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x1e0cfc0, 0xc00219fe60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x4d57be7483a, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001ccdd40}, {0x2076920, 0x203000}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x203000, 0xc00219fe30, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001ccdd40}, {0x2076920, 0x46}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000ddbe50, {0x2392298, 0xc001ccdd40}, {0x2076920, 0x100ddbed0}, {0x1f33320, 0xc002197580}, {0x1f736c0, 0xc002195a70}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001ccdd40}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001821ba0, {0x2392298, 0xc001ccdd40})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1266183 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0dc0, 0x8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1319038 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002491b00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9810, {0x1f736c0, 0xc00115f320}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010abc00, 0xc000aa55c0, 0xc000aa5590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010abc00, {0x1f736c0, 0xc00115f320})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0025603c0}, {0x2076920, 0x8}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x1c75901, {0xc00053e6c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x7fb451d0c108, {0xc00053e6c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x20, 0xc001e8fd10, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0xc0006ca340, {0xc001165500, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x0, 0xc001e8fce0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x0, 0xc001e8fc50, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0025603c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x203000, 0xc001e8fc20, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0025603c0}, {0x2076920, 0x46}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc0025603c0}, {0x2076920, 0x2685721ff}, {0x1f33320, 0xc0011654c0}, {0x1f736c0, 0xc00115f320}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0025603c0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001821ba0, {0x2392298, 0xc0025603c0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1266167 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0d10, 0xc001f90e00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266090 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1272274 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000622070, {0x2392228, 0xc0023d4340})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1277169 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006859e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022ba370, {0x1f7b560, 0xc00156f9a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6f00, 0xc0019ae440, 0xc0019ae410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6f00, {0x1f7b560, 0xc00156f9a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00223e0f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0x20, {0xc0021d75c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00223e0f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00223e0f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0xc001678f20, {0xc0021d75c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00223e0f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0xc0021b2800, 0xc00223e120, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00223e0f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002233100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0x237ac70, 0xc00223e060, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002233100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002233100}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc002233080}, {0x1f7b560, 0xc00156f9a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc002233100}, 0xc0021339c6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0025032c0}, 0xc002233080, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0025032c0}, 0xc002233080, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002503020, {0x2392298, 0xc0025032c0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002503260, {0x2392298, 0xc0025032c0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000684ea0, {0x23922d0, 0xc002223440}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002223440}, 0xc002228c60, 0x1bd58c0, 0x2228c70)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0022233e0, 0xc002232800)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc002223140}, 0xc002232800)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002223140}, {0x1f353a0, 0xc002232800})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002223140}, {0x1f353a0, 0xc002232800}, 0x237bc38, 0xc002213290)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002223140}, {0x1f353a0, 0xc002232800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002222fc0, {0x23922d0, 0xc002222ff0}, {0x1f353a0, 0xc002232800}, 0x203000, 0xc002232840)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002222ff0}, {0x1f353a0, 0xc002232800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002222ff0}, {0x1f353a0, 0xc002232800}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002222ff0}, {0x1f353a0, 0xc002232800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002222f60}, {0x1f353a0, 0xc002232800}, 0x40, 0xc002232840)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002222f60}, {0x1f353a0, 0xc002232800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002222f60}, {0x1f353a0, 0xc002232800}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002222f60}, {0x1f353a0, 0xc002232800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002222f60}, {0x1f353a0, 0xc002232800}, 0xc0021d71e0, 0xc002213290)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc002222f60}, 0xc002502d80, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0006417a0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0006417a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266135 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc0009e37c0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286282 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00110fda0, {0x2392228, 0xc001d2f340})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1282666 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002c5ade0, {0x2392298, 0xc002282f00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00213ad80, {0x23922d0, 0xc001d6e4e0}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002224b10, {0x23922d0, 0xc001d6e4e0}, 0xc001d68c20, 0x1, 0x1d68c30)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001d6e480, 0xc001d5f740)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001d6e1e0}, 0xc001d5f740)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001d6e1e0}, {0x1f35540, 0xc001d5f740})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001d6e1e0}, {0x1f35540, 0xc001d5f740}, 0x237bc38, 0xc0027c4048)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d6e1e0}, {0x1f35540, 0xc001d5f740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001d6e090}, {0x1f35540, 0xc001d5f740}, 0xc001d5eec0, 0xc001d5f7c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d6e090}, {0x1f35540, 0xc001d5f740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d6e090}, {0x1f35540, 0xc001d5f740}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d6e090}, {0x1f35540, 0xc001d5f740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d6e060}, {0x1f35540, 0xc001d5f740}, 0x40, 0xc001d5f7c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d6e060}, {0x1f35540, 0xc001d5f740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d6e060}, {0x1f35540, 0xc001d5f740}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d6e060}, {0x1f35540, 0xc001d5f740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d6e060}, {0x1f35540, 0xc001d5f740}, 0xc0019a42a0, 0xc0027c4048)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001d6e060}, 0xc002282ea0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001083b00, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001083b00, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266208 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0xc002474d80)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1201980 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef1660, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000957f80, 0xc000852000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000957f80, {0xc000852000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000957f80, {0xc000852000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0021a1b08, {0xc000852000, 0xff8e, 0xc051a60d0c})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c23500, {0xc000328d60, 0x9, 0x18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c23500}, {0xc000328d60, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000328d60, 0x9, 0xc0008e4090}, {0x23585c0, 0xc001c23500})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000328d20)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000c08f00, 0xc000787f98, 0x1418c25)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000750380, {0x23d1088, 0xc000c08f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1312 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00181c400, 0xc001e96580)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00181c400)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1314 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00181c400, 0xc002162aa0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00181c400)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1315 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00181c400, 0xc001d69fb0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00181c400)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1316 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00177a3c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1282758 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0017f8fc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5080, {0x1f7c860, 0xc001cb4fd0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968e00, 0xc001b495e8, 0xc001b495b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968e00, {0x1f7c860, 0xc001cb4fd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0008a9860}, {0x20768da, 0x8}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x1c75901, {0xc0019a52c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x7fb451d0c108, {0xc0019a52c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x20, 0xc001e985d0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0xc0007c0000, {0xc001e90e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x0, 0xc001e985a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x1e0cfc0, 0xc001e98510, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0008a9860}, {0x20768da, 0x203000}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0x8, 0xc001e984e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0008a9860}, {0x20768da, 0x46}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, 0xc000dd5e50, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc0008a9860}, {0x20768da, 0x0}, {0x1f33180, 0xc001e98480}, {0x1f7c860, 0xc001cb4fd0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0008a9860}, 0xc000dd5f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00181c400, {0x2392298, 0xc0008a9860})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1355 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc0019a4b80)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a690)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a690)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1290274 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a11d40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001556630, {0x1f7b7c0, 0xc000020be0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084b300, 0xc00278e4d0, 0xc00278e4a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084b300, {0x1f7b7c0, 0xc000020be0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000c4b410}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0x20, {0xc0019a5220, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000c4b410}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c4b410}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0xc001580c60, {0xc0019a5220, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c4b410}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0xc0021b2800, 0xc000c4b440, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c4b410}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00101df00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0x237ac70, 0xc000c4b380, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00101df00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00101df00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00101de40}, {0x1f7b7c0, 0xc000020be0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc0000ab0f8, {0x2392228, 0xc00101df00}, 0xc000c46406, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0024cb560}, 0xc00101de40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000799920, {0x2392298, 0xc0024cb560}, 0xc00101de40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0024cb2c0, {0x2392298, 0xc0024cb560})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024cb500, {0x2392298, 0xc0024cb560}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000a11680, {0x23922d0, 0xc000c4a570}, 0x2d)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001957b00, {0x23922d0, 0xc000c4a570}, 0xc0019ed720, 0x1, 0x19ed730)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc000c4a510, 0xc00101cfc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc000c4a210}, 0xc00101cfc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000c4a210}, {0x1f35540, 0xc00101cfc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc000c4a210}, {0x1f35540, 0xc00101cfc0}, 0x237bc38, 0xc0003240c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4a210}, {0x1f35540, 0xc00101cfc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc000c4a090}, {0x1f35540, 0xc00101cfc0}, 0xc00101c9c0, 0xc00101d040)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4a090}, {0x1f35540, 0xc00101cfc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c4a090}, {0x1f35540, 0xc00101cfc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4a090}, {0x1f35540, 0xc00101cfc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0009e5fb0}, {0x1f35540, 0xc00101cfc0}, 0x40, 0xc00101d040)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e5fb0}, {0x1f35540, 0xc00101cfc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0009e5fb0}, {0x1f35540, 0xc00101cfc0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e5fb0}, {0x1f35540, 0xc00101cfc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0009e5fb0}, {0x1f35540, 0xc00101cfc0}, 0xc0019a4f00, 0xc0003240c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0009e5fb0}, 0xc0024cb020, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000a10c60, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000a10c60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266134 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc0009e37c0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1277037 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002467e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205fef0, {0x1f7c860, 0xc00205fe40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6300, 0xc001b4b5e8, 0xc001b4b5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6300, {0x1f7c860, 0xc00205fe40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002893260}, {0x20768da, 0x8}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x1c75901, {0xc0021d6780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x7fb451d0c108, {0xc0021d6780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x20, 0xc00215fd40, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0xc0007c0000, {0xc002157280, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x0, 0xc00215fd10, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x1e0cfc0, 0xc00215fc80, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002893260}, {0x20768da, 0x203000}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x8, 0xc00215fc50, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002893260}, {0x20768da, 0x46}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c108, {0x2392298, 0xc002893260}, {0x20768da, 0x0}, {0x1f33180, 0xc00215fbf0}, {0x1f7c860, 0xc00205fe40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc002893260}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00181c400, {0x2392298, 0xc002893260})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1266061 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69d90, 0xc000edacc0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1277073 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000641560)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022ba2c0, {0x1f7b560, 0xc00156f400}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6700, 0xc001546440, 0xc001546410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6700, {0x1f7b560, 0xc00156f400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002203380}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0x20, {0xc0021d6fe0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002203380}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002203380}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0xc001678b00, {0xc0021d6fe0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002203380}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0xc0021b2800, 0xc0022033b0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002203380}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00217fc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0x237ac70, 0xc0022032f0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00217fc40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00217fc40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc00217fbc0}, {0x1f7b560, 0xc00156f400}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc00217fc40}, 0xc002132fc6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc001ba4c60}, 0xc00217fbc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc001ba4c60}, 0xc00217fbc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001ba48a0, {0x2392298, 0xc001ba4c60})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001ba4ae0, {0x2392298, 0xc001ba4c60}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000640a20, {0x23922d0, 0xc0022026f0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0022026f0}, 0xc002200f90, 0x1bd58c0, 0x2200fa0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc002202690, 0xc00217f340)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0022023f0}, 0xc00217f340)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0022023f0}, {0x1f353a0, 0xc00217f340})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0022023f0}, {0x1f353a0, 0xc00217f340}, 0x237bc38, 0xc00216b950)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022023f0}, {0x1f353a0, 0xc00217f340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002202270, {0x23922d0, 0xc0022022a0}, {0x1f353a0, 0xc00217f340}, 0x7fb42ace0020, 0xc00217f380)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022022a0}, {0x1f353a0, 0xc00217f340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022022a0}, {0x1f353a0, 0xc00217f340}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022022a0}, {0x1f353a0, 0xc00217f340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002202210}, {0x1f353a0, 0xc00217f340}, 0x40, 0xc00217f380)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002202210}, {0x1f353a0, 0xc00217f340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002202210}, {0x1f353a0, 0xc00217f340}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002202210}, {0x1f353a0, 0xc00217f340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002202210}, {0x1f353a0, 0xc00217f340}, 0xc0021d6e40, 0xc00216b950)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc002202210}, 0xc001ba4600, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b85680, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc001b85680, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1319036 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00174e180, {0x2392298, 0xc002560180}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0015f97a0, {0x23922d0, 0xc001e8f620}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002410cc0, {0x23922d0, 0xc001e8f620}, 0xc001161d00, 0x1, 0x1161d10)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001e8f5c0, 0xc001165080)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001e8f350}, 0xc001165080)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e8f350}, {0x1f35540, 0xc001165080})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e8f350}, {0x1f35540, 0xc001165080}, 0x237bc38, 0xc00205cfd8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8f350}, {0x1f35540, 0xc001165080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e8f200}, {0x1f35540, 0xc001165080}, 0xc001164b00, 0xc001165100)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8f200}, {0x1f35540, 0xc001165080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e8f200}, {0x1f35540, 0xc001165080}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8f200}, {0x1f35540, 0xc001165080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e8f1d0}, {0x1f35540, 0xc001165080}, 0x40, 0xc001165100)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8f1d0}, {0x1f35540, 0xc001165080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e8f1d0}, {0x1f35540, 0xc001165080}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e8f1d0}, {0x1f35540, 0xc001165080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e8f1d0}, {0x1f35540, 0xc001165080}, 0xc00053e0a0, 0xc00205cfd8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e8f1d0}, 0xc0024d1f80, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00196b7a0, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00196b7a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1277042 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006be6c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00077b4a0, {0x1f7b560, 0xc00204afa0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969a00, 0xc0014aa440, 0xc0014aa410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969a00, {0x1f7b560, 0xc00204afa0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00215d2f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0x20, {0xc00247da00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00215d2f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00215d2f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0xc0018346e0, {0xc00247da00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00215d2f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0xc0021b2800, 0xc00215d320, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00215d2f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001dacb80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0x237ac70, 0xc00215d260, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001dacb80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001dacb80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001dacb00}, {0x1f7b560, 0xc00204afa0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001dacb80}, 0xc0022dfc46, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0028938c0}, 0xc001dacb00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0028938c0}, 0xc001dacb00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002817f80, {0x2392298, 0xc0028938c0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000086c60, {0x2392298, 0xc0028938c0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00067a120, {0x23922d0, 0xc002177500}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002177500}, 0xc002163420, 0x1bd58c0, 0x2163430)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0021774a0, 0xc002157b00)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc002177200}, 0xc002157b00)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002177200}, {0x1f353a0, 0xc002157b00})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002177200}, {0x1f353a0, 0xc002157b00}, 0x237bc38, 0xc00216a8b8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002177200}, {0x1f353a0, 0xc002157b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002177080, {0x23922d0, 0xc0021770b0}, {0x1f353a0, 0xc002157b00}, 0xc002176120, 0xc002157b40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021770b0}, {0x1f353a0, 0xc002157b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021770b0}, {0x1f353a0, 0xc002157b00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021770b0}, {0x1f353a0, 0xc002157b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002177020}, {0x1f353a0, 0xc002157b00}, 0x40, 0xc002157b40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002177020}, {0x1f353a0, 0xc002157b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002177020}, {0x1f353a0, 0xc002157b00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002177020}, {0x1f353a0, 0xc002157b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002177020}, {0x1f353a0, 0xc002157b00}, 0xc0021d6a20, 0xc00216a8b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc002177020}, 0xc002893860, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0017af320, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0017af320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1357 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0019e29c0, 0xc001177a80)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0019e29c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1356 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00177a690)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1358 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0019e29c0, 0xc0019ecf70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0019e29c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1359 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00177a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1319088 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006bfd40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9970, {0x1f736c0, 0xc00122c630}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee6700, 0xc001c035c0, 0xc001c03590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee6700, {0x1f736c0, 0xc00122c630})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002561500}, {0x2076920, 0x8}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x1c75901, {0xc00053f980, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x7fb451d0c108, {0xc00053f980, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x20, 0xc001ea3980, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0xc0006ca340, {0xc00117f0c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x0, 0xc001ea3950, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x0, 0xc001ea3800, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x40ce34, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002561500}, {0x2076920, 0x203000}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x203000, 0xc001ea37d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002561500}, {0x2076920, 0x46}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, 0x7fb451d0c108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc002561500}, {0x2076920, 0x268ae5cf2}, {0x1f33320, 0xc00117f080}, {0x1f736c0, 0xc00122c630}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002561500}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0019e29c0, {0x2392298, 0xc002561500})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1082922 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001038050)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1379 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001b80f50)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1380 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000c9960, {0x2392228, 0xc001b7d8c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1381 [select, 13 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001b8e120, {0x2392228, 0xc001b7d900})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1382 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001b8e120, {0x2392228, 0xc001b7d900})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1290239 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000767200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0015564d0, {0x1f736c0, 0xc000dae090}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x78)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010abe00, 0xc001c055c0, 0xc001c05590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010abe00, {0x1f736c0, 0xc000dae090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0024cab40}, {0x2076920, 0x8}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x1c75901, {0xc0019a4440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x7fb451d0c5b8, {0xc0019a4440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x20, 0xc0009e4720, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0xc0006ca340, {0xc00101c7c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x0, 0xc0009e46f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x1e0cfc0, 0xc0009e4660, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0xc0bd44db08, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0024cab40}, {0x2076920, 0x203000}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x203000, 0xc0009e4630, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0024cab40}, {0x2076920, 0x46}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001b61650, {0x2392298, 0xc0024cab40}, {0x2076920, 0x100000000}, {0x1f33320, 0xc00101c780}, {0x1f736c0, 0xc000dae090}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0024cab40}, 0x1b616a0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0019e29c0, {0x2392298, 0xc0024cab40})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1277152 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001988c60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00077b1e0, {0x1f7c860, 0xc00077b130}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x63)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084ad00, 0xc001c3f5e8, 0xc001c3f5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084ad00, {0x1f7c860, 0xc00077b130})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00200e1e0}, {0x20768da, 0x8}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x1c75901, {0xc0008e6b40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x7fb451d0c5b8, {0xc0008e6b40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x20, 0xc0021f4a20, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0xc0007c0000, {0xc0021e7800, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x0, 0xc0021f49f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x1e0cfc0, 0xc0021f4960, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x6, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00200e1e0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0x8, 0xc0021f4930, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00200e1e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, 0xa01, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc00200e1e0}, {0x20768da, 0x0}, {0x1f33180, 0xc0021f48d0}, {0x1f7c860, 0xc00077b130}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc00200e1e0}, 0xc000ec1720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00181db00, {0x2392298, 0xc00200e1e0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1399 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00181db00, 0xc0021f0980)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00181db00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a780)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a780)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1400 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00181db00, 0xc001e1fac0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00181db00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00177a780)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00177a780)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1401 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00177a780)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1286366 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006525a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f3f0, {0x1f7c860, 0xc00205f340}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x63)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa700, 0xc001c635e8, 0xc001c635b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa700, {0x1f7c860, 0xc00205f340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0022268a0}, {0x20768da, 0x8}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x1c75901, {0xc001fa1860, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x7fb451d0c5b8, {0xc001fa1860, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x20, 0xc00219ea80, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0xc0007c0000, {0xc001e91200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x0, 0xc00219ea50, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x1e0cfc0, 0xc00219e9c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x60, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0022268a0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x8, 0xc00219e990, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0022268a0}, {0x20768da, 0x46}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, 0x144aacf, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc0022268a0}, {0x20768da, 0x0}, {0x1f33180, 0xc00219e930}, {0x1f7c860, 0xc00205f340}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc0022268a0}, 0xc000fa26e0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00181db00, {0x2392298, 0xc0022268a0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1082924 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0028d1c20, {0x2392228, 0xc000e8e3c0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266193 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0x0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286357 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00286bb00, {0x2392228, 0xc001d73e40})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1316414 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1286355 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000e88c30)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1286327 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000ded400)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1286305 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000645320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060ed10, {0x1f7b7c0, 0xc00156e820}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xbf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6500, 0xc001773bf8, 0xc001773bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6500, {0x1f7b7c0, 0xc00156e820})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001853800}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0x20, {0xc0021d7fc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001853800}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001853800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0xc0026446e0, {0xc0021d7fc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001853800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0xc0023ec2f0, 0xc001853830, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001853800}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d2ffc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0x237ac70, 0xc001853770, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d2ffc0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d2ffc0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001d2ff40}, {0x1f7b7c0, 0xc00156e820}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001d2ffc0}, 0xc00074d560, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0018530b0}, 0xc001d2ff40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0018530b0}, 0xc001d2ff40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001774480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001774528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ac8a208, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0018530b0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c001da9840})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001774778, {0x235a640, 0xc000045ad0}, 0x382d646265342d38)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0018530b0}, 0xc001d2fc40)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001c40db8, 0x4c1cb4}, {0xc057031280f0cf2f, 0xb3e837e6ae}, {0xc001c40db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0018530b0}, 0xc001d2fc40)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40}, 0xc0021d7e00, 0xc0022af548)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40}, 0x15, 0xc001d2fc80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40}, 0x2076951, 0xc001d2fc80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40}, 0x237bc38, 0xc001d2fc80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018530b0}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001852f60}, {0x1f33320, 0xc001d2fc40}, 0x1e5a940, 0xc001d2fc80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001852f60}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40}, 0x23c27b0, 0xc001d2fc80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40}, 0x5, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00260fb60, {0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001852f30}, {0x1f33320, 0xc001d2fc40}, 0xc0021d7e00, 0xc0022af548)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001852f30}, 0xc001b66780, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001961200, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001961200, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1268461 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1201979 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000c08f00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1358741 [runnable]:
github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent·dwrap·3()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1
created by github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133 +0x13a

goroutine 1266109 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc00177b1d0)
	/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
	/temporal/service/history/queueProcessor.go:158 +0x23b

goroutine 1286323 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000126120)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e000, {0x1f7c860, 0xc00060fce0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee6500, 0xc0017e75e8, 0xc0017e75b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee6500, {0x1f7c860, 0xc00060fce0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002216cc0}, {0x20768da, 0x8}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x1c75901, {0xc0008e6ee0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x7fb451d0c5b8, {0xc0008e6ee0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x20, 0xc0015cb6b0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0xc0007c0000, {0xc001dc2e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x0, 0xc0015cb680, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x1e0cfc0, 0xc0015cb5f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x60, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002216cc0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x8, 0xc0015cb5c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002216cc0}, {0x20768da, 0x46}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, 0x6c0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc002216cc0}, {0x20768da, 0x0}, {0x1f33180, 0xc0015cb560}, {0x1f7c860, 0xc00060fce0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc002216cc0}, 0xc00109cf20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00076c300, {0x2392298, 0xc002216cc0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1272432 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0024906c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb760, {0x1f7b560, 0xc00204b040}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000734500, 0xc001770440, 0xc001770410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000734500, {0x1f7b560, 0xc00204b040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e3e630}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0x20, {0xc001fa1960, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e3e630}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e3e630}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0xc001580580, {0xc001fa1960, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e3e630}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0xc0024e7e90, 0xc001e3e660, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e3e630}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0x8, {0xc00062b440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e2b8c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0x237ac70, 0xc001e3e5a0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e2b8c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e2b8c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e2b840}, {0x1f7b560, 0xc00204b040}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001e2b8c0}, 0xc001862756, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0024cbc20}, 0xc001e2b840, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0024cbc20}, 0xc001e2b840, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00242efc0, {0x2392298, 0xc0024cbc20})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00242f320, {0x2392298, 0xc0024cbc20}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002465200, {0x23922d0, 0xc0023ba120}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0023ba120}, 0xc001359fa0, 0x1bd58c0, 0x1359fb0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0023ba0c0, 0xc0023e8c40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0007b1c50}, 0xc0023e8c40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0007b1c50}, {0x1f353a0, 0xc0023e8c40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0007b1c50}, {0x1f353a0, 0xc0023e8c40}, 0x237bc38, 0xc00086a600)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007b1c50}, {0x1f353a0, 0xc0023e8c40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0007b1800, {0x23922d0, 0xc0007b1830}, {0x1f353a0, 0xc0023e8c40}, 0xc000c655f0, 0xc0023e8c80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007b1830}, {0x1f353a0, 0xc0023e8c40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0007b1830}, {0x1f353a0, 0xc0023e8c40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007b1830}, {0x1f353a0, 0xc0023e8c40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0007b17a0}, {0x1f353a0, 0xc0023e8c40}, 0x40, 0xc0023e8c80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007b17a0}, {0x1f353a0, 0xc0023e8c40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0007b17a0}, {0x1f353a0, 0xc0023e8c40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007b17a0}, {0x1f353a0, 0xc0023e8c40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0007b17a0}, {0x1f353a0, 0xc0023e8c40}, 0xc00247c540, 0xc00086a600)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0007b17a0}, 0xc0024cbbc0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c17680, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c17680, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1318928 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000a923c0, {0x2392298, 0xc0028c5500}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000378b40, {0x23922d0, 0xc001e0a1e0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0024106c0, {0x23922d0, 0xc001e0a1e0}, 0xc001160430, 0x1, 0x1160440)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001e0a180, 0xc001147000)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001d39ef0}, 0xc001147000)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001d39ef0}, {0x1f35540, 0xc001147000})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001d39ef0}, {0x1f35540, 0xc001147000}, 0x237bc38, 0xc0025dba58)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d39ef0}, {0x1f35540, 0xc001147000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7fb42ad30510, {0x23922d0, 0xc001d39da0}, {0x1f35540, 0xc001147000}, 0x0, 0xc001147080)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d39da0}, {0x1f35540, 0xc001147000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d39da0}, {0x1f35540, 0xc001147000}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d39da0}, {0x1f35540, 0xc001147000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d39d70}, {0x1f35540, 0xc001147000}, 0x40, 0xc001147080)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d39d70}, {0x1f35540, 0xc001147000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d39d70}, {0x1f35540, 0xc001147000}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d39d70}, {0x1f35540, 0xc001147000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d39d70}, {0x1f35540, 0xc001147000}, 0xc001fa1600, 0xc0025dba58)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001d39d70}, 0xc0028c54a0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00123fc20, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00123fc20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1272379 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020734a0, {0x2392298, 0xc0024ca8a0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019977a0, {0x23922d0, 0xc0021be630}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0027bd920, {0x23922d0, 0xc0021be630}, 0xc0023f0db0, 0x1, 0x23f0dd0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc0021be420, 0xc0006b6bc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc0021be150}, 0xc0006b6bc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021be150}, {0x1f35540, 0xc0006b6bc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0021be150}, {0x1f35540, 0xc0006b6bc0}, 0x237bc38, 0xc001c96978)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021be150}, {0x1f35540, 0xc0006b6bc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0021be000}, {0x1f35540, 0xc0006b6bc0}, 0x203000, 0xc0006b6d80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021be000}, {0x1f35540, 0xc0006b6bc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021be000}, {0x1f35540, 0xc0006b6bc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021be000}, {0x1f35540, 0xc0006b6bc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c57fb0}, {0x1f35540, 0xc0006b6bc0}, 0x40, 0xc0006b6d80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c57fb0}, {0x1f35540, 0xc0006b6bc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c57fb0}, {0x1f35540, 0xc0006b6bc0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c57fb0}, {0x1f35540, 0xc0006b6bc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c57fb0}, {0x1f35540, 0xc0006b6bc0}, 0xc001fa1d00, 0xc001c96978)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc000c57fb0}, 0xc0024ca7e0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc002481320, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc002481320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1282735 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b686c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb4d10, {0x1f7b560, 0xc001a55680}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968c00, 0xc00292dd18, 0xc00292dce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968c00, {0x1f7b560, 0xc001a55680})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e8e2a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0x20, {0xc0019a4660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e8e2a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e8e2a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0xc001632000, {0xc0019a4660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e8e2a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0xc0023ec2f0, 0xc001e8e2d0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e8e2a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d73f40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0x237ac70, 0xc001e8e210, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d73f40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d73f40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001d73ec0}, {0x1f7b560, 0xc001a55680}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001d73f40}, 0xc0008e4eb8, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001e83b30}, 0xc001d73ec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001e83b30}, 0xc001d73ec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc00292e5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00292e648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acdfb58, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001e83b30}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00209cb90})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00292e898, {0x235a640, 0xc000045ad0}, 0x392d623439342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001e83b30}, 0xc001e83980)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00158edb8, 0x4c1cb4}, {0xc057031242abf201, 0xb3ae583f83}, {0xc00158edb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001e83b30}, 0xc001e83980)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980}, 0xc0019a4520, 0xc001e883d8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980}, 0x15, 0xc001d73c80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980}, 0x207690b, 0xc001d73c80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980}, 0x237bc38, 0xc001d73c80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83b30}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001e839e0}, {0x1f33180, 0xc001e83980}, 0x1e5a940, 0xc001d73c80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e839e0}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980}, 0x9, 0xc001d73c80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980}, 0xc000a07768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a07b60, {0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980}, 0x40, 0xc00005fc00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e83950}, {0x1f33180, 0xc001e83980}, 0xc0019a4520, 0xc001e883d8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001e83950}, 0xc0008a8ae0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00123f0e0, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00123f0e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286304 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002036240, {0x2392298, 0xc0024ac360}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00196bb00, {0x23922d0, 0xc0021868d0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0021868d0}, 0xc001d7e080, 0x1bd58c0, 0x1d7e090)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc002186690, 0xc001dad140)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc002186270}, 0xc001dad140)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002186270}, {0x1f353a0, 0xc001dad140})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002186270}, {0x1f353a0, 0xc001dad140}, 0x237bc38, 0xc002771fc8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002186270}, {0x1f353a0, 0xc001dad140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7, {0x23922d0, 0xc00215dfb0}, {0x1f353a0, 0xc001dad140}, 0xc000dcbec0, 0xc001dad180)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215dfb0}, {0x1f353a0, 0xc001dad140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00215dfb0}, {0x1f353a0, 0xc001dad140}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215dfb0}, {0x1f353a0, 0xc001dad140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00215df50}, {0x1f353a0, 0xc001dad140}, 0x40, 0xc001dad180)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215df50}, {0x1f353a0, 0xc001dad140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00215df50}, {0x1f353a0, 0xc001dad140}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215df50}, {0x1f353a0, 0xc001dad140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00215df50}, {0x1f353a0, 0xc001dad140}, 0xc00247dce0, 0xc002771fc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc00215df50}, 0xc0024ac300, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc001b84480, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc001b84480, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266084 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568000, 0xc002372d40)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 120119 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0007366c0, {0x2392228, 0xc0006b7740})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1266139 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc001f90f00)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1319404 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1266086 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc00177ae10)
	/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
	/temporal/service/history/queueProcessor.go:158 +0x23b

goroutine 1266194 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0xc0005693f8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1358744 [runnable]:
github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent·dwrap·3()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1
created by github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133 +0x13a

goroutine 1266126 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc002422fc0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266072 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc000eda340)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266082 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568000, 0xc00185f8d0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266052 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69d90, 0xc0019d2a40)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266209 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0xc0025074c0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1277173 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002502f60, {0x2392228, 0xc002232fc0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1266203 [select, 1 minutes]:
go.temporal.io/server/service/history.(*timerQueueProcessorImpl).completeTimersLoop(0xc000328620)
	/temporal/service/history/timerQueueProcessor.go:274 +0x14f
created by go.temporal.io/server/service/history.(*timerQueueProcessorImpl).Start
	/temporal/service/history/timerQueueProcessor.go:166 +0x10f

goroutine 1266119 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0790, 0xc000a0b440)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266190 [select, 1 minutes]:
go.temporal.io/server/service/history.(*timerQueueProcessorImpl).completeTimersLoop(0xc000828540)
	/temporal/service/history/timerQueueProcessor.go:274 +0x14f
created by go.temporal.io/server/service/history.(*timerQueueProcessorImpl).Start
	/temporal/service/history/timerQueueProcessor.go:166 +0x10f

goroutine 1164326 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003d2930, {0x2392228, 0xc000ea0b80})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1290273 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a107e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001556580, {0x1f7b7c0, 0xc000020960}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010abf00, 0xc0027efbf8, 0xc0027efbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010abf00, {0x1f7b7c0, 0xc000020960})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0009e5980}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0x20, {0xc0019a4a00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0009e5980}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0009e5980}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0xc0015802c0, {0xc0019a4a00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0009e5980}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0xc0017985b0, 0xc0009e59b0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0009e5980}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00101cd40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0x237ac70, 0xc0009e5830, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00101cd40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00101cd40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00101ccc0}, {0x1f7b7c0, 0xc000020960}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc00101cd40}, 0xc0019578c0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0009e4d50}, 0xc00101ccc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0009e4d50}, 0xc00101ccc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0027f0480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0027f0528, {0x235a640, 0xc0000a8b10}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad3a388, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0009e4d50}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002781450})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc0027f0778, {0x235a640, 0xc000045ad0}, 0x382d376433342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0009e4d50}, 0xc00101c9c0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a58db8, 0x4c1cb4}, {0xc0570312c5526865, 0xb4283449e3}, {0xc001a58db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0009e4d50}, 0xc00101c9c0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0}, 0xc0019a4760, 0xc00086bc20)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0}, 0x15, 0xc00101ca00)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0}, 0x2076951, 0xc00101ca00)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0}, 0x237bc38, 0xc00101ca00)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4d50}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0009e4bd0}, {0x1f33320, 0xc00101c9c0}, 0x1e5a940, 0xc00101ca00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4bd0}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0}, 0x23c27b0, 0xc00101ca00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00194cb60, {0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0009e4ba0}, {0x1f33320, 0xc00101c9c0}, 0xc0019a4760, 0xc00086bc20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0009e4ba0}, 0xc0024cade0, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000767320, 0xc0009e06f0, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000767320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1201968 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00262ea50, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001157920)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1266068 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc000edac80)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266081 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568000, 0xc00185f8d0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286310 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001b66cc0, {0x2392228, 0xc001d46a40})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1286279 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000da9130)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1358745 [runnable]:
github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent·dwrap·3()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1
created by github.com/temporalio/ringpop-go/events.(*AsyncEventEmitter).EmitEvent
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/events/events.go:133 +0x13a

goroutine 120747 [IO wait, 12 minutes]:
internal/poll.runtime_pollWait(0x7fb42ad117e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00181d500, 0xc000ff4000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00181d500, {0xc000ff4000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00181d500, {0xc000ff4000, 0xa8a172, 0x0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000abc38, {0xc000ff4000, 0xa95530, 0x10})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002073560, {0xc000828d60, 0x9, 0xc000c1f200})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc002073560}, {0xc000828d60, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000828d60, 0x9, 0xc00208e6c8}, {0x23585c0, 0xc002073560})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000828d20)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0008234a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1170554 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00213cde0, {0x2392228, 0xc001dd8f40})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 120070 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00077a420)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 1266111 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc0003282a0)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1277076 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001ba47e0, {0x2392228, 0xc00217fb00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 120067 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 1266064 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc00185fce8)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286331 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024d1380, {0x2392298, 0xc0028f8120}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001b68b40, {0x23922d0, 0xc002116f00}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002116f00}, 0xc001dd45c0, 0x1bd58c0, 0x1dd45d0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc002116cf0, 0xc001e0ed80)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc002116a50}, 0xc001e0ed80)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002116a50}, {0x1f353a0, 0xc001e0ed80})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc002116a50}, {0x1f353a0, 0xc001e0ed80}, 0x237bc38, 0xc0022e2750)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002116a50}, {0x1f353a0, 0xc001e0ed80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002116900}, {0x1f353a0, 0xc001e0ed80}, 0x203000, 0xc001e0ee40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002116900}, {0x1f353a0, 0xc001e0ed80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002116900}, {0x1f353a0, 0xc001e0ed80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002116900}, {0x1f353a0, 0xc001e0ed80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0021168a0}, {0x1f353a0, 0xc001e0ed80}, 0x40, 0xc001e0ee40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021168a0}, {0x1f353a0, 0xc001e0ed80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0021168a0}, {0x1f353a0, 0xc001e0ed80}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021168a0}, {0x1f353a0, 0xc001e0ed80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0021168a0}, {0x1f353a0, 0xc001e0ed80}, 0xc0008e7780, 0xc0022e2750)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0021168a0}, 0xc0028f80c0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc00123eea0, 0xc000eeee10, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc00123eea0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286342 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00196ab40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205ed10, {0x1f736c0, 0xc000dadef0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x75)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee7e00, 0xc001a5f5c0, 0xc001a5f590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee7e00, {0x1f736c0, 0xc000dadef0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0028f9e00}, {0x2076920, 0x8}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x1c75901, {0xc001fa0560, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x7fb451d0c5b8, {0xc001fa0560, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x20, 0xc0020fb140, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0xc0006ca340, {0xc001d5f180, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x0, 0xc0020fb0e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x0, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x1e0cfc0, 0xc0020faf90, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0xc051a60d0c, {0xc0000849c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0028f9e00}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x203000, 0xc0020faf60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0028f9e00}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, 0x7fb451d0c5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000e35e50, {0x2392298, 0xc0028f9e00}, {0x2076920, 0x1d7dde1e5}, {0x1f33320, 0xc001d5f140}, {0x1f736c0, 0xc000dadef0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0028f9e00}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001821ba0, {0x2392298, 0xc0028f9e00})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1290279 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00164aea0, {0x2392298, 0xc0024cbe00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010830e0, {0x23922d0, 0xc000a22840}, 0x2d)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001957ef0, {0x23922d0, 0xc000a22840}, 0xc00184a590, 0x1, 0x184a5a0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc000a225a0, 0xc000eda440)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc000a221b0}, 0xc000eda440)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000a221b0}, {0x1f35540, 0xc000eda440})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc000a221b0}, {0x1f35540, 0xc000eda440}, 0x237bc38, 0xc000324750)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000a221b0}, {0x1f35540, 0xc000eda440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000c4bd10}, {0x1f35540, 0xc000eda440}, 0x203000, 0xc000eda500)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4bd10}, {0x1f35540, 0xc000eda440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c4bd10}, {0x1f35540, 0xc000eda440}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4bd10}, {0x1f35540, 0xc000eda440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c4bce0}, {0x1f35540, 0xc000eda440}, 0x40, 0xc000eda500)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4bce0}, {0x1f35540, 0xc000eda440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c4bce0}, {0x1f35540, 0xc000eda440}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c4bce0}, {0x1f35540, 0xc000eda440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c4bce0}, {0x1f35540, 0xc000eda440}, 0xc0019a5420, 0xc000324750)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc000c4bce0}, 0xc0024cbda0, 0xc000799a00)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc000a11e60, 0xc000eeee10, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc000a11e60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1286280 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000622a80, {0x2392228, 0xc001d2f2c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 120069 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00077a370)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 120163 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 1266117 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0790, 0xc000a0b440)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1266063 [select, 1 minutes]:
go.temporal.io/server/service/history.(*transferQueueProcessorImpl).completeTransferLoop(0xc001520180)
	/temporal/service/history/transferQueueProcessor.go:270 +0x125
created by go.temporal.io/server/service/history.(*transferQueueProcessorImpl).Start
	/temporal/service/history/transferQueueProcessor.go:166 +0xfb

goroutine 120072 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00077a2c0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 1266055 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69d90, 0xc00185fd58)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 120373 [select, 1 minutes]:
go.temporal.io/server/service/history/shard.(*ControllerImpl).shardManagementPump(0xc001980d20)
	/temporal/service/history/shard/controller_impl.go:312 +0x148
created by go.temporal.io/server/service/history/shard.(*ControllerImpl).Start
	/temporal/service/history/shard/controller_impl.go:145 +0xaf

goroutine 1272376 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000622230, {0x2392228, 0xc0009d9b40})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1266195 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0xc001361f80)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1272451 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000838240)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060f760, {0x1f7b560, 0xc000020820}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa2)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a600, 0xc00159bd18, 0xc00159bce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a600, {0x1f7b560, 0xc000020820})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000141590}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0x20, {0xc0019a4fa0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000141590}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000141590}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0xc000eb3b80, {0xc0019a4fa0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000141590}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0xc0023ec2f0, 0xc0001415c0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000141590}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0x8, {0xc000c5f1f0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0007733c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0x237ac70, 0xc000141500, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0007733c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0007733c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000773340}, {0x1f7b560, 0xc000020820}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc0007733c0}, 0xc00074c100, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc0010a3b90}, 0xc000773340, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc0010a3b90}, 0xc000773340, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00159c5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00159c648, {0x235a640, 0xc0000a8b10}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ae211b8, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001a31140, {0x23922d0, 0xc0010a3b90}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0006c6940})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00159c898, {0x235a640, 0xc000045ad0}, 0x392d396466342d35)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc0010a3b90}, 0xc0010a39e0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00165edb8, 0x4c1cb4}, {0xc0570311c31b3174, 0xb33791eaf6}, {0xc00165edb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc0010a3b90}, 0xc0010a39e0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0}, 0xc0019a4e20, 0xc0019a3218)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0}, 0x15, 0xc000773080)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0}, 0x207690b, 0xc000773080)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0}, 0x237bc38, 0xc000773080)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a3b90}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0010a3a40}, {0x1f33180, 0xc0010a39e0}, 0x1e5a940, 0xc000773080)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a3a40}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0}, 0x9, 0xc000773080)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0}, 0xc000a05768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a05b60, {0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0}, 0x40, 0xc0007c4800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0010a39b0}, {0x1f33180, 0xc0010a39e0}, 0xc0019a4e20, 0xc0019a3218)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc0010a39b0}, 0xc00275fb00, 0xc0007987c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000668360, 0xc0009e06f0, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc000668360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1266099 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc001d9c480)
	/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
	/temporal/service/history/taskProcessor.go:145 +0x2c

goroutine 1286303 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002036240, {0x2392298, 0xc000dcbe60}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00196bb00, {0x23922d0, 0xc00215dcb0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00215dcb0}, 0xc001d91c00, 0x1bd58c0, 0x1d91c10)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc00215daa0, 0xc001dacd80)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc00215d800}, 0xc001dacd80)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00215d800}, {0x1f353a0, 0xc001dacd80})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc00215d800}, {0x1f353a0, 0xc001dacd80}, 0x237bc38, 0xc002771c50)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215d800}, {0x1f353a0, 0xc001dacd80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7, {0x23922d0, 0xc00215d6b0}, {0x1f353a0, 0xc001dacd80}, 0xc000dcb380, 0xc001dacdc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215d6b0}, {0x1f353a0, 0xc001dacd80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00215d6b0}, {0x1f353a0, 0xc001dacd80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00215d6b0}, {0x1f353a0,