node-1

goroutine 1386454 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc00038a620})
	/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc00038a620}, 0x200)
	/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc00038a620}, 0xc)
	/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc001ea40d1, 0x34934e0}, {0x2386300, 0xc00038a620}, 0xc001ea40c4)
	/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc00038a620}, 0xc00016e300)
	/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc00038a620}, 0x0)
	/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc001ea40db, {0x2386300, 0xc00038a620}, 0xc00016e300)
	/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc001a3d2f0}, {0x2386300, 0xc00038a620}, 0xc00016e300)
	/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc000dee140, {0x23922d0, 0xc000586750})
	/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3033 +0x4e8

goroutine 1 [chan receive, 7 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc00051e000)
	/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc00053dcc0)
	/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc000485440, 0xc00053dc80)
	/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc0005311e0, {0x2392260, 0xc000040040}, {0xc00003a080, 0x8, 0x8})
	/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
	/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
	/temporal/cmd/server/main.go:50 +0x45

goroutine 8 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0000b6300)
	/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
	/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92

goroutine 9 [syscall, 7 minutes]:
os/signal.signal_recv()
	/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x2c

goroutine 10 [chan receive, 7 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
	/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
	/temporal/temporal/interrupt.go:38 +0xef

goroutine 11 [chan receive, 7 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
	/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
	/temporal/temporal/interrupt.go:38 +0xef

goroutine 97 [sleep]:
time.Sleep(0x306dc4200)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 50 [select]:
database/sql.(*DB).connectionOpener(0xc0008efe10, {0x2392228, 0xc000544c80})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 47 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00011cf00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 67 [select]:
database/sql.(*DB).connectionCleaner(0xc0008efe10, 0xc00011cf00)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 73 [select]:
database/sql.(*DB).connectionOpener(0xc0005beb60, {0x2392228, 0xc000667900})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 37 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf23d69f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000b7a00, 0x416f26, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0000b7a00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0000b7a00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000c078)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000c078)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc00038a380, {0x23860f0, 0xc00000c078})
	/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc00038a380)
	/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
	/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
	/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
	/temporal/common/pprof/pprof.go:72 +0x9c

goroutine 194 [select, 2 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0006a4e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 1073655 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a558, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0020ac100, 0xc002168000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020ac100, {0xc002168000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0020ac100, {0xc002168000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f918, {0xc002168000, 0x0, 0x40ce34})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000847d40, {0xc000792740, 0x9, 0x18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000847d40}, {0xc000792740, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000792740, 0x9, 0xc00003e8d0}, {0x23585c0, 0xc000847d40})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000792700)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00258ef00, 0xc0011fdf98, 0x1418c25)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000183880, {0x23d1088, 0xc00258ef00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 79 [select]:
database/sql.(*DB).connectionCleaner(0xc0005beb60, 0xc000593720)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 1059764 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000412620, {0x2392228, 0xc001053140})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 57 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6ad8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841b00, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000841b00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000841b00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000095a58)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000095a58)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0000854e0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000690600)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 58 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc0000da820, {0x2392228, 0xc00011b140})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 661066 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002005220)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 70 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6908, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841f80, 0xc00004e800, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000841f80)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000841f80)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000095dd0)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000095dd0)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000182fc0, {0x23860f0, 0xc000095dd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc000850fd0)
	/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc0005935e0}, 0x0)
	/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
	/temporal/service/history/fx.go:168 +0x9a

goroutine 71 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000593720)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 1245921 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000f2a5f0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 63 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 80 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 83 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6738, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003b200, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003b200)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003b200)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000cbd0)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000cbd0)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000574860)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000702200)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 86 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0006813b0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 85 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d6650, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003b500, 0xc00004e800, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003b500)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003b500)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000ced0)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000ced0)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0003fbc00, {0x23860f0, 0xc00000ced0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc000606fd0)
	/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc00068e990}, 0xc00082001c)
	/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
	/temporal/service/matching/fx.go:129 +0x9a

goroutine 110 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc000b245b0, {0x2392228, 0xc0006a1800})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 88 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc00083b5f0, {0x2392228, 0xc0007d6bc0})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 142 [sleep]:
time.Sleep(0x306dc4200)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 106 [select]:
database/sql.(*DB).connectionCleaner(0xc00083b5f0, 0x1c)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 109 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf23d60e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006d0a00, 0x532ca5, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0006d0a00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0006d0a00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0006c8870)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0006c8870)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000575040)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000702400)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 93 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
	/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
	/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8

goroutine 94 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 95 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 319 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 1250260 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ffefc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647970, {0x1f7b7c0, 0xc0015c20a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73d00, 0xc001747bf8, 0xc001747bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73d00, {0x1f7b7c0, 0xc0015c20a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x20, {0xc0016f7660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0xc000715b80, {0xc0016f7660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0xc00218fb70, 0xc000f9f710, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000f9f6e0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0017d9f00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x33, 0xc000f9f650, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0017d9f00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0017d9f00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0017d9e80}, {0x1f7b7c0, 0xc0015c20a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0017d9f00}, 0xc00249f6e0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc000f9ef90}, 0xc0017d9e80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc000f9ef90}, 0xc0017d9e80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001748480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001748528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21864e8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc000f9ef90}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c000fa0e00})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001748778, {0x235a640, 0xc0000a9fb0}, 0x392d316135342d32)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc000f9ef90}, 0xc0017d9b80)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000c0adb8, 0x4c1cb4}, {0xc057f6a228813d20, 0x6118cdc4f8}, {0xc000c0adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc000f9ef90}, 0xc0017d9b80)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0xc0016f7460, 0xc001efe258)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0x15, 0xc0017d9bc0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0x2076951, 0xc0017d9bc0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80}, 0x237bc38, 0xc0017d9bc0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ef90}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000f9ee40}, {0x1f33320, 0xc0017d9b80}, 0x1e5a940, 0xc0017d9bc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee40}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0x23c27b0, 0xc0017d9bc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00296db60, {0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0x40, 0xc00005ec00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000f9ee10}, {0x1f33320, 0xc0017d9b80}, 0xc0016f7460, 0xc001efe258)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000f9ee10}, 0xc0025f4b40, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00083cfc0, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00083cfc0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 404 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472b00)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 184 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207658, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000afbe00, 0xc000aeb950, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000afbe00, {0xc000aeb950, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000afbe00, {0xc000aeb950, 0xc00085af50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000663498, {0xc000aeb950, 0x46b213, 0x617f6333})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000663498}, {0xc000aeb950, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006fc2c0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 320 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 1250263 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000b43f0, {0x2392228, 0xc0027548c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 211 [chan receive, 7 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0x1000000010000)
	/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc0000880a0}, 0x0)
	/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
	/temporal/service/worker/fx.go:78 +0x9a

goroutine 147 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 148 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 1242817 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008e97a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac370, {0x1f7c860, 0xc001bac2c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73e00, 0xc001c975e8, 0xc001c975b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73e00, {0x1f7c860, 0xc001bac2c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001f1faa0}, {0x20768da, 0x8}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x1c75901, {0xc0016f67c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x7f6c191ae108, {0xc0016f67c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x20, 0xc00078a7b0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0xc000da4820, {0xc001faa7c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x0, 0xc00078a780, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x1e0cfc0, 0xc00078a6f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001f1faa0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x8, 0xc00078a6c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001f1faa0}, {0x20768da, 0x46}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc001f1faa0}, {0x20768da, 0x0}, {0x1f33180, 0xc00078a630}, {0x1f7c860, 0xc001bac2c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc001f1faa0}, 0xc000600720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00166f600, {0x2392298, 0xc001f1faa0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1242762 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001e289c0, {0x2392298, 0xc001f1ef60}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001789d40, {0x23922d0, 0xc00073ae10}, 0x2a)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002618b70, {0x23922d0, 0xc00073ae10}, 0xc000bd2ba0, 0x1, 0xbd2bb0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0007bbdd0, 0xc000edd840)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0007ba480}, 0xc000edd840)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0007ba480}, {0x1f35540, 0xc000edd840})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0007ba480}, {0x1f35540, 0xc000edd840}, 0x237bc38, 0xc001d475c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007ba480}, {0x1f35540, 0xc000edd840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0007f75b0, {0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840}, 0xc0010efb40, 0xc000edd8c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839380}, {0x1f35540, 0xc000edd840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840}, 0x40, 0xc000edd8c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000839260}, {0x1f35540, 0xc000edd840}, 0xc0016f6600, 0xc001d475c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000839260}, 0xc001f1ef00, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0017887e0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0017887e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250194 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001855320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac630, {0x1f7b560, 0xc000defa40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d200, 0xc00221fd18, 0xc00221fce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d200, {0x1f7b560, 0xc000defa40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000c23080}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x20, {0xc000ba13a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000c23080}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c23080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x7f6bf21ba780, {0xc000ba13a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c23080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0xc00218fb70, 0xc000c230b0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000c23080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001eb31c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x237ac70, 0xc000c22ff0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001eb31c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001eb31c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001eb3100}, {0x1f7b560, 0xc000defa40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001eb31c0}, 0xc0016ff140, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc000c22930}, 0xc001eb3100, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc000c22930}, 0xc001eb3100, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0022205a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc002220648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf218ede8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc000c22930}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00206ef40})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc002220898, {0x235a640, 0xc0000a9fb0}, 0x622d343166342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc000c22930}, 0xc000c22780)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001808db8, 0x4c1cb4}, {0xc057f6a2279fe76f, 0x6117ec6f39}, {0xc001808db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc000c22930}, 0xc000c22780)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0xc000ba1240, 0xc000899440)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0x15, 0xc001eb2ec0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0x207690b, 0xc001eb2ec0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780}, 0x237bc38, 0xc001eb2ec0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22930}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000c227e0}, {0x1f33180, 0xc000c22780}, 0x1e5a940, 0xc001eb2ec0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c227e0}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0x9, 0xc001eb2ec0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0xc0007f7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0x40, 0xc00005ec00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c22750}, {0x1f33180, 0xc000c22780}, 0xc000ba1240, 0xc000899440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000c22750}, 0xc0021cb860, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001854000, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001854000, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 401 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00040c420)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 318 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 189 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 1254954 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0022d7680, {0x2392228, 0xc001fabb40})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250117 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077f320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0f600, {0x1f7c860, 0xc001a0ea50}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9800, 0xc000ee35e8, 0xc000ee35b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9800, {0x1f7c860, 0xc001a0ea50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00275c000}, {0x20768da, 0x8}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x1c75901, {0xc001880f60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x7f6c191ae5b8, {0xc001880f60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x20, 0xc0016b3320, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0xc000da4820, {0xc000ffda80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x0, 0xc0016b32f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x1e0cfc0, 0xc0016b3260, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00275c000}, {0x20768da, 0x203000}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x8, 0xc0016b3230, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00275c000}, {0x20768da, 0x46}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc00275c000}, {0x20768da, 0x0}, {0x1f33180, 0xc0016b31d0}, {0x1f7c860, 0xc001a0ea50}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc00275c000}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00003b480, {0x2392298, 0xc00275c000})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1254950 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d2ea20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0630, {0x1f7b560, 0xc0015c3220}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3da00, 0xc0019ce440, 0xc0019ce410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3da00, {0x1f7b560, 0xc0015c3220})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00078a720}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x20, {0xc0013e8b00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00078a720}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00078a720}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x7f6bf23dd368, {0xc0013e8b00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00078a720}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0xc00218ff30, 0xc00078a750, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00078a720}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fabc80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x237ac70, 0xc00078a690, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fabc80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fabc80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001fabc00}, {0x1f7b560, 0xc0015c3220}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc001fabc80}, 0xc000594ea6, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0022d7aa0}, 0xc001fabc00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0022d7aa0}, 0xc001fabc00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0022d77a0, {0x2392298, 0xc0022d7aa0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0022d7a40, {0x2392298, 0xc0022d7aa0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001d2e120, {0x23922d0, 0xc000784e10}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000784e10}, 0xc0007573a0, 0x1bd58c0, 0x7573c0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc000784db0, 0xc001fab140)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000784b10}, 0xc001fab140)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000784b10}, {0x1f353a0, 0xc001fab140})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000784b10}, {0x1f353a0, 0xc001fab140}, 0x237bc38, 0xc00086c858)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000784b10}, {0x1f353a0, 0xc001fab140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00066adb0, {0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140}, 0xc0007a1950, 0xc001fab1c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ade0}, {0x1f353a0, 0xc001fab140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140}, 0x40, 0xc001fab1c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00066ad50}, {0x1f353a0, 0xc001fab140}, 0xc0013e89c0, 0xc00086c858)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00066ad50}, 0xc0022d7440, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c51200, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c51200, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 185 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006fc2c0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 337 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730210)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 338 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730000)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 1270884 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001be10e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbc840, {0x1f736c0, 0xc00132cea0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8b00, 0xc0015af5c0, 0xc0015af590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8b00, {0x1f736c0, 0xc00132cea0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000efc8a0}, {0x2076920, 0x8}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x1c75901, {0xc001b58400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x7f6c191ae108, {0xc001b58400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x20, 0xc00133d4d0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0xc000d3fa00, {0xc001327d40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x0, 0xc00133d4a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x0, 0xc00133d410, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000efc8a0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x203000, 0xc00133d3e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000efc8a0}, {0x2076920, 0x46}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc000efc8a0}, {0x2076920, 0x200efa298}, {0x1f33320, 0xc001327d00}, {0x1f736c0, 0xc00132cea0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc000efc8a0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc000efc8a0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 587 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00040def0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 366 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 1270930 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0018f1d40, {0x2392298, 0xc000ebe420}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019e0900, {0x23922d0, 0xc0015ed4a0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001898240, {0x23922d0, 0xc0015ed4a0}, 0xc0015d4d20, 0x1, 0x15d4d30)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0015ed440, 0xc0015d6e40)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0015ed1d0}, 0xc0015d6e40)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015ed1d0}, {0x1f35540, 0xc0015d6e40})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0015ed1d0}, {0x1f35540, 0xc0015d6e40}, 0x237bc38, 0xc002423440)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed1d0}, {0x1f35540, 0xc0015d6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40}, 0xc0015d6500, 0xc0015d6ec0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed080}, {0x1f35540, 0xc0015d6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40}, 0x40, 0xc0015d6ec0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015ed050}, {0x1f35540, 0xc0015d6e40}, 0xc001b59ca0, 0xc002423440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0015ed050}, 0xc000ebe3c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a9560, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a9560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1255012 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002663aa0, {0x2392228, 0xc001fe3900})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 271 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf22071d0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0005adb80, 0xc00051c640, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005adb80, {0xc00051c640, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0005adb80, {0xc00051c640, 0xc000cb6f00, 0xc0005782a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096798, {0xc00051c640, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000096798}, {0xc00051c640, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00072d080, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 581 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 269 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207000, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841900, 0xc00051c530, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000841900, {0xc00051c530, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000841900, {0xc00051c530, 0xc000b90f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000967a0, {0xc00051c530, 0x46b213, 0x617f6332})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000967a0}, {0xc00051c530, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00072cdc0, 0x678630)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 270 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00072cdc0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 1271024 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 582 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 181 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf22073a0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000afbf80, 0xc000aeb760, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000afbf80, {0xc000aeb760, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000afbf80, {0xc000aeb760, 0xc0015d0960, 0xc000756c70})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0006634a0, {0xc000aeb760, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0006634a0}, {0xc000aeb760, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006fc000, 0xbd3220)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 182 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006fc000, 0x690600)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 193 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf23d5ff8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006d0d00, 0xc00004c000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0006d0d00)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0006d0d00)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0006c8b70)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0006c8b70)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000183880, {0x23860f0, 0xc0006c8b70})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0xc000af5fd0)
	/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc0006d0080}, 0x0)
	/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
	/temporal/service/frontend/fx.go:282 +0x9a

goroutine 197 [select, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc000b25a00, {0x2392228, 0xc000544d40})
	/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
	/usr/local/go/src/database/sql/sql.go:794 +0x188

goroutine 190 [select]:
database/sql.(*DB).connectionCleaner(0xc000b25a00, 0x1fef77d)
	/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
	/usr/local/go/src/database/sql/sql.go:1055 +0x105

goroutine 209 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7f6bf2207910, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce0680, 0xc00004e800, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000ce0680)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000ce0680)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0008cdba8)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0008cdba8)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000ba0c40)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000690200)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef

goroutine 212 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
	/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
	/temporal/common/metrics/runtime.go:142 +0x75

goroutine 204 [sleep]:
time.Sleep(0x37e11d600)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 1254955 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002016c00, {0x2392298, 0xc0022d7e60}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005f37a0, {0x23922d0, 0xc000752390}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000752390}, 0xc000ea1020, 0x1bd58c0, 0xea1040)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc000636990, 0xc001fabf80)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000636570}, 0xc001fabf80)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000636570}, {0x1f353a0, 0xc001fabf80})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000636570}, {0x1f353a0, 0xc001fabf80}, 0x237bc38, 0xc00086cd98)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000636570}, {0x1f353a0, 0xc001fabf80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80}, 0x203000, 0xc001fabfc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006363f0}, {0x1f353a0, 0xc001fabf80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80}, 0x40, 0xc001fabfc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000636390}, {0x1f353a0, 0xc001fabf80}, 0xc0013e8d40, 0xc00086cd98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000636390}, 0xc0022d7e00, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001d2eb40, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001d2eb40, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 465 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730370)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 218 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 225 [sleep]:
time.Sleep(0x37e11d600)
	/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
	/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
	/temporal/common/membership/rpMonitor.go:284 +0x77

goroutine 217 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 1049 [select, 7 minutes]:
net/http.(*persistConn).readLoop(0xc0010fd7a0)
	/usr/local/go/src/net/http/transport.go:2207 +0xd8a
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 272 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00072d080, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 340 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730160)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 339 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007300b0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 367 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 368 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 661067 [select, 3 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000fe4d0, {0x2392228, 0xc002042200})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 586 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0004728f0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 402 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0004729a0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 403 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472a50)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 1254935 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0016d2000, {0x2392228, 0xc000966c80})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 266 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a8f8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841880, 0xc0005409b0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000841880, {0xc0005409b0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000841880, {0xc0005409b0, 0xc001f97920, 0xc0007574a0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096778, {0xc0005409b0, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000096778}, {0xc0005409b0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00061c000, 0x6782f0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 267 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00061c000, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 229 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a9e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000841800, 0xc0003d5be0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000841800, {0xc0003d5be0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000841800, {0xc0003d5be0, 0xc000903f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096730, {0xc0003d5be0, 0x46b213, 0x617f6333})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc000096730}, {0xc0003d5be0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000700840, 0x1fe0001)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 230 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000700840, 0x1c)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 301 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000584580, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 232 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105

goroutine 233 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8

goroutine 300 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221ad80, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1800, 0xc000616040, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1800, {0xc000616040, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1800, {0xc000616040, 0xc000900f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f118, {0xc000616040, 0x46b213, 0x617f6333})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f118}, {0xc000616040, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000584580, 0x5ba530)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 583 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 466 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730420)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 304 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a2a0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1700, 0xc0006161b0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1700, {0xc0006161b0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1700, {0xc0006161b0, 0xc000f22a80, 0xc0005784f0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f0f0, {0xc0006161b0, 0x46b213, 0x3493960})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f0f0}, {0xc0006161b0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000584b00, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 398 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 1245925 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002166d80, {0x2392298, 0xc00247a240}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fc6c0, {0x23922d0, 0xc002409230}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc002409230}, 0xc00184c740, 0x1bd58c0, 0x184c750)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc002409020, 0xc000f27340)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc002408d80}, 0xc000f27340)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002408d80}, {0x1f353a0, 0xc000f27340})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc002408d80}, {0x1f353a0, 0xc000f27340}, 0x237bc38, 0xc0022f85a0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408d80}, {0x1f353a0, 0xc000f27340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340}, 0x203000, 0xc000f27380)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408c30}, {0x1f353a0, 0xc000f27340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340}, 0x40, 0xc000f27380)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002408bd0}, {0x1f353a0, 0xc000f27340}, 0xc0013e9540, 0xc0022f85a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc002408bd0}, 0xc00247a1e0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc000886c60, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc000886c60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245961 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00177c360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0016460b0, {0x1f7b560, 0xc00068a320}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036600, 0xc001605d18, 0xc001605ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036600, {0x1f7b560, 0xc00068a320})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x20, {0xc0013e99a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0xc001852840, {0xc0013e99a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0xc00218fb70, 0xc00132fe00, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00132fdd0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000f38d40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x237ac70, 0xc00132fd40, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000f38d40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000f38d40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000f38cc0}, {0x1f7b560, 0xc00068a320}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000f38d40}, 0xc001ba8c00, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc00132f500}, 0xc000f38cc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc00132f500}, 0xc000f38cc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0016065a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001606648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20d7388, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc00132f500}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00189eb80})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001606898, {0x235a640, 0xc0000a9fb0}, 0x612d313030342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc00132f500}, 0xc00132f350)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001caadb8, 0x4c1cb4}, {0xc057f6a1ea49df62, 0x60defb9d2e}, {0xc001caadb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc00132f500}, 0xc00132f350)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0xc0013e98e0, 0xc0022f9560)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0x15, 0xc000f38a80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0x207690b, 0xc000f38a80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350}, 0x237bc38, 0xc000f38a80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f500}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00132f3b0}, {0x1f33180, 0xc00132f350}, 0x1e5a940, 0xc000f38a80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f3b0}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0x9, 0xc000f38a80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0xc002993768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002993b60, {0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00132f320}, {0x1f33180, 0xc00132f350}, 0xc0013e98e0, 0xc0022f9560)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00132f320}, 0xc00247afc0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000830480, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000830480, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 302 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221ae68, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce0d80, 0xc0006161a0, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce0d80, {0xc0006161a0, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce0d80, {0xc0006161a0, 0xc000901f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f110, {0xc0006161a0, 0x46b213, 0x617f6333})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f110}, {0xc0006161a0, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000584840, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 303 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000584840, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 584 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472790)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 568 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 467 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0007304d0)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 468 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000730580)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 305 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000584b00, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 399 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132

goroutine 400 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
	/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70

goroutine 1245991 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000fb65f0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1255008 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001788b40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbc6e0, {0x1f7b560, 0xc000cd5860}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016f900, 0xc0026cc440, 0xc0026cc410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016f900, {0x1f7b560, 0xc000cd5860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00059def0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x20, {0xc000fe9800, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00059def0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00059def0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0xc001c74840, {0xc000fe9800, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00059def0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0xc00218ff30, 0xc00059df20, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00059def0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fe3a40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x237ac70, 0xc00059de60, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fe3a40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fe3a40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001fe39c0}, {0x1f7b560, 0xc000cd5860}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc001fe3a40}, 0xc001640006, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc002663e00}, 0xc001fe39c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc002663e00}, 0xc001fe39c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002663b60, {0x2392298, 0xc002663e00})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002663da0, {0x2392298, 0xc002663e00}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001788240, {0x23922d0, 0xc00059c630}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00059c630}, 0xc001300a70, 0x1bd58c0, 0x1300a90)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00059c330, 0xc001fe3140)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000683020}, 0xc001fe3140)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000683020}, {0x1f353a0, 0xc001fe3140})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000683020}, {0x1f353a0, 0xc001fe3140}, 0x237bc38, 0xc0020b3b90)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000683020}, {0x1f353a0, 0xc001fe3140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc000682e70, {0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140}, 0x7f6bf2166268, 0xc001fe3180)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682ea0}, {0x1f353a0, 0xc001fe3140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140}, 0x40, 0xc001fe3180)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000682e10}, {0x1f353a0, 0xc001fe3140}, 0xc000fe9620, 0xc0020b3b90)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000682e10}, 0xc0026638c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0005d2120, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0005d2120, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 585 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000472840)
	/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
	/temporal/common/membership/rpServiceResolver.go:127 +0x27b

goroutine 297 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a1b8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1780, 0xc000616030, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1780, {0xc000616030, 0x10, 0x10})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1780, {0xc000616030, 0xc000b71f50, 0x1449ef8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f0f8, {0xc000616030, 0x46b213, 0x617f6333})
	/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f0f8}, {0xc000616030, 0x10, 0x10}, 0x10)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc000700dc0, 0x5ba0e0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4

goroutine 298 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc000700dc0, 0x0)
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
	/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348

goroutine 387 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 386 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001c8fc0, {0x2392228, 0xc0007bf540})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1270886 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00083d440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbc8f0, {0x1f7b7c0, 0xc001a11220}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8c00, 0xc001615bf8, 0xc001615bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8c00, {0x1f7b7c0, 0xc001a11220})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x20, {0xc001b58640, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0xc001bca580, {0xc001b58640, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0xc00218fb70, 0xc001346120, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013460f0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013442c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x237ac70, 0xc001346060, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013442c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013442c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001344240}, {0x1f7b7c0, 0xc001a11220}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0013442c0}, 0xc000e28680, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00133d9e0}, 0xc001344240, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00133d9e0}, 0xc001344240, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001616480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001616528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20dc980, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00133d9e0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001fbfbe0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001616778, {0x235a640, 0xc0000a9fb0}, 0x622d646363342d31)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00133d9e0}, 0xc001327f40)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0018b2db8, 0x4c1cb4}, {0xc057f6a3664625cd, 0x6240989f9c}, {0xc0018b2db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00133d9e0}, 0xc001327f40)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0xc001b58520, 0xc001eedb30)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0x15, 0xc001327f80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0x2076951, 0xc001327f80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40}, 0x237bc38, 0xc001327f80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d9e0}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00133d890}, {0x1f33320, 0xc001327f40}, 0x1e5a940, 0xc001327f80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d890}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0x23c27b0, 0xc001327f80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00133d860}, {0x1f33320, 0xc001327f40}, 0xc001b58520, 0xc001eedb30)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00133d860}, 0xc000efcae0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001be1200, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001be1200, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242712 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001a70fc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0c60, {0x1f7b7c0, 0xc000dee8c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016fd00, 0xc00080a4d0, 0xc00080a4a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016fd00, {0x1f7b7c0, 0xc000dee8c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x20, {0xc0016f7560, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0xc001fb0dc0, {0xc0016f7560, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0xc001fb6920, 0xc001e91950, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e91920}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001dfd780}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x237ac70, 0xc001e91890, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001dfd780}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001dfd780}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001dfd6c0}, {0x1f7b7c0, 0xc000dee8c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000097128, {0x2392228, 0xc001dfd780}, 0xc001640a56, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc00068e150, {0x2392298, 0xc001ee2ea0}, 0xc001dfd6c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc00062aa20, {0x2392298, 0xc001ee2ea0}, 0xc001dfd6c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001ee2b40, {0x2392298, 0xc001ee2ea0})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001ee2e40, {0x2392298, 0xc001ee2ea0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a70900, {0x23922d0, 0xc001e90d50}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0016409f0, {0x23922d0, 0xc001e90d50}, 0xc001dbb920, 0x1, 0x1dbb930)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001e90cf0, 0xc001dfcdc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001e90a50}, 0xc001dfcdc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e90a50}, {0x1f35540, 0xc001dfcdc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001e90a50}, {0x1f35540, 0xc001dfcdc0}, 0x237bc38, 0xc00237f080)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e90a50}, {0x1f35540, 0xc001dfcdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0}, 0xc001dfc840, 0xc001dfce40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e90900}, {0x1f35540, 0xc001dfcdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0}, 0x40, 0xc001dfce40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0}, 0x203000, 0x203000)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e908d0}, {0x1f35540, 0xc001dfcdc0}, 0xc0016f7400, 0xc00237f080)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001e908d0}, 0xc001ee2840, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001d89e60, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001d89e60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 941 [select]:
go.temporal.io/server/service/history/shard.(*ControllerImpl).shardManagementPump(0xc00068a500)
	/temporal/service/history/shard/controller_impl.go:312 +0x148
created by go.temporal.io/server/service/history/shard.(*ControllerImpl).Start
	/temporal/service/history/shard/controller_impl.go:145 +0xaf

goroutine 1245920 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000886a20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001badd90, {0x1f7b560, 0xc000cd5f40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036400, 0xc001b84440, 0xc001b84410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036400, {0x1f7b560, 0xc000cd5f40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002408750}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x20, {0xc0013e9420, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002408750}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002408750}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0xc000db6160, {0xc0013e9420, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002408750}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0xc001fb6920, 0xc002408780, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002408750}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000f27040}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x237ac70, 0xc0024086c0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000f27040}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000f27040}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000f26fc0}, {0x1f7b560, 0xc000cd5f40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc000f27040}, 0xc0016b6f36, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0025afda0}, 0xc000f26fc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0025afda0}, 0xc000f26fc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0025afb00, {0x2392298, 0xc0025afda0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0025afd40, {0x2392298, 0xc0025afda0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a61e60, {0x23922d0, 0xc001fd79e0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fd79e0}, 0xc0011f58f0, 0x1bd58c0, 0x11f5900)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001fd7980, 0xc000f26740)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001fd76e0}, 0xc000f26740)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001fd76e0}, {0x1f353a0, 0xc000f26740})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001fd76e0}, {0x1f353a0, 0xc000f26740}, 0x237bc38, 0xc0022f8078)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd76e0}, {0x1f353a0, 0xc000f26740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001fd7560, {0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740}, 0xc001fd68a0, 0xc000f26780)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7590}, {0x1f353a0, 0xc000f26740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740}, 0x40, 0xc000f26780)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001fd7500}, {0x1f353a0, 0xc000f26740}, 0xc0013e9300, 0xc0022f8078)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001fd7500}, 0xc0025af860, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a61320, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a61320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270931 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0016d2540, {0x2392298, 0xc000ebe300}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0016c1680, {0x23922d0, 0xc0015ecf60}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001898090, {0x23922d0, 0xc0015ecf60}, 0xc0015d4b00, 0x1, 0x15d4b10)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0015ecf00, 0xc0015d6b00)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0015ecc90}, 0xc0015d6b00)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015ecc90}, {0x1f35540, 0xc0015d6b00})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0015ecc90}, {0x1f35540, 0xc0015d6b00}, 0x237bc38, 0xc002423290)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecc90}, {0x1f35540, 0xc0015d6b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00}, 0xc0015d6500, 0xc0015d6b80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb40}, {0x1f35540, 0xc0015d6b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00}, 0x40, 0xc0015d6b80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015ecb10}, {0x1f35540, 0xc0015d6b00}, 0xc001b59bc0, 0xc002423290)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0015ecb10}, 0xc000ebe2a0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a97a0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008a97a0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 499 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 498 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0006186c0, {0x2392228, 0xc000e7e640})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1068202 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002371ea0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1255013 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0022c8ea0, {0x2392298, 0xc0023e83c0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fcd80, {0x23922d0, 0xc00070ae40}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00070ae40}, 0xc0007d8280, 0x1bd58c0, 0x7d8290)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00070abd0, 0xc001fe3e40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc00070a840}, 0xc001fe3e40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00070a840}, {0x1f353a0, 0xc001fe3e40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00070a840}, {0x1f353a0, 0xc001fe3e40}, 0x237bc38, 0xc001f9e0f0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a840}, {0x1f353a0, 0xc001fe3e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40}, 0x203000, 0xc001fe3e80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a690}, {0x1f353a0, 0xc001fe3e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40}, 0x40, 0xc001fe3e80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00070a5a0}, {0x1f353a0, 0xc001fe3e40}, 0xc000fe9ac0, 0xc001f9e0f0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00070a5a0}, 0xc0023e8360, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001788c60, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001788c60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 775 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
	/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
	/temporal/common/namespace/registry.go:344 +0x106

goroutine 763 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001c8d80, {0x2392228, 0xc000ef6340})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 764 [select, 7 minutes]:
go.temporal.io/server/service/history/events.(*NotifierImpl).dequeueHistoryEventNotifications(0xc000680370)
	/temporal/service/history/events/notifier.go:234 +0x105
created by go.temporal.io/server/service/history/events.(*NotifierImpl).Start
	/temporal/service/history/events/notifier.go:253 +0x71

goroutine 1250119 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016e4900)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0f6b0, {0x1f7b560, 0xc00068bb80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9900, 0xc000d61d18, 0xc000d61ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9900, {0x1f7b560, 0xc00068bb80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x20, {0xc0018811a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0xc000ec8c60, {0xc0018811a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0xc00218fb70, 0xc0016d0060, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0016d0030}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000ffdf40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x237ac70, 0xc0016b3f80, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000ffdf40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000ffdf40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000ffdec0}, {0x1f7b560, 0xc00068bb80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000ffdf40}, 0xc000ff86c0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc0016b38c0}, 0xc000ffdec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc0016b38c0}, 0xc000ffdec0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000d625a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000d62648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d6cc8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc0016b38c0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00074b560})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000d62898, {0x235a640, 0xc0000a9fb0}, 0x392d643232342d61)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc0016b38c0}, 0xc0016b3710)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001bd0db8, 0x4c1cb4}, {0xc057f6a22685af7e, 0x6116d23755}, {0xc001bd0db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc0016b38c0}, 0xc0016b3710)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0xc0018810e0, 0xc001668108)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0x15, 0xc000ffdc80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0x207690b, 0xc000ffdc80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710}, 0x237bc38, 0xc000ffdc80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b38c0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0016b3770}, {0x1f33180, 0xc0016b3710}, 0x1e5a940, 0xc000ffdc80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b3770}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0x9, 0xc000ffdc80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0xc0007f7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0016b36b0}, {0x1f33180, 0xc0016b3710}, 0xc0018810e0, 0xc001668108)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0016b36b0}, 0xc00275c240, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077f440, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077f440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1059766 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0020c0960, {0x2392228, 0xc001053180})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1050 [select, 7 minutes]:
net/http.(*persistConn).writeLoop(0xc0010fd7a0)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 1064362 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002620460)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1073651 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00205d4a0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001da53e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 647 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001c9200, {0x2392228, 0xc000ef6100})
	/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 648 [select, 2 minutes]:
go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).purgeProcessor(0xc000124150)
	/temporal/common/persistence/namespaceReplicationQueue.go:404 +0xed
created by go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).Start
	/temporal/common/persistence/namespaceReplicationQueue.go:123 +0x71

goroutine 649 [select, 7 minutes]:
go.temporal.io/server/service/frontend.(*VersionChecker).versionCheckLoop(0xc0006a4d20)
	/temporal/service/frontend/versionChecker.go:88 +0xe5
created by go.temporal.io/server/service/frontend.(*VersionChecker).Start.func1
	/temporal/service/frontend/versionChecker.go:70 +0x5f

goroutine 1270897 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017ce120, {0x2392298, 0xc000efd8c0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0017898c0, {0x23922d0, 0xc00136c9f0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001f5fce0, {0x23922d0, 0xc00136c9f0}, 0xc001341b60, 0x1, 0x1341b70)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc00136c990, 0xc0013456c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc00136c720}, 0xc0013456c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00136c720}, {0x1f35540, 0xc0013456c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00136c720}, {0x1f35540, 0xc0013456c0}, 0x237bc38, 0xc001f90c00)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c720}, {0x1f35540, 0xc0013456c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0}, 0xc001345040, 0xc001345740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5d0}, {0x1f35540, 0xc0013456c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0}, 0x40, 0xc001345740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00136c5a0}, {0x1f35540, 0xc0013456c0}, 0xc001b58ba0, 0xc001f90c00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00136c5a0}, 0xc000efd860, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000e5d8c0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000e5d8c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245874 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008f3320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1c30, {0x1f7b560, 0xc001769b80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8100, 0xc00168e440, 0xc00168e410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8100, {0x1f7b560, 0xc001769b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x20, {0xc000fe9740, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0xc00176e2c0, {0xc000fe9740, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0xc001fb6920, 0xc001edaa80, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001edaa50}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e5b080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x237ac70, 0xc001eda960, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e5b080}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e5b080}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e5b000}, {0x1f7b560, 0xc001769b80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc000e5b080}, 0xc001cf6486, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0025ae5a0}, 0xc000e5b000, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0025ae5a0}, 0xc000e5b000, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002016180, {0x2392298, 0xc0025ae5a0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020164e0, {0x2392298, 0xc0025ae5a0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005de360, {0x23922d0, 0xc001fff050}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001fff050}, 0xc00117ceb0, 0x1bd58c0, 0x117cec0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001ffeff0, 0xc000f12080)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001ffed50}, 0xc000f12080)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ffed50}, {0x1f353a0, 0xc000f12080})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001ffed50}, {0x1f353a0, 0xc000f12080}, 0x237bc38, 0xc0025942d0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffed50}, {0x1f353a0, 0xc000f12080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001ffebd0, {0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080}, 0x7f6bf2195f30, 0xc000f120c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffec00}, {0x1f353a0, 0xc000f12080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080}, 0x40, 0xc000f120c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ffeb70}, {0x1f353a0, 0xc000f12080}, 0xc0013e8de0, 0xc0025942d0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001ffeb70}, 0xc0025ae540, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ef1440, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ef1440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250270 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0025f5b60, {0x2392228, 0xc0027555c0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1245994 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc000ba58c0, {0x2392228, 0xc000fca200})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1073579 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a470, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001f8b480, 0xc001ef2000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f8b480, {0xc001ef2000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001f8b480, {0xc001ef2000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f808, {0xc001ef2000, 0x466a4e, 0x4383a7})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002337e60, {0xc00038a040, 0x9, 0xc001a23d18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc002337e60}, {0xc00038a040, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038a040, 0x9, 0xc001a23d6f}, {0x23585c0, 0xc002337e60})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038a000)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0005c0000)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1250269 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0025f5b60, {0x2392228, 0xc0027555c0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1270927 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ffeb40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcdc0, {0x1f7b7c0, 0xc000dee1e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9b00, 0xc0010e1bf8, 0xc0010e1bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9b00, {0x1f7b7c0, 0xc000dee1e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x20, {0xc001b59760, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0xc0015c6000, {0xc001b59760, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0xc00218fb70, 0xc0015bf8f0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015bf8c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0015d6140}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x33, 0xc0015bf830, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0015d6140}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0015d6140}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0015d60c0}, {0x1f7b7c0, 0xc000dee1e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0015d6140}, 0xc000e29000, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0015bf1a0}, 0xc0015d60c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0015bf1a0}, 0xc0015d60c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0010e2480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0010e2528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2180980, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0015bf1a0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e0cfd0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0010e2778, {0x235a640, 0xc0000a9fb0}, 0x612d666564342d63)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0015bf1a0}, 0xc00137bdc0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000bbcdb8, 0x4c1cb4}, {0xc057f6a366b64ed9, 0x624108c8a5}, {0xc000bbcdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0015bf1a0}, 0xc00137bdc0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0xc001b59600, 0xc0024229a8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0x15, 0xc00137be00)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0x2076951, 0xc00137be00)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0}, 0x237bc38, 0xc00137be00)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf1a0}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0015bf050}, {0x1f33320, 0xc00137bdc0}, 0x1e5a940, 0xc00137be00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf050}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0x23c27b0, 0xc00137be00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015bf020}, {0x1f33320, 0xc00137bdc0}, 0xc001b59600, 0xc0024229a8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0015bf020}, 0xc0017fdb00, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0010fd8c0, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0010fd8c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245896 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000846de0, {0x2392298, 0xc0021adb00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001ba4000, {0x23922d0, 0xc001edb410}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001edb410}, 0xc0012e3c10, 0x1bd58c0, 0x12e3c20)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001edb200, 0xc000e5b280)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001edaf60}, 0xc000e5b280)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001edaf60}, {0x1f353a0, 0xc000e5b280})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001edaf60}, {0x1f353a0, 0xc000e5b280}, 0x237bc38, 0xc001cfe7f8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edaf60}, {0x1f353a0, 0xc000e5b280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001635710, {0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280}, 0xc001634a50, 0xc000e5b2c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edae10}, {0x1f353a0, 0xc000e5b280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280}, 0x40, 0xc000e5b2c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001edadb0}, {0x1f353a0, 0xc000e5b280}, 0xc000fe9880, 0xc001cfe7f8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001edadb0}, 0xc0021adaa0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0008f3440, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0008f3440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1073675 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a640, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0020ac400, 0xc002226000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020ac400, {0xc002226000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0020ac400, {0xc002226000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f960, {0xc002226000, 0x466a4e, 0x4383a7})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00256e240, {0xc00038a580, 0x9, 0xc00134ad18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc00256e240}, {0xc00038a580, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038a580, 0x9, 0xc00134ad6f}, {0x23585c0, 0xc00256e240})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038a540)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0005c05a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1250144 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ec6ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647550, {0x1f7b560, 0xc001a11900}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72700, 0xc001b5dd18, 0xc001b5dce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72700, {0x1f7b560, 0xc001a11900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x20, {0xc0016f66e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0xc0019d3340, {0xc0016f66e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0xc00218fb70, 0xc0018f5f20, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0018f5ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001ca3c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x237ac70, 0xc0018f5e60, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001ca3c40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001ca3c40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001ca3bc0}, {0x1f7b560, 0xc001a11900}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001ca3c40}, 0xc00249e420, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc0018f55c0}, 0xc001ca3bc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc0018f55c0}, 0xc001ca3bc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001b5e5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b5e648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2123e90, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc0018f55c0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00218ef30})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b5e898, {0x235a640, 0xc0000a9fb0}, 0x612d643330342d31)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc0018f55c0}, 0xc0018f5410)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0018dedb8, 0x4c1cb4}, {0xc057f6a22709e97a, 0x6117567147}, {0xc0018dedb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc0018f55c0}, 0xc0018f5410)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0xc0016f6580, 0xc001866e40)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0x15, 0xc001ca3980)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0x207690b, 0xc001ca3980)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410}, 0x237bc38, 0xc001ca3980)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f55c0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0018f5470}, {0x1f33180, 0xc0018f5410}, 0x1e5a940, 0xc001ca3980)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f5470}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0x9, 0xc001ca3980)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0xc001a1e768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001a1eb60, {0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0018f53e0}, {0x1f33180, 0xc0018f5410}, 0xc0016f6580, 0xc001866e40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0018f53e0}, 0xc00260d3e0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008e8360, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008e8360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245787 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000484a20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1810, {0x1f7b560, 0xc001768d20}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9d00, 0xc0016dfd18, 0xc0016dfce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9d00, {0x1f7b560, 0xc001768d20})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00189be90}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x20, {0xc000fe8d00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00189be90}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00189be90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0xc000eee580, {0xc000fe8d00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00189be90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0xc00218fb70, 0xc00189bec0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00189be90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e3c7c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x237ac70, 0xc00189be00, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e3c7c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e3c7c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e3c740}, {0x1f7b560, 0xc001768d20}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000e3c7c0}, 0xc001a6ec30, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc00189b4a0}, 0xc000e3c740, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc00189b4a0}, 0xc000e3c740, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0016e05a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016e0648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21c9820, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc00189b4a0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001742280})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016e0898, {0x235a640, 0xc0000a9fb0}, 0x612d386233342d65)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc00189b4a0}, 0xc00189b2f0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001b44db8, 0x4c1cb4}, {0xc057f6a1e74c1ce2, 0x60dbfddaba}, {0xc001b44db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc00189b4a0}, 0xc00189b2f0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0xc000fe8c40, 0xc0017393c8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0x15, 0xc000e3c500)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0x207690b, 0xc000e3c500)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0}, 0x237bc38, 0xc000e3c500)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b4a0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00189b350}, {0x1f33180, 0xc00189b2f0}, 0x1e5a940, 0xc000e3c500)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b350}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0x9, 0xc000e3c500)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0xc002239768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002239b60, {0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00189b2c0}, {0x1f33180, 0xc00189b2f0}, 0xc000fe8c40, 0xc0017393c8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00189b2c0}, 0xc002167800, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008a90e0, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008a90e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1073676 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002095540, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00256e2a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1254951 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0007de0a0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1270992 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077b560)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee8c60, {0x1f7b7c0, 0xc000cd4e60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000b8100, 0xc00080dbf8, 0xc00080dbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000b8100, {0x1f7b7c0, 0xc000cd4e60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x20, {0xc001cb43a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0xc0019d2160, {0xc001cb43a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0xc00218fb70, 0xc0013eef30, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013eef00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013e6ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x237ac70, 0xc0013eee70, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013e6ec0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013e6ec0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0013e6e40}, {0x1f7b7c0, 0xc000cd4e60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0013e6ec0}, 0xc000f00fc0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0013ee810}, 0xc0013e6e40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0013ee810}, 0xc0013e6e40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00080e480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc00080e528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf211ea10, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0013ee810}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001dee530})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc00080e778, {0x235a640, 0xc0000a9fb0}, 0x612d306465342d30)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0013ee810}, 0xc0013e6b40)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000bbedb8, 0x4c1cb4}, {0xc057f6a3680f8aca, 0x62426204a1}, {0xc000bbedb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0013ee810}, 0xc0013e6b40)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40}, 0xc001cb41c0, 0xc001ebcd38)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40}, 0x15, 0xc0013e6b80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40}, 0x2076951, 0xc0013e6b80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40}, 0x237bc38, 0xc0013e6b80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee810}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0013ee6c0}, {0x1f33320, 0xc0013e6b40}, 0x1e5a940, 0xc0013e6b80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee6c0}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40}, 0x23c270d, 0xc0013e6b80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0017a3b60, {0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0013ee690}, {0x1f33320, 0xc0013e6b40}, 0xc001cb41c0, 0xc001ebcd38)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0013ee690}, 0xc002291740, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077a000, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077a000, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1073678 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002095630, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00256e360)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 1245923 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0025afa40, {0x2392228, 0xc000f26f00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1259553 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010fd0e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbd6b0, {0x1f7b560, 0xc00068b9a0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036c00, 0xc001c83d18, 0xc001c83ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036c00, {0x1f7b560, 0xc00068b9a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ee6ab0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0x20, {0xc0016f6200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ee6ab0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ee6ab0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0xc00182c420, {0xc0016f6200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ee6ab0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0xc00218fb70, 0xc001ee6ae0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ee6ab0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001ed9000}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0x237ac70, 0xc001ee6a20, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001ed9000}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001ed9000}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001ed8f80}, {0x1f7b560, 0xc00068b9a0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001ed9000}, 0xc001f5eba0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001ee6360}, 0xc001ed8f80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001ee6360}, 0xc001ed8f80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001c845a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c84648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d5cd8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001ee6360}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001defc60})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c84898, {0x235a640, 0xc0000a9fb0}, 0x392d323461342d62)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001ee6360}, 0xc001ee61b0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000c0adb8, 0x4c1cb4}, {0xc057f6a2a93aa33f, 0x6190bcbf17}, {0xc000c0adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001ee6360}, 0xc001ee61b0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0}, 0xc000e9bfe0, 0xc001e93f80)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0}, 0x15, 0xc001ed8d40)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0}, 0x207690b, 0xc001ed8d40)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0}, 0x237bc38, 0xc001ed8d40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6360}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ee6210}, {0x1f33180, 0xc001ee61b0}, 0x1e5a940, 0xc001ed8d40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6210}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0}, 0x9, 0xc001ed8d40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0}, 0xc00296c768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00296cb60, {0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ee6180}, {0x1f33180, 0xc001ee61b0}, 0xc000e9bfe0, 0xc001e93f80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001ee6180}, 0xc002512300, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0016c18c0, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0016c18c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245970 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0022c8ea0, {0x2392298, 0xc00247be00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fcd80, {0x23922d0, 0xc0013944b0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0013944b0}, 0xc001bab760, 0x1bd58c0, 0x1bab770)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc0013942a0, 0xc000f39e00)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001394000}, 0xc000f39e00)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001394000}, {0x1f353a0, 0xc000f39e00})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001394000}, {0x1f353a0, 0xc000f39e00}, 0x237bc38, 0xc001f9a168)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001394000}, {0x1f353a0, 0xc000f39e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001343e90}, {0x1f353a0, 0xc000f39e00}, 0x203000, 0xc000f39e40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001343e90}, {0x1f353a0, 0xc000f39e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001343e90}, {0x1f353a0, 0xc000f39e00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001343e90}, {0x1f353a0, 0xc000f39e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001343e30}, {0x1f353a0, 0xc000f39e00}, 0x40, 0xc000f39e40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001343e30}, {0x1f353a0, 0xc000f39e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001343e30}, {0x1f353a0, 0xc000f39e00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001343e30}, {0x1f353a0, 0xc000f39e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001343e30}, {0x1f353a0, 0xc000f39e00}, 0xc0013e9d00, 0xc001f9a168)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001343e30}, 0xc00247bda0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc00177d8c0, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc00177d8c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250091 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d2f320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0e630, {0x1f7b7c0, 0xc00068afa0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xbc)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9300, 0xc001c35bf8, 0xc001c35bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9300, {0x1f7b7c0, 0xc00068afa0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001c52690}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0x20, {0xc0018808c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001c52690}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c52690}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0xc001bca000, {0xc0018808c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c52690}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0xc00218fb70, 0xc001c527b0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c52690}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000ffc2c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0x237ac70, 0xc001c52600, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000ffc2c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000ffc2c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000ffc240}, {0x1f7b7c0, 0xc00068afa0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc000ffc2c0}, 0xc000e67830, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc001883ef0}, 0xc000ffc240, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc001883ef0}, 0xc000ffc240, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001c36480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c36528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d6cc8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001883ef0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00074adb0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c36778, {0x235a640, 0xc0000a9fb0}, 0x392d646465342d32)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc001883ef0}, 0xc001119ec0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001bd2db8, 0x4c1cb4}, {0xc057f6a2264aa2dc, 0x6116972aa5}, {0xc001bd2db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc001883ef0}, 0xc001119ec0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0}, 0xc0018807c0, 0xc0020c8738)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0}, 0x15, 0xc001119f00)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0}, 0x2076951, 0xc001119f00)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0}, 0x237bc38, 0xc001119f00)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883ef0}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001883da0}, {0x1f33320, 0xc001119ec0}, 0x1e5a940, 0xc001119f00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883da0}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0}, 0x23c27b0, 0xc001119f00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001883d70}, {0x1f33320, 0xc001119ec0}, 0xc0018807c0, 0xc0020c8738)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001883d70}, 0xc002407380, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001d2e360, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001d2e360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1377290 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1242778 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0005d30e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc00b0, {0x1f7b7c0, 0xc0015c3400}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc3)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3ca00, 0xc001c05bf8, 0xc001c05bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3ca00, {0x1f7b7c0, 0xc0015c3400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00068f200}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0x20, {0xc000ba1400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00068f200}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00068f200}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0xc000e76f20, {0xc000ba1400, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00068f200}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0xc00218fb70, 0xc00068f230, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00068f200}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00011b900}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0x237ac70, 0xc00068f170, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00011b900}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00011b900}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00011b880}, {0x1f7b7c0, 0xc0015c3400}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc00011b900}, 0xc001640390, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc000bc4030}, 0xc00011b880, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc000bc4030}, 0xc00011b880, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001c06480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c06528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6c191b9418, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc000bc4030}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c000638b20})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c06778, {0x235a640, 0xc0000a9fb0}, 0x622d616534342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc000bc4030}, 0xc00011a400)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0016b8db8, 0x4c1cb4}, {0xc057f6a1a98c0901, 0x60a2a2fccb}, {0xc0016b8db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc000bc4030}, 0xc00011a400)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400}, 0xc000ba02a0, 0xc001daf518)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400}, 0x15, 0xc00011a600)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400}, 0x2076951, 0xc00011a600)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400}, 0x237bc38, 0xc00011a600)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000bc4030}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000652c30}, {0x1f33320, 0xc00011a400}, 0x1e5a940, 0xc00011a600)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000652c30}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400}, 0x23c27b0, 0xc00011a600)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001e11b60, {0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000652c00}, {0x1f33320, 0xc00011a400}, 0xc000ba02a0, 0xc001daf518)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000652c00}, 0xc001609560, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00166ac60, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00166ac60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1254934 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0016d2000, {0x2392228, 0xc000966c80})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1254936 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021d11a0, {0x2392298, 0xc0016d2cc0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00077e900, {0x23922d0, 0xc002856960}, 0x14)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0020dd3e0, {0x23922d0, 0xc002856960}, 0xc000ed4ac0, 0x1, 0xed4ae0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc002856750, 0xc00072a4c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0028563c0}, 0xc00072a4c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0028563c0}, {0x1f35540, 0xc00072a4c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0028563c0}, {0x1f35540, 0xc00072a4c0}, 0x237bc38, 0xc002214930)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0028563c0}, {0x1f35540, 0xc00072a4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002856270}, {0x1f35540, 0xc00072a4c0}, 0xc0016d2ae0, 0xc00072a540)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002856270}, {0x1f35540, 0xc00072a4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002856270}, {0x1f35540, 0xc00072a4c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002856270}, {0x1f35540, 0xc00072a4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002856240}, {0x1f35540, 0xc00072a4c0}, 0x40, 0xc00072a540)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002856240}, {0x1f35540, 0xc00072a4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002856240}, {0x1f35540, 0xc00072a4c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002856240}, {0x1f35540, 0xc00072a4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002856240}, {0x1f35540, 0xc00072a4c0}, 0xc000fe86c0, 0xc002214930)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc002856240}, 0xc0016d2c60, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001a70b40, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001a70b40, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245732 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016c0c60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc13f0, {0x1f7b7c0, 0xc0015c3a40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9400, 0xc0016c9bf8, 0xc0016c9bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9400, {0x1f7b7c0, 0xc0015c3a40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00165cd20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0x20, {0xc000fe8460, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00165cd20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00165cd20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0xc002196420, {0xc000fe8460, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00165cd20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0xc00218fb70, 0xc00165cd50, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00165cd20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e03400}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0x237ac70, 0xc00165cc90, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e03400}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e03400}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000e03380}, {0x1f7b7c0, 0xc0015c3a40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc000e03400}, 0xc001e9ff20, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00165c5d0}, 0xc000e03380, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00165c5d0}, 0xc000e03380, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0016ca480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016ca528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2134730, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00165c5d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001d91010})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016ca778, {0x235a640, 0xc0000a9fb0}, 0x612d383666342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00165c5d0}, 0xc000e03080)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0015c8db8, 0x4c1cb4}, {0xc057f6a1e66c6058, 0x60db1e1e23}, {0xc0015c8db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00165c5d0}, 0xc000e03080)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080}, 0xc000fe8380, 0xc001e94f30)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080}, 0x15, 0xc000e030c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080}, 0x2076951, 0xc000e030c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080}, 0x237bc38, 0xc000e030c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c5d0}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00165c480}, {0x1f33320, 0xc000e03080}, 0x1e5a940, 0xc000e030c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c480}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080}, 0x23c27b0, 0xc000e030c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0029a2b60, {0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00165c450}, {0x1f33320, 0xc000e03080}, 0xc000fe8380, 0xc001e94f30)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00165c450}, 0xc002025aa0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000fff680, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000fff680, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270996 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008bb320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcfd0, {0x1f7b7c0, 0xc000deeaa0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72400, 0xc0028cfbf8, 0xc0028cfbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72400, {0x1f7b7c0, 0xc000deeaa0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001656e10}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0x20, {0xc00062a7c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001656e10}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001656e10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0xc0015931e0, {0xc00062a7c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001656e10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0xc00218fb70, 0xc001656e40, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001656e10}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00161f440}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0x237ac70, 0xc001656d80, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00161f440}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00161f440}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00161f3c0}, {0x1f7b7c0, 0xc000deeaa0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc00161f440}, 0xc000e29880, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0016566f0}, 0xc00161f3c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0016566f0}, 0xc00161f3c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0028d0480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0028d0528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2191ec8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0016566f0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c0023bc2a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0028d0778, {0x235a640, 0xc0000a9fb0}, 0x382d366638342d62)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0016566f0}, 0xc00161f0c0)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000b9edb8, 0x4c1cb4}, {0xc057f6a367b9c554, 0x62420c3f1f}, {0xc000b9edb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0016566f0}, 0xc00161f0c0)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0}, 0xc001b59fe0, 0xc00160a948)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0}, 0x15, 0xc00161f100)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0}, 0x2076951, 0xc00161f100)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0}, 0x237bc38, 0xc00161f100)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016566f0}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0016565a0}, {0x1f33320, 0xc00161f0c0}, 0x1e5a940, 0xc00161f100)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016565a0}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0}, 0x23c27b0, 0xc00161f100)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001656570}, {0x1f33320, 0xc00161f0c0}, 0xc001b59fe0, 0xc00160a948)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001656570}, 0xc0024a4120, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00175b320, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00175b320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270889 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0016848a0, {0x2392298, 0xc000efcf00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001688000, {0x23922d0, 0xc001346b40}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001f5f950, {0x23922d0, 0xc001346b40}, 0xc001340e60, 0x1, 0x1340e70)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001346ae0, 0xc0013446c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001346870}, 0xc0013446c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001346870}, {0x1f35540, 0xc0013446c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001346870}, {0x1f35540, 0xc0013446c0}, 0x237bc38, 0xc001f90030)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001346870}, {0x1f35540, 0xc0013446c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001346720}, {0x1f35540, 0xc0013446c0}, 0xc001327f40, 0xc001344740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001346720}, {0x1f35540, 0xc0013446c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001346720}, {0x1f35540, 0xc0013446c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001346720}, {0x1f35540, 0xc0013446c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0013466f0}, {0x1f35540, 0xc0013446c0}, 0x40, 0xc001344740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013466f0}, {0x1f35540, 0xc0013446c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0013466f0}, {0x1f35540, 0xc0013446c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013466f0}, {0x1f35540, 0xc0013446c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0013466f0}, {0x1f35540, 0xc0013446c0}, 0xc001b58760, 0xc001f90030)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0013466f0}, 0xc000efcea0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016a8000, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016a8000, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1254946 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00087f8c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc04d0, {0x1f7c860, 0xc001dc0420}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d800, 0xc000ee55e8, 0xc000ee55b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d800, {0x1f7c860, 0xc001dc0420})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0022d6f00}, {0x20768da, 0x8}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x1c75901, {0xc0013e8580, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x7f6c191ae108, {0xc0013e8580, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x20, 0xc000bc5e00, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0xc000da4820, {0xc001faa5c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x0, 0xc000bc5dd0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x1e0cfc0, 0xc000bc5d10, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0022d6f00}, {0x20768da, 0x203000}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x8, 0xc000bc5ce0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0022d6f00}, {0x20768da, 0x46}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, 0x3cc0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc0022d6f00}, {0x20768da, 0x0}, {0x1f33180, 0xc000bc5c80}, {0x1f7c860, 0xc001dc0420}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0022d6f00}, 0xc000af8720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00003b480, {0x2392298, 0xc0022d6f00})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1250216 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00247b740, {0x2392298, 0xc001dc5320}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00177cea0, {0x23922d0, 0xc001728570}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001728570}, 0xc001845ba0, 0x1bd58c0, 0x1845bb0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001728510, 0xc0015eed40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc0017281b0}, 0xc0015eed40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0017281b0}, {0x1f353a0, 0xc0015eed40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0017281b0}, {0x1f353a0, 0xc0015eed40}, 0x237bc38, 0xc0018cf098)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017281b0}, {0x1f353a0, 0xc0015eed40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001728030, {0x23922d0, 0xc001728060}, {0x1f353a0, 0xc0015eed40}, 0xc000cb6d50, 0xc0015eed80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001728060}, {0x1f353a0, 0xc0015eed40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001728060}, {0x1f353a0, 0xc0015eed40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001728060}, {0x1f353a0, 0xc0015eed40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000cb7fb0}, {0x1f353a0, 0xc0015eed40}, 0x40, 0xc0015eed80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000cb7fb0}, {0x1f353a0, 0xc0015eed40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000cb7fb0}, {0x1f353a0, 0xc0015eed40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000cb7fb0}, {0x1f353a0, 0xc0015eed40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000cb7fb0}, {0x1f353a0, 0xc0015eed40}, 0xc0016f7000, 0xc0018cf098)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000cb7fb0}, 0xc001dc52c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001be1440, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001be1440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245996 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021ecf00, {0x2392298, 0xc002092240}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0008f37a0, {0x23922d0, 0xc001821e60}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001821e60}, 0xc000fd0440, 0x1bd58c0, 0xfd0450)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001821c50, 0xc000fca740)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc0018219b0}, 0xc000fca740)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0018219b0}, {0x1f353a0, 0xc000fca740})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0018219b0}, {0x1f353a0, 0xc000fca740}, 0x237bc38, 0xc0017d16e0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018219b0}, {0x1f353a0, 0xc000fca740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001821860}, {0x1f353a0, 0xc000fca740}, 0x7f6bf237dd50, 0xc000fca780)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001821860}, {0x1f353a0, 0xc000fca740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001821860}, {0x1f353a0, 0xc000fca740}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001821860}, {0x1f353a0, 0xc000fca740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001821800}, {0x1f353a0, 0xc000fca740}, 0x40, 0xc000fca780)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001821800}, {0x1f353a0, 0xc000fca740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001821800}, {0x1f353a0, 0xc000fca740}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001821800}, {0x1f353a0, 0xc000fca740})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001821800}, {0x1f353a0, 0xc000fca740}, 0xc000e9a3a0, 0xc0017d16e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001821800}, 0xc0020921e0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001b76900, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001b76900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1254932 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001fac0f0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1245786 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00175a7e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc18c0, {0x1f7b7c0, 0xc001768dc0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb7)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9e00, 0xc00174bbf8, 0xc00174bbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9e00, {0x1f7b7c0, 0xc001768dc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001c1cdb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0x20, {0xc000fe8ea0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001c1cdb0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c1cdb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0xc000eef1e0, {0xc000fe8ea0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c1cdb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0xc00218fb70, 0xc001c1cde0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c1cdb0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e3ce80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0x237ac70, 0xc001c1cd20, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e3ce80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e3ce80}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000e3ce00}, {0x1f7b7c0, 0xc001768dc0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc000e3ce80}, 0xc000595640, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc001c1c4b0}, 0xc000e3ce00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc001c1c4b0}, 0xc000e3ce00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00174c480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc00174c528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21c9820, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001c1c4b0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c0017423c0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc00174c778, {0x235a640, 0xc0000a9fb0}, 0x612d623439342d35)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc001c1c4b0}, 0xc000e3cb00)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001b44db8, 0x4c1cb4}, {0xc057f6a1e751dea2, 0x60dc039c7a}, {0xc001b44db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc001c1c4b0}, 0xc000e3cb00)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00}, 0xc000fe8dc0, 0xc0017399c8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00}, 0x15, 0xc000e3cb40)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00}, 0x2076951, 0xc000e3cb40)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00}, 0x237bc38, 0xc000e3cb40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c4b0}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001c1c330}, {0x1f33320, 0xc000e3cb00}, 0x1e5a940, 0xc000e3cb40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c330}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00}, 0x9, 0xc000e3cb40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002239b60, {0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001c1c300}, {0x1f33320, 0xc000e3cb00}, 0xc000fe8dc0, 0xc0017399c8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001c1c300}, 0xc002167a40, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008a8ea0, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008a8ea0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1265184 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1245967 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc00247b440, {0x2392228, 0xc000f39840})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1254966 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021d11a0, {0x2392298, 0xc002662540}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00077e900, {0x23922d0, 0xc001f3be30}, 0x14)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0020dd920, {0x23922d0, 0xc001f3be30}, 0xc0007cd410, 0x1, 0x7cd430)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001f3bc20, 0xc0008fe6c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001f3b980}, 0xc0008fe6c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001f3b980}, {0x1f35540, 0xc0008fe6c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001f3b980}, {0x1f35540, 0xc0008fe6c0}, 0x237bc38, 0xc0020b21f8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3b980}, {0x1f35540, 0xc0008fe6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f3b830}, {0x1f35540, 0xc0008fe6c0}, 0xc0016d31a0, 0xc0008fe740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3b830}, {0x1f35540, 0xc0008fe6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f3b830}, {0x1f35540, 0xc0008fe6c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3b830}, {0x1f35540, 0xc0008fe6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f3b800}, {0x1f35540, 0xc0008fe6c0}, 0x40, 0xc0008fe740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3b800}, {0x1f35540, 0xc0008fe6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f3b800}, {0x1f35540, 0xc0008fe6c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3b800}, {0x1f35540, 0xc0008fe6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f3b800}, {0x1f35540, 0xc0008fe6c0}, 0xc000fe8ba0, 0xc0020b21f8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001f3b800}, 0xc0026624e0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc00077fc20, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc00077fc20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1064365 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002205680, {0x2392228, 0xc001119b00})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250268 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000b4460, {0x2392228, 0xc002755580})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1254933 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000b4150, {0x2392228, 0xc000966c40})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1254953 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0022d7680, {0x2392228, 0xc001fabb40})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1054790 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001b69b60, {0x2392228, 0xc001bc6a00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250261 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000e5d320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647a20, {0x1f7b7c0, 0xc0015c2320}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016e000, 0xc0010e64d0, 0xc0010e64a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016e000, {0x1f7b7c0, 0xc0015c2320})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f0d290}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0x20, {0xc0016f7940, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f0d290}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f0d290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0xc001a78160, {0xc0016f7940, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f0d290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0xc00218ff30, 0xc001f0d2c0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f0d290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002754a80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0x237ac70, 0xc001f0d1a0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002754a80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002754a80}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0027549c0}, {0x1f7b7c0, 0xc0015c2320}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000097128, {0x2392228, 0xc002754a80}, 0xc00060e146, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc00068e150, {0x2392298, 0xc0025f54a0}, 0xc0027549c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc00062aa20, {0x2392298, 0xc0025f54a0}, 0xc0027549c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0025f5140, {0x2392298, 0xc0025f54a0})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0025f5440, {0x2392298, 0xc0025f54a0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000e5c360, {0x23922d0, 0xc000f9ffb0}, 0x2d)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc00249f890, {0x23922d0, 0xc000f9ffb0}, 0xc0010ba0c0, 0x1, 0x10ba0d0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc000f9ff50, 0xc002754100)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc000f9fbc0}, 0xc002754100)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc000f9fbc0}, {0x1f35540, 0xc002754100})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000f9fbc0}, {0x1f35540, 0xc002754100}, 0x237bc38, 0xc001efe618)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9fbc0}, {0x1f35540, 0xc002754100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc000f9fa70}, {0x1f35540, 0xc002754100}, 0xc0017d9b80, 0xc002754180)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9fa70}, {0x1f35540, 0xc002754100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000f9fa70}, {0x1f35540, 0xc002754100}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9fa70}, {0x1f35540, 0xc002754100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000f9fa40}, {0x1f35540, 0xc002754100}, 0x40, 0xc002754180)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9fa40}, {0x1f35540, 0xc002754100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000f9fa40}, {0x1f35540, 0xc002754100}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f9fa40}, {0x1f35540, 0xc002754100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000f9fa40}, {0x1f35540, 0xc002754100}, 0xc0016f7740, 0xc001efe618)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000f9fa40}, 0xc0025f4de0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000fff0e0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000fff0e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242724 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ae1b00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bad760, {0x1f736c0, 0xc001e06d80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73900, 0xc0007fd5c0, 0xc0007fd590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73900, {0x1f736c0, 0xc001e06d80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0021bb620}, {0x2076920, 0x8}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x1, {0xc000ba0c80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x7f6c191ae5b8, {0xc000ba0c80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x20, 0xc001e6f980, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0xc000d3fa00, {0xc001e5b500, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x0, 0xc001e6f950, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0xc000729040, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x1e0cfc0, 0xc001e6f8c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0xc000bad4e8, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0021bb620}, {0x2076920, 0x203000}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x203000, 0xc001e6f890, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0021bb620}, {0x2076920, 0x46}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x468c27, {0x2392298, 0xc0021bb620}, {0x2076920, 0x100000000}, {0x1f33320, 0xc001e5b4c0}, {0x1f736c0, 0xc001e06d80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc0021bb620}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc0021bb620})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1242737 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c50900)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0dc0, {0x1f7b7c0, 0xc000deee60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xbf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3c500, 0xc000dd3bf8, 0xc000dd3bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3c500, {0x1f7b7c0, 0xc000deee60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ea1110}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0x20, {0xc0016f7980, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ea1110}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ea1110}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0xc000ec9e40, {0xc0016f7980, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ea1110}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0xc00218fb70, 0xc001ea1140, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ea1110}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001ea6280}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0x237ac70, 0xc001ea1080, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001ea6280}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001ea6280}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001ea6200}, {0x1f7b7c0, 0xc000deee60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc001ea6280}, 0xc000594ea0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc001ea09c0}, 0xc001ea6200, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc001ea09c0}, 0xc001ea6200, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000dd4480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000dd4528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf217e808, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001ea09c0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c000639840})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000dd4778, {0x235a640, 0xc0000a9fb0}, 0x382d623065342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc001ea09c0}, 0xc001dfdf00)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0015acdb8, 0x4c1cb4}, {0xc057f6a1a831606f, 0x60a1485445}, {0xc0015acdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc001ea09c0}, 0xc001dfdf00)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00}, 0xc0016f7880, 0xc00237fa58)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00}, 0x15, 0xc001dfdf40)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00}, 0x2076951, 0xc001dfdf40)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00}, 0x237bc38, 0xc001dfdf40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea09c0}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc000729520, {0x23922d0, 0xc001ea0870}, {0x1f33320, 0xc001dfdf00}, 0x203000, 0xc001dfdf40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea0870}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00}, 0x23c27b0, 0xc001dfdf40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ea0840}, {0x1f33320, 0xc001dfdf00}, 0xc0016f7880, 0xc00237fa58)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001ea0840}, 0xc001ee3680, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001a719e0, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001a719e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245992 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000408d90, {0x2392228, 0xc000fca1c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1259557 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021ecf00, {0x2392298, 0xc002512720}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0008f37a0, {0x23922d0, 0xc001ee7620}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001ee7620}, 0xc001ed1c40, 0x1bd58c0, 0x1ed1c50)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001ee75c0, 0xc001ed9440)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001ee7320}, 0xc001ed9440)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ee7320}, {0x1f353a0, 0xc001ed9440})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001ee7320}, {0x1f353a0, 0xc001ed9440}, 0x237bc38, 0xc001eec4e0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee7320}, {0x1f353a0, 0xc001ed9440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001ee71a0, {0x23922d0, 0xc001ee71d0}, {0x1f353a0, 0xc001ed9440}, 0xc001ee61b0, 0xc001ed9480)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee71d0}, {0x1f353a0, 0xc001ed9440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ee71d0}, {0x1f353a0, 0xc001ed9440}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee71d0}, {0x1f353a0, 0xc001ed9440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ee7140}, {0x1f353a0, 0xc001ed9440}, 0x40, 0xc001ed9480)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee7140}, {0x1f353a0, 0xc001ed9440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ee7140}, {0x1f353a0, 0xc001ed9440}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ee7140}, {0x1f353a0, 0xc001ed9440})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ee7140}, {0x1f353a0, 0xc001ed9440}, 0xc0016f6680, 0xc001eec4e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001ee7140}, 0xc0025126c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0010fd440, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0010fd440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1255005 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00166a120)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbc580, {0x1f7c860, 0xc000dbc4d0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016f600, 0xc0005335e8, 0xc0005335b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016f600, {0x1f7c860, 0xc000dbc4d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0026633e0}, {0x20768da, 0x8}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x1c75901, {0xc000fe9040, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x7f6c191ae5b8, {0xc000fe9040, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x20, 0xc0005b98f0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0xc000da4820, {0xc001fe2a00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x0, 0xc0005b98c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x1e0cfc0, 0xc0005b9740, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0xc002255d98, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0026633e0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x8, 0xc0005b9710, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0026633e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc0026633e0}, {0x20768da, 0x0}, {0x1f33180, 0xc0005b96b0}, {0x1f7c860, 0xc000dbc4d0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0026633e0}, 0xc002255f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000daaf80, {0x2392298, 0xc0026633e0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1250121 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c3efc0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc02c0, {0x1f7b560, 0xc0015c2a00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3cf00, 0xc0017e0440, 0xc0017e0410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3cf00, {0x1f7b560, 0xc0015c2a00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e7ffb0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0x20, {0xc0013e81e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e7ffb0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e7ffb0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0x0, {0xc0013e81e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e7ffb0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0xc001fb6920, 0xc0008382a0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e7ffb0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000634000}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0x237ac70, 0xc001e7ff20, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000634000}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000634000}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc00011bd80}, {0x1f7b560, 0xc0015c2a00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc000634000}, 0xc001ba8046, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc00275c5a0}, 0xc00011bd80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc00275c5a0}, 0xc00011bd80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002119980, {0x2392298, 0xc00275c5a0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002119bc0, {0x2392298, 0xc00275c5a0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0008e99e0, {0x23922d0, 0xc0016d0b70}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0016d0b70}, 0xc001f53f60, 0x1bd58c0, 0x1f53f70)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc0016d0b10, 0xc0017ca4c0)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc0016d06f0}, 0xc0017ca4c0)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0016d06f0}, {0x1f353a0, 0xc0017ca4c0})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0016d06f0}, {0x1f353a0, 0xc0017ca4c0}, 0x237bc38, 0xc001668588)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016d06f0}, {0x1f353a0, 0xc0017ca4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0016d0540, {0x23922d0, 0xc0016d0570}, {0x1f353a0, 0xc0017ca4c0}, 0xc0016b3710, 0xc0017ca500)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016d0570}, {0x1f353a0, 0xc0017ca4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0016d0570}, {0x1f353a0, 0xc0017ca4c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016d0570}, {0x1f353a0, 0xc0017ca4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0016d04e0}, {0x1f353a0, 0xc0017ca4c0}, 0x40, 0xc0017ca500)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016d04e0}, {0x1f353a0, 0xc0017ca4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0016d04e0}, {0x1f353a0, 0xc0017ca4c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016d04e0}, {0x1f353a0, 0xc0017ca4c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0016d04e0}, {0x1f353a0, 0xc0017ca4c0}, 0xc0018812c0, 0xc001668588)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0016d04e0}, 0xc00275c540, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016e4a20, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016e4a20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245924 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0025afa40, {0x2392228, 0xc000f26f00})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250098 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000a15720)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 846497 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001a46640)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1301 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000da4750, 0xc0013a41e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000da4750)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1281 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc001f251f0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1282 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc001e23c00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1283 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc001f528d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1250093 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d2fe60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0e840, {0x1f7c860, 0xc001a0e790}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9400, 0xc000e5f5e8, 0xc000e5f5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9400, {0x1f7c860, 0xc001a0e790})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002407740}, {0x20768da, 0x8}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x1c75901, {0xc001880a60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x7f6c191ae5b8, {0xc001880a60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x20, 0xc001c53200, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0xc000da4820, {0xc000ffcc00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x0, 0xc001c531d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x1e0cfc0, 0xc001c53140, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x6, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002407740}, {0x20768da, 0x203000}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0x8, 0xc001c53110, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002407740}, {0x20768da, 0x46}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, 0xc000e83650, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc002407740}, {0x20768da, 0x0}, {0x1f33180, 0xc001c530b0}, {0x1f7c860, 0xc001a0e790}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc002407740}, 0xc000e836c8, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc002407740})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1246 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc001fee830)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1248 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc002014d50)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1247 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc001d73a90)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1302 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000da4750, 0xc00079c0e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000da4750)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1303 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1270921 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010fc480)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcc60, {0x1f736c0, 0xc00136f4d0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9500, 0xc000d595c0, 0xc000d59590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9500, {0x1f736c0, 0xc00136f4d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0017fd500}, {0x2076920, 0x8}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x1c75901, {0xc001b59200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x7f6c191ae108, {0xc001b59200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x20, 0xc0015be570, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0xc000d3fa00, {0xc00137b840, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x0, 0xc0015be540, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x0, 0xc0015be4b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0017fd500}, {0x2076920, 0x203000}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x203000, 0xc0015be480, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0017fd500}, {0x2076920, 0x46}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc0017fd500}, {0x2076920, 0x200e97cd8}, {0x1f33320, 0xc00137b800}, {0x1f736c0, 0xc00136f4d0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc0017fd500}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000da4750, {0x2392298, 0xc0017fd500})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1300 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000da4750, 0xc000ea1500)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000da4750)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1073600 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00258ed80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1250264 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0025f5080, {0x2392228, 0xc002754900})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1297 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc000126960)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b4a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b4a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1299 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000da4750, 0xc001379ae0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000da4750)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b3b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1284 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1245781 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010fdd40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1600, {0x1f736c0, 0xc000e19cb0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x69)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9800, 0xc000dc15c0, 0xc000dc1590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9800, {0x1f736c0, 0xc000e19cb0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002167320}, {0x2076920, 0x8}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x203001, {0xc000fe89c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x7f6c191ae5b8, {0xc000fe89c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x20, 0xc00189a8a0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0xc000d3fa00, {0xc000e3c000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x0, 0xc00189a870, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x1e0cfc0, 0xc00189a7e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0xc0026acc88, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002167320}, {0x2076920, 0x203000}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x203000, 0xc00189a7b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002167320}, {0x2076920, 0x46}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0026ace50, {0x2392298, 0xc002167320}, {0x2076920, 0x1026ace58}, {0x1f33320, 0xc000e27fc0}, {0x1f736c0, 0xc000e19cb0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc002167320}, 0xc000e279c0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000ec4f70, {0x2392298, 0xc002167320})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1270865 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ec7d40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee8420, {0x1f736c0, 0xc0013855f0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d700, 0xc000d5d5c0, 0xc000d5d590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d700, {0x1f736c0, 0xc0013855f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001c12c00}, {0x2076920, 0x8}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x1c75901, {0xc000ba1ce0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x7f6c191ae5b8, {0xc000ba1ce0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x20, 0xc0013a8270, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0xc000d3fa00, {0xc0013a0d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x0, 0xc0013a8240, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x0, 0xc0013a81b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001c12c00}, {0x2076920, 0x203000}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x203000, 0xc0013a8180, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001c12c00}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0017a3e50, {0x2392298, 0xc001c12c00}, {0x2076920, 0x2fc03a08a}, {0x1f33320, 0xc0013a0d40}, {0x1f736c0, 0xc0013855f0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001c12c00}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc001c12c00})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1250193 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016a9e60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac580, {0x1f7c860, 0xc001bac4d0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3ce00, 0xc000d5b5e8, 0xc000d5b5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3ce00, {0x1f7c860, 0xc001bac4d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0021cb620}, {0x20768da, 0x8}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x1c75901, {0xc000ba0ea0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x7f6c191ae5b8, {0xc000ba0ea0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x20, 0xc000c223c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0xc000da4820, {0xc001eb2900, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x0, 0xc000c22390, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x1e0cfc0, 0xc000c22300, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0xc0026acd98, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0021cb620}, {0x20768da, 0x203000}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x8, 0xc000c222d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0021cb620}, {0x20768da, 0x46}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, 0x6, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc0021cb620}, {0x20768da, 0x0}, {0x1f33180, 0xc000c22270}, {0x1f7c860, 0xc001bac4d0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0021cb620}, 0x34c4800, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc0021cb620})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1054837 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000412930, {0x2392228, 0xc001cfbd80})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 828619 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d29b30, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0016d3320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 1298 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00087b4a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1249 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc000e9a5a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef0e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef0e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1270997 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0018f1d40, {0x2392298, 0xc0024a4420}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0019e0900, {0x23922d0, 0xc0016575c0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001898990, {0x23922d0, 0xc0016575c0}, 0xc00161d630, 0x1, 0x161d640)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001657560, 0xc00161f6c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0016572f0}, 0xc00161f6c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0016572f0}, {0x1f35540, 0xc00161f6c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0016572f0}, {0x1f35540, 0xc00161f6c0}, 0x237bc38, 0xc00160ad38)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016572f0}, {0x1f35540, 0xc00161f6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0016571a0}, {0x1f35540, 0xc00161f6c0}, 0xc00161f0c0, 0xc00161f740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016571a0}, {0x1f35540, 0xc00161f6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0016571a0}, {0x1f35540, 0xc00161f6c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016571a0}, {0x1f35540, 0xc00161f6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001657170}, {0x1f35540, 0xc00161f6c0}, 0x40, 0xc00161f740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001657170}, {0x1f35540, 0xc00161f6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001657170}, {0x1f35540, 0xc00161f6c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001657170}, {0x1f35540, 0xc00161f6c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001657170}, {0x1f35540, 0xc00161f6c0}, 0xc00062a920, 0xc00160ad38)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001657170}, 0xc0024a43c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008bb560, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008bb560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 828632 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000cbc6e0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0016d3080)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1244 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc0011f51d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1250 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0007ef0e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1251 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc001381360)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1252 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc0016593f0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1253 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc0013219f0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1254 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc0013a4350)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1255 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc001e60fd0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1256 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc001340590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1257 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc001f520f0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1258 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e89c70, 0xc001dbb210)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000e89c70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1259 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0007eeff0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1270841 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00089c7e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee8210, {0x1f736c0, 0xc001384c60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3cb00, 0xc0008675c0, 0xc000867590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3cb00, {0x1f736c0, 0xc001384c60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001c12000}, {0x2076920, 0x8}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x1c75901, {0xc000ba1020, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x7f6c191ae5b8, {0xc000ba1020, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x20, 0xc00138b890, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0xc000d3fa00, {0xc0012f9b00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x0, 0xc00138b860, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x0, 0xc00138b7d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001c12000}, {0x2076920, 0x203000}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x203000, 0xc00138b7a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001c12000}, {0x2076920, 0x46}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0017a3e50, {0x2392298, 0xc001c12000}, {0x2076920, 0x22cf192d3}, {0x1f33320, 0xc0012f9ac0}, {0x1f736c0, 0xc001384c60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001c12000}, 0x4802f6, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc001c12000})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1245 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000e91100, 0xc00207ab20)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000e91100)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007ef1d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1270869 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c085a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee84d0, {0x1f7b7c0, 0xc000cd4140}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3dd00, 0xc000eb7bf8, 0xc000eb7bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3dd00, {0x1f7b7c0, 0xc000cd4140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0013a9020}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0x20, {0xc000ba1f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0013a9020}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013a9020}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0xc001853ce0, {0xc000ba1f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013a9020}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0xc00218fb70, 0xc0013a9050, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0013a9020}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013a13c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0x237ac70, 0xc0013a8f90, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013a13c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013a13c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0013a1340}, {0x1f7b7c0, 0xc000cd4140}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0013a13c0}, 0xc000f00580, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0013a8930}, 0xc0013a1340, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0013a8930}, 0xc0013a1340, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000eb8480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000eb8528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d0e50, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0013a8930}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001ec8da0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000eb8778, {0x235a640, 0xc0000a9fb0}, 0x622d666362342d63)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0013a8930}, 0xc0013a1040)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001818db8, 0x4c1cb4}, {0xc057f6a36622a0a4, 0x6240751a7a}, {0xc001818db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0013a8930}, 0xc0013a1040)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040}, 0xc000ba1e60, 0xc001e52a50)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040}, 0x15, 0xc0013a1080)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040}, 0x2076951, 0xc0013a1080)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040}, 0x237bc38, 0xc0013a1080)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a8930}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0013a87e0}, {0x1f33320, 0xc0013a1040}, 0x1e5a940, 0xc0013a1080)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a87e0}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040}, 0x23c27b0, 0xc0013a1080)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0017a3b60, {0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0013a87b0}, {0x1f33320, 0xc0013a1040}, 0xc000ba1e60, 0xc001e52a50)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0013a87b0}, 0xc001c132c0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008d0360, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008d0360, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1263 [select, 7 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000bba640)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 1270904 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000796ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcb00, {0x1f736c0, 0xc00136e360}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9000, 0xc0015e55c0, 0xc0015e5590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9000, {0x1f736c0, 0xc00136e360})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000efdf20}, {0x2076920, 0x8}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x203001, {0xc001b58ca0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x7f6c191ae108, {0xc001b58ca0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x20, 0xc00136d770, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0xc000d3fa00, {0xc00137a000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x0, 0xc00136d740, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x0, 0xc00136d6b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000efdf20}, {0x2076920, 0x203000}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x203000, 0xc00136d680, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000efdf20}, {0x2076920, 0x46}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc000efdf20}, {0x2076920, 0x200000000}, {0x1f33320, 0xc001345fc0}, {0x1f736c0, 0xc00136e360}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc000efdf20}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000ec4f70, {0x2392298, 0xc000efdf20})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1268 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2206f18, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000ce1680, 0xc000d68000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000ce1680, {0xc000d68000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000ce1680, {0xc000d68000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000096ef8, {0xc000d68000, 0x4383e0, 0xc000d564e0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000d54960, {0xc000792580, 0x9, 0xc000d4bd18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000d54960}, {0xc000792580, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000792580, 0x9, 0x18}, {0x23585c0, 0xc000d54960})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000792540)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000d33c20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1245730 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000fff200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1340, {0x1f736c0, 0xc000cd9b00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9100, 0xc000d5f5c0, 0xc000d5f590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9100, {0x1f736c0, 0xc000cd9b00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002025860}, {0x2076920, 0x8}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x1c75901, {0xc000fe8280, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x7f6c191ae5b8, {0xc000fe8280, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x20, 0xc00165c0c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0xc000d3fa00, {0xc000e02ec0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x0, 0xc00165c090, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x1e0cfc0, 0xc00165c000, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0xc03ee56584, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002025860}, {0x2076920, 0x203000}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x203000, 0xc0023d7f80, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002025860}, {0x2076920, 0x46}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0026ace50, {0x2392298, 0xc002025860}, {0x2076920, 0x100bc9930}, {0x1f33320, 0xc000e02e80}, {0x1f736c0, 0xc000cd9b00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc002025860}, 0xc000700840, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc002025860})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1245779 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002166c00, {0x2392228, 0xc000e27a00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250142 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008e8120)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0016473f0, {0x1f7c860, 0xc001647340}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72500, 0xc000dc35e8, 0xc000dc35b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72500, {0x1f7c860, 0xc001647340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00260d0e0}, {0x20768da, 0x8}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x1c75901, {0xc0016f6280, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x7f6c191ae108, {0xc0016f6280, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x20, 0xc0018f5050, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0xc000da4820, {0xc001ca3780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x0, 0xc0018f5020, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x1e0cfc0, 0xc0018f4f90, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x1, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00260d0e0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x8, 0xc0018f4f60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00260d0e0}, {0x20768da, 0x46}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc00260d0e0}, {0x20768da, 0x0}, {0x1f33180, 0xc0018f4f00}, {0x1f7c860, 0xc001647340}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc00260d0e0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc00260d0e0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1255010 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000b4230, {0x2392228, 0xc001fe38c0})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1250150 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1bc20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000730a50, {0x1f7b560, 0xc000deea00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9d00, 0xc001a35d18, 0xc001a35ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9d00, {0x1f7b560, 0xc000deea00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001857c80}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0x20, {0xc001881d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001857c80}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001857c80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0xc0015c78c0, {0xc001881d80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001857c80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0xc00218fb70, 0xc001857cb0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001857c80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0017cbc80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0x237ac70, 0xc001857bf0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0017cbc80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0017cbc80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0017cbc00}, {0x1f7b560, 0xc000deea00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc0017cbc80}, 0xc0016fe780, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001857530}, 0xc0017cbc00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001857530}, 0xc0017cbc00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001a365a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001a36648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21c4fb8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001857530}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00074bf10})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001a36898, {0x235a640, 0xc0000a9fb0}, 0x612d336238342d37)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001857530}, 0xc001857290)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001c98db8, 0x4c1cb4}, {0xc057f6a2271f5c6d, 0x61176be43c}, {0xc001c98db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001857530}, 0xc001857290)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290}, 0xc001881c00, 0xc001669578)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290}, 0x15, 0xc0017cb9c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290}, 0x207690b, 0xc0017cb9c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290}, 0x237bc38, 0xc0017cb9c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857530}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0018572f0}, {0x1f33180, 0xc001857290}, 0x1e5a940, 0xc0017cb9c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018572f0}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290}, 0x9, 0xc0017cb9c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290}, 0xc0007f7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001857260}, {0x1f33180, 0xc001857290}, 0xc001881c00, 0xc001669578)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001857260}, 0xc00275d2c0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001c1a900, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001c1a900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270848 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ec6c60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee8370, {0x1f736c0, 0xc0013853b0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d600, 0xc000dbf5c0, 0xc000dbf590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d600, {0x1f736c0, 0xc0013853b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001c12a20}, {0x2076920, 0x8}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x1c75901, {0xc000ba1ae0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x7f6c191ae5b8, {0xc000ba1ae0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x20, 0xc00139de30, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0xc000d3fa00, {0xc0013a0b00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x0, 0xc00139de00, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x0, 0xc00139dd70, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001c12a20}, {0x2076920, 0x203000}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x203000, 0xc00139dd40, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001c12a20}, {0x2076920, 0x46}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc0017a3e50, {0x2392298, 0xc001c12a20}, {0x2076920, 0x2fc03a08a}, {0x1f33320, 0xc0013a0ac0}, {0x1f736c0, 0xc0013853b0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001c12a20}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000da4750, {0x2392298, 0xc001c12a20})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1254910 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ba50e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbc160, {0x1f736c0, 0xc0024cbdd0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5f)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016ec00, 0xc000e635c0, 0xc000e63590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016ec00, {0x1f736c0, 0xc0024cbdd0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x8}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x1c75901, {0xc000fe8180, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x7f6c191ae5b8, {0xc000fe8180, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x20, 0xc0006656e0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0xc000d3fa00, {0xc000edde80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x0, 0xc000665680, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x1e0cfc0, 0xc000665470, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x7f6bf23f61e0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x203000}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x203000, 0xc000665440, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000c2b7a0}, {0x2076920, 0x46}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000c4ce50, {0x2392298, 0xc000c2b7a0}, {0x2076920, 0x1025d7200}, {0x1f33320, 0xc000edde40}, {0x1f736c0, 0xc0024cbdd0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc000c2b7a0}, 0x237bda0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000da4750, {0x2392298, 0xc000c2b7a0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1255007 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00166bd40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbc630, {0x1f7b560, 0xc000cd5540}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016f800, 0xc0013cdd18, 0xc0013cdce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016f800, {0x1f7b560, 0xc000cd5540})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0006829c0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0x20, {0xc000fe9440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0006829c0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0006829c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0xc001c74160, {0xc000fe9440, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0006829c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0xc00218fb70, 0xc0006829f0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0006829c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fe2ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0x237ac70, 0xc000682930, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fe2ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fe2ec0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001fe2e40}, {0x1f7b560, 0xc000cd5540}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001fe2ec0}, 0xc001ba8d00, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc000682240}, 0xc001fe2e40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc000682240}, 0xc001fe2e40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0013ce5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0013ce648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21dfd30, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc000682240}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c0003d4990})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0013ce898, {0x235a640, 0xc0000a9fb0}, 0x612d376262342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc000682240}, 0xc000682090)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001868db8, 0x4c1cb4}, {0xc057f6a266e5489a, 0x6152cc9a6b}, {0xc001868db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc000682240}, 0xc000682090)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090}, 0xc000fe91c0, 0xc0020b37a0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090}, 0x15, 0xc001fe2c00)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090}, 0x207690b, 0xc001fe2c00)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090}, 0x237bc38, 0xc001fe2c00)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682240}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0006820f0}, {0x1f33180, 0xc000682090}, 0x1e5a940, 0xc001fe2c00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0006820f0}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090}, 0x9, 0xc001fe2c00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090}, 0xc000987768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000987b60, {0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090}, 0x40, 0xc00005ec00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000682060}, {0x1f33180, 0xc000682090}, 0xc000fe91c0, 0xc0020b37a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000682060}, 0xc002663680, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00166a240, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00166a240, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245963 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00177d680)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001646160, {0x1f7b560, 0xc00068ae60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xac)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036700, 0xc0017a0440, 0xc0017a0410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036700, {0x1f7b560, 0xc00068ae60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001343710}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0x20, {0xc0013e9bc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001343710}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001343710}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0xc001852f20, {0xc0013e9bc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001343710}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0xc001fb6920, 0xc001343740, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001343710}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000f39980}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0x237ac70, 0xc001343680, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000f39980}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000f39980}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000f39900}, {0x1f7b560, 0xc00068ae60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc000f39980}, 0xc0016b7b96, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc00247b7a0}, 0xc000f39900, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc00247b7a0}, 0xc000f39900, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00247b500, {0x2392298, 0xc00247b7a0})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00247b740, {0x2392298, 0xc00247b7a0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00177cea0, {0x23922d0, 0xc001342900}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001342900}, 0xc001baa5b0, 0x1bd58c0, 0x1baa5c0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc0013428a0, 0xc000f39080)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001342600}, 0xc000f39080)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001342600}, {0x1f353a0, 0xc000f39080})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001342600}, {0x1f353a0, 0xc000f39080}, 0x237bc38, 0xc0022f99c8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001342600}, {0x1f353a0, 0xc000f39080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001342480, {0x23922d0, 0xc0013424b0}, {0x1f353a0, 0xc000f39080}, 0xc00132f350, 0xc000f390c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013424b0}, {0x1f353a0, 0xc000f39080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013424b0}, {0x1f353a0, 0xc000f39080}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013424b0}, {0x1f353a0, 0xc000f39080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001342420}, {0x1f353a0, 0xc000f39080}, 0x40, 0xc000f390c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001342420}, {0x1f353a0, 0xc000f39080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001342420}, {0x1f353a0, 0xc000f39080}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001342420}, {0x1f353a0, 0xc000f39080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001342420}, {0x1f353a0, 0xc000f39080}, 0xc0013e9a80, 0xc0022f99c8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001342420}, 0xc00247b260, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00177c480, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00177c480, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1324 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00003b480, 0xc001f53620)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00003b480)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1325 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00003b480, 0xc000756300)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00003b480)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1326 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00003b480, 0xc0013e2570)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00003b480)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1327 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00003b480, 0xc001fefc00)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00003b480)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1328 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00087b590)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1269 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bbaf50, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000d55260)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1242709 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d88d80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0b00, {0x1f736c0, 0xc001a27710}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016f300, 0xc000c095c0, 0xc000c09590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016f300, {0x1f736c0, 0xc001a27710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001ee23c0}, {0x2076920, 0x8}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x1, {0xc0016f7100, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x7f6c191ae108, {0xc0016f7100, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x20, 0xc001df5860, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0xc000d3fa00, {0xc001dfc640, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x0, 0xc001df5830, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0xc000d56680, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x1e0cfc0, 0xc001df57a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x2aea33c1e65, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001ee23c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x203000, 0xc001df5770, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001ee23c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x468c27, {0x2392298, 0xc001ee23c0}, {0x2076920, 0x1022586d0}, {0x1f33320, 0xc001dfc600}, {0x1f736c0, 0xc001a27710}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001ee23c0}, 0x1fe4e76, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc001ee23c0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1271016 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1259463 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001be17a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000731d90, {0x1f7c860, 0xc000731ce0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c8c00, 0xc001c935e8, 0xc001c935b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c8c00, {0x1f7c860, 0xc000731ce0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0022dac60}, {0x20768da, 0x8}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x1c75901, {0xc001881660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x7f6c191ae108, {0xc001881660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x20, 0xc001e03c20, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0xc000da4820, {0xc001d6dec0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x0, 0xc001e03bf0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x1e0cfc0, 0xc001e03b60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0022dac60}, {0x20768da, 0x203000}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x8, 0xc001e03b30, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0022dac60}, {0x20768da, 0x46}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc0022dac60}, {0x20768da, 0x0}, {0x1f33180, 0xc001e03ad0}, {0x1f7c860, 0xc000731ce0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0022dac60}, 0xc0011fdf20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc0022dac60})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1333 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0007df1d0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001608600)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 1334 [select, 7 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000817e00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1335 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf23d61c8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003b080, 0xc000c66000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00003b080, {0xc000c66000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00003b080, {0xc000c66000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e7e0, {0xc000c66000, 0xc000c04cf8, 0x44d2f4})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000c2a840, {0xc00038b000, 0x9, 0x62426b82bc})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc000c2a840}, {0xc00038b000, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038b000, 0x9, 0xc0007df220}, {0x23585c0, 0xc000c2a840})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038afc0)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000817e00, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0003fbc00, {0x23d1088, 0xc000817e00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1250089 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001d2e240)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0e580, {0x1f736c0, 0xc0025b5710}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9100, 0xc000ee15c0, 0xc000ee1590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9100, {0x1f736c0, 0xc0025b5710})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002407140}, {0x2076920, 0x8}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x1c75901, {0xc0018805a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x7f6c191ae5b8, {0xc0018805a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x20, 0xc0018839e0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0xc000d3fa00, {0xc001119a80, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x0, 0xc0018839b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x1e0cfc0, 0xc001883920, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x1e0cfc0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002407140}, {0x2076920, 0x203000}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x203000, 0xc0018838f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002407140}, {0x2076920, 0x46}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, 0x7f6c191ae5b8, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000e83650, {0x2392298, 0xc002407140}, {0x2076920, 0x1019c70d8}, {0x1f33320, 0xc0011199c0}, {0x1f736c0, 0xc0025b5710}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc002407140}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e89c70, {0x2392298, 0xc002407140})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1242715 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001ee2a80, {0x2392228, 0xc001dfd600})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1254949 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c510e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0580, {0x1f7b560, 0xc0015c2f00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d900, 0xc001b87d18, 0xc001b87ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d900, {0x1f7b560, 0xc0015c2f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00066a960}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0x20, {0xc0013e88e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00066a960}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00066a960}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0xc001853760, {0xc0013e88e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00066a960}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0xc00218fb70, 0xc00066a990, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00066a960}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001faae00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0x237ac70, 0xc00066a8d0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001faae00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001faae00}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001faad80}, {0x1f7b560, 0xc0015c2f00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001faae00}, 0xc00079e4b0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc0007a1b90}, 0xc001faad80, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc0007a1b90}, 0xc001faad80, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001b885a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b88648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20db5d8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc0007a1b90}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c002a0c490})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b88898, {0x235a640, 0xc0000a9fb0}, 0x382d613435342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc0007a1b90}, 0xc0007a1950)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc00180cdb8, 0x4c1cb4}, {0xc057f6a26647f582, 0x61522f474d}, {0xc00180cdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc0007a1b90}, 0xc0007a1950)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950}, 0xc0013e87e0, 0xc00086c3a8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950}, 0x15, 0xc001faa9c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950}, 0x207690b, 0xc001faa9c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950}, 0x237bc38, 0xc001faa9c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1b90}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0007a19b0}, {0x1f33180, 0xc0007a1950}, 0x1e5a940, 0xc001faa9c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a19b0}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950}, 0x9, 0xc001faa9c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950}, 0xc00296e768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00296eb60, {0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0007a1920}, {0x1f33180, 0xc0007a1950}, 0xc0013e87e0, 0xc00086c3a8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0007a1920}, 0xc0022d7200, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00087fb00, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00087fb00, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270926 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008a9320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbce70, {0x1f7b7c0, 0xc000dee320}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72000, 0xc0019c9bf8, 0xc0019c9bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72000, {0x1f7b7c0, 0xc000dee320})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0015ec600}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0x20, {0xc001b59a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0015ec600}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015ec600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0xc0015c6dc0, {0xc001b59a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015ec600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0xc00218fb70, 0xc0015ec630, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015ec600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0015d6880}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0x237ac70, 0xc0015ec570, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0015d6880}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0015d6880}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0015d6800}, {0x1f7b7c0, 0xc000dee320}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0015d6880}, 0xc000e29140, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc0015bfef0}, 0xc0015d6800, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc0015bfef0}, 0xc0015d6800, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0019ca480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0019ca528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2180980, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0015bfef0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e0d180})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0019ca778, {0x235a640, 0xc0000a9fb0}, 0x622d356136342d65)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc0015bfef0}, 0xc0015d6500)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000bbcdb8, 0x4c1cb4}, {0xc057f6a366bc234c, 0x62410e9d25}, {0xc000bbcdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc0015bfef0}, 0xc0015d6500)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500}, 0xc001b59880, 0xc002422df8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500}, 0x15, 0xc0015d6540)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500}, 0x2076951, 0xc0015d6540)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500}, 0x237bc38, 0xc0015d6540)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfef0}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0015bfda0}, {0x1f33320, 0xc0015d6500}, 0x1e5a940, 0xc0015d6540)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfda0}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500}, 0x23c27b0, 0xc0015d6540)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015bfd70}, {0x1f33320, 0xc0015d6500}, 0xc001b59880, 0xc002422df8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0015bfd70}, 0xc0017fde60, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0010fd680, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0010fd680, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242716 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001ee2a80, {0x2392228, 0xc001dfd600})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250095 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c8d200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001a0e8f0, {0x1f7b560, 0xc00068b400}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9700, 0xc001601d18, 0xc001601ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9700, {0x1f7b560, 0xc00068b400})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001c53ec0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0x20, {0xc001880c60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001c53ec0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c53ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0xc001fb1080, {0xc001880c60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c53ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0xc00218fb70, 0xc001c53ef0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c53ec0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000ffd180}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0x237ac70, 0xc001c53e30, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000ffd180}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000ffd180}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000ffd100}, {0x1f7b560, 0xc00068b400}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000ffd180}, 0xc000e67aa0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001c53770}, 0xc000ffd100, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001c53770}, 0xc000ffd100, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0016025a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001602648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d6cc8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001c53770}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00074afc0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001602898, {0x235a640, 0xc0000a9fb0}, 0x392d333364342d35)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001c53770}, 0xc001c535c0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001bd2db8, 0x4c1cb4}, {0xc057f6a226580ec1, 0x6116a4968d}, {0xc001bd2db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001c53770}, 0xc001c535c0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0}, 0xc001880ba0, 0xc0020c92a8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0}, 0x15, 0xc000ffcec0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0}, 0x207690b, 0xc000ffcec0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0}, 0x237bc38, 0xc000ffcec0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53770}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001c53620}, {0x1f33180, 0xc001c535c0}, 0x1e5a940, 0xc000ffcec0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53620}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0}, 0x9, 0xc000ffcec0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0}, 0xc0007f7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001c53590}, {0x1f33180, 0xc001c535c0}, 0xc001880ba0, 0xc0020c92a8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001c53590}, 0xc002407980, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001c8c000, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001c8c000, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1254964 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077f8c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbc370, {0x1f7b7c0, 0xc000cd4f00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016f500, 0xc0018024d0, 0xc0018024a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016f500, {0x1f7b7c0, 0xc000cd4f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f3b3b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0x20, {0xc000fe8a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f3b3b0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f3b3b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0xc001fb1e40, {0xc000fe8a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f3b3b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0xc00218ff30, 0xc001f3b3e0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f3b3b0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0008fe3c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0x237ac70, 0xc001f3b320, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0008fe3c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0008fe3c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0008fe200}, {0x1f7b7c0, 0xc000cd4f00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000097128, {0x2392228, 0xc0008fe3c0}, 0xc000595486, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc00068e150, {0x2392298, 0xc002662240}, 0xc0008fe200, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc00062aa20, {0x2392298, 0xc002662240}, 0xc0008fe200, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc002205740, {0x2392298, 0xc002662240})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002205980, {0x2392298, 0xc002662240}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000796240, {0x23922d0, 0xc001f3b050}, 0x14)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0020dd7d0, {0x23922d0, 0xc001f3b050}, 0xc0007cc9b0, 0x1, 0x7cc9c0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001f3aff0, 0xc00072be00)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001f3aa50}, 0xc00072be00)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001f3aa50}, {0x1f35540, 0xc00072be00})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001f3aa50}, {0x1f35540, 0xc00072be00}, 0x237bc38, 0xc002215e48)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3aa50}, {0x1f35540, 0xc00072be00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f3a900}, {0x1f35540, 0xc00072be00}, 0xc0016d31a0, 0xc00072bf80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a900}, {0x1f35540, 0xc00072be00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f3a900}, {0x1f35540, 0xc00072be00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a900}, {0x1f35540, 0xc00072be00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f3a8d0}, {0x1f35540, 0xc00072be00}, 0x40, 0xc00072bf80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a8d0}, {0x1f35540, 0xc00072be00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f3a8d0}, {0x1f35540, 0xc00072be00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a8d0}, {0x1f35540, 0xc00072be00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f3a8d0}, {0x1f35540, 0xc00072be00}, 0xc000fe8900, 0xc002215e48)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001f3a8d0}, 0xc0026621e0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a71e60, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a71e60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1254957 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c8c5a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc06e0, {0x1f736c0, 0xc00063c3f0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5f)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3db00, 0xc000b855c0, 0xc000b85590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3db00, {0x1f736c0, 0xc00063c3f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001e28000}, {0x2076920, 0x8}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x1c75901, {0xc0013e8ea0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x7f6c191ae108, {0xc0013e8ea0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x20, 0xc0007535c0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0xc000d3fa00, {0xc0008f6e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x0, 0xc000753590, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x1e0cfc0, 0xc000753500, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0xc000af5ce8, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001e28000}, {0x2076920, 0x203000}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x203000, 0xc0007534a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001e28000}, {0x2076920, 0x46}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000af5e50, {0x2392298, 0xc001e28000}, {0x2076920, 0x10aae3250}, {0x1f33320, 0xc0008f6580}, {0x1f736c0, 0xc00063c3f0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001e28000}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000da4750, {0x2392298, 0xc001e28000})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 828631 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221aac8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c85e00, 0xc000e34000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000c85e00, {0xc000e34000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000c85e00, {0xc000e34000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0006636a0, {0xc000e34000, 0x4383e0, 0xc000c31040})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0016d3020, {0xc000792200, 0x9, 0xc0007f5d18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc0016d3020}, {0xc000792200, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000792200, 0x9, 0x18}, {0x23585c0, 0xc0016d3020})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0007921c0)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001599e00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1073650 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221a728, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0020ac080, 0xc0020ee000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020ac080, {0xc0020ee000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0020ac080, {0xc0020ee000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc002201418, {0xc0020ee000, 0x466a4e, 0x4383a7})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001da5380, {0xc0007924a0, 0x9, 0xc000861d18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001da5380}, {0xc0007924a0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0007924a0, 0x9, 0xc000861d6f}, {0x23585c0, 0xc001da5380})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000792460)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000a960)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1242760 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016e4d80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac0b0, {0x1f7b7c0, 0xc000defb80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcb)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73b00, 0xc001c39bf8, 0xc001c39bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73b00, {0x1f7b7c0, 0xc000defb80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0008ca2d0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0x20, {0xc0016f62e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0008ca2d0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0008ca2d0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0xc0019d3b80, {0xc0016f62e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0008ca2d0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0xc00218fb70, 0xc0008ca300, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0008ca2d0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000bb62c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0x237ac70, 0xc0008ca090, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000bb62c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000bb62c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000bb6240}, {0x1f7b7c0, 0xc000defb80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc000bb62c0}, 0xc002618840, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00050fb60}, 0xc000bb6240, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00050fb60}, 0xc000bb6240, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001c3a480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c3a528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf224f510, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00050fb60}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c002a0c700})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c3a778, {0x235a640, 0xc0000a9fb0}, 0x382d366335342d32)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00050fb60}, 0xc0010efb40)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001cacdb8, 0x4c1cb4}, {0xc057f6a1a95250d6, 0x60a26944af}, {0xc001cacdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00050fb60}, 0xc0010efb40)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40}, 0xc0016f6200, 0xc001d46ed0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40}, 0x15, 0xc0010efb80)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40}, 0x2076951, 0xc0010efb80)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40}, 0x237bc38, 0xc0010efb80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00050fb60}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000465560}, {0x1f33320, 0xc0010efb40}, 0x1e5a940, 0xc0010efb80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000465560}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40}, 0x23c27b0, 0xc0010efb80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000465500}, {0x1f33320, 0xc0010efb40}, 0xc0016f6200, 0xc001d46ed0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000465500}, 0xc001f1e9c0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077fb00, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00077fb00, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1259509 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000846de0, {0x2392298, 0xc00240d980}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001ba4000, {0x23922d0, 0xc001dfbf50}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001dfbf50}, 0xc001e82ed0, 0x1bd58c0, 0x1e82ee0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001dfbd40, 0xc001e8a780)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001dfbaa0}, 0xc001e8a780)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001dfbaa0}, {0x1f353a0, 0xc001e8a780})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001dfbaa0}, {0x1f353a0, 0xc001e8a780}, 0x237bc38, 0xc001e923a8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dfbaa0}, {0x1f353a0, 0xc001e8a780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7, {0x23922d0, 0xc001dfb950}, {0x1f353a0, 0xc001e8a780}, 0xc0017c75b0, 0xc001e8a7c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dfb950}, {0x1f353a0, 0xc001e8a780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001dfb950}, {0x1f353a0, 0xc001e8a780}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dfb950}, {0x1f353a0, 0xc001e8a780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001dfb8f0}, {0x1f353a0, 0xc001e8a780}, 0x40, 0xc001e8a7c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dfb8f0}, {0x1f353a0, 0xc001e8a780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001dfb8f0}, {0x1f353a0, 0xc001e8a780}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001dfb8f0}, {0x1f353a0, 0xc001e8a780})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001dfb8f0}, {0x1f353a0, 0xc001e8a780}, 0xc000e9b140, 0xc001e923a8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001dfb8f0}, 0xc00240d920, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0016a9c20, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0016a9c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245964 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000f37e50)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1250137 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0022c8ea0, {0x2392298, 0xc00260c720}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fcd80, {0x23922d0, 0xc00060bd70}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00060bd70}, 0xc002014390, 0x1bd58c0, 0x20143a0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00060bd10, 0xc001ca2f00)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc00060ba40}, 0xc001ca2f00)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00060ba40}, {0x1f353a0, 0xc001ca2f00})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00060ba40}, {0x1f353a0, 0xc001ca2f00}, 0x237bc38, 0xc001866450)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00060ba40}, {0x1f353a0, 0xc001ca2f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00060b8c0, {0x23922d0, 0xc00060b8f0}, {0x1f353a0, 0xc001ca2f00}, 0xc002431b60, 0xc001ca2f40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00060b8f0}, {0x1f353a0, 0xc001ca2f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00060b8f0}, {0x1f353a0, 0xc001ca2f00}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00060b8f0}, {0x1f353a0, 0xc001ca2f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00060b860}, {0x1f353a0, 0xc001ca2f00}, 0x40, 0xc001ca2f40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00060b860}, {0x1f353a0, 0xc001ca2f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00060b860}, {0x1f353a0, 0xc001ca2f00}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00060b860}, {0x1f353a0, 0xc001ca2f00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00060b860}, {0x1f353a0, 0xc001ca2f00}, 0xc000e9bd80, 0xc001866450)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00060b860}, 0xc00260c6c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00166b9e0, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00166b9e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270844 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0016848a0, {0x2392298, 0xc001c125a0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001688000, {0x23922d0, 0xc00139d0e0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001ea5e30, {0x23922d0, 0xc00139d0e0}, 0xc001381c80, 0x1, 0x1381c90)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc00139d080, 0xc0013a0340)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc00139cdb0}, 0xc0013a0340)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00139cdb0}, {0x1f35540, 0xc0013a0340})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00139cdb0}, {0x1f35540, 0xc0013a0340}, 0x237bc38, 0xc001e52000)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00139cdb0}, {0x1f35540, 0xc0013a0340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00139cc60}, {0x1f35540, 0xc0013a0340}, 0xc0012f9d00, 0xc0013a03c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00139cc60}, {0x1f35540, 0xc0013a0340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00139cc60}, {0x1f35540, 0xc0013a0340}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00139cc60}, {0x1f35540, 0xc0013a0340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00139cc30}, {0x1f35540, 0xc0013a0340}, 0x40, 0xc0013a03c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00139cc30}, {0x1f35540, 0xc0013a0340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00139cc30}, {0x1f35540, 0xc0013a0340}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00139cc30}, {0x1f35540, 0xc0013a0340})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00139cc30}, {0x1f35540, 0xc0013a0340}, 0xc000ba1860, 0xc001e52000)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00139cc30}, 0xc001c12540, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c1aea0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c1aea0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1068203 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000412e70, {0x2392228, 0xc002375800})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1250162 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002166d80, {0x2392298, 0xc00260dc20}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fc6c0, {0x23922d0, 0xc0027088d0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0027088d0}, 0xc002015630, 0x1bd58c0, 0x2015640)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc002708870, 0xc000fc2100)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc0027085d0}, 0xc000fc2100)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0027085d0}, {0x1f353a0, 0xc000fc2100})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0027085d0}, {0x1f353a0, 0xc000fc2100}, 0x237bc38, 0xc0018675d8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0027085d0}, {0x1f353a0, 0xc000fc2100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002708450, {0x23922d0, 0xc002708480}, {0x1f353a0, 0xc000fc2100}, 0xc0018f5410, 0xc000fc2140)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002708480}, {0x1f353a0, 0xc000fc2100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002708480}, {0x1f353a0, 0xc000fc2100}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002708480}, {0x1f353a0, 0xc000fc2100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0027083f0}, {0x1f353a0, 0xc000fc2100}, 0x40, 0xc000fc2140)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0027083f0}, {0x1f353a0, 0xc000fc2100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0027083f0}, {0x1f353a0, 0xc000fc2100}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0027083f0}, {0x1f353a0, 0xc000fc2100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0027083f0}, {0x1f353a0, 0xc000fc2100}, 0xc0016f6920, 0xc0018675d8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0027083f0}, 0xc00260dbc0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ec70e0, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ec70e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1360 [select, 7 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0015d8320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246

goroutine 1250315 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000797200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac840, {0x1f7c860, 0xc001bac790}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d300, 0xc001ccb5e8, 0xc001ccb5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d300, {0x1f7c860, 0xc001bac790})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001e86a20}, {0x20768da, 0x8}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x1c75901, {0xc000ba1a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x7f6c191ae5b8, {0xc000ba1a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x20, 0xc001f71560, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0xc000da4820, {0xc001e19f40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x0, 0xc001f71530, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x1e0cfc0, 0xc001f71440, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001e86a20}, {0x20768da, 0x203000}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x8, 0xc001f71410, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001e86a20}, {0x20768da, 0x46}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, 0x144aacf, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc001e86a20}, {0x20768da, 0x0}, {0x1f33180, 0xc001f70f90}, {0x1f7c860, 0xc001bac790}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc001e86a20}, 0xc0013726e0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc001e86a20})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1245899 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020164e0, {0x2392298, 0xc0021adf20}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005de360, {0x23922d0, 0xc00209a990}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00209a990}, 0xc00137e420, 0x1bd58c0, 0x137e430)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00209a930, 0xc000e5bac0)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc00209a690}, 0xc000e5bac0)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00209a690}, {0x1f353a0, 0xc000e5bac0})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00209a690}, {0x1f353a0, 0xc000e5bac0}, 0x237bc38, 0xc001cff3b0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00209a690}, {0x1f353a0, 0xc000e5bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00209a510, {0x23922d0, 0xc00209a540}, {0x1f353a0, 0xc000e5bac0}, 0xc001edb7d0, 0xc000e5bb00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00209a540}, {0x1f353a0, 0xc000e5bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00209a540}, {0x1f353a0, 0xc000e5bac0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00209a540}, {0x1f353a0, 0xc000e5bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00209a4b0}, {0x1f353a0, 0xc000e5bac0}, 0x40, 0xc000e5bb00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00209a4b0}, {0x1f353a0, 0xc000e5bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00209a4b0}, {0x1f353a0, 0xc000e5bac0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00209a4b0}, {0x1f353a0, 0xc000e5bac0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00209a4b0}, {0x1f353a0, 0xc000e5bac0}, 0xc000fe9ac0, 0xc001cff3b0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00209a4b0}, 0xc0021adec0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a52000, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001a52000, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1385 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf22079f8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0015d2480, 0xc001670000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0015d2480, {0xc001670000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0015d2480, {0xc001670000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f2a0, {0xc001670000, 0xa95530, 0xc000de99a0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001653080, {0xc00038b460, 0x9, 0x18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001653080}, {0xc00038b460, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038b460, 0x9, 0x6190b298e0}, {0x23585c0, 0xc001653080})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038b420)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0005c0780)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef

goroutine 1250195 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0025afd40, {0x2392298, 0xc0021cbb00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a61e60, {0x23922d0, 0xc000c239b0}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000c239b0}, 0xc00207b290, 0x1bd58c0, 0x207b2a0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc000c23950, 0xc001eb3500)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc000c235c0}, 0xc001eb3500)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000c235c0}, {0x1f353a0, 0xc001eb3500})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc000c235c0}, {0x1f353a0, 0xc001eb3500}, 0x237bc38, 0xc000899800)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c235c0}, {0x1f353a0, 0xc001eb3500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc000c23440, {0x23922d0, 0xc000c23470}, {0x1f353a0, 0xc001eb3500}, 0xc000c22780, 0xc001eb3540)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c23470}, {0x1f353a0, 0xc001eb3500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000c23470}, {0x1f353a0, 0xc001eb3500}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c23470}, {0x1f353a0, 0xc001eb3500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000c233e0}, {0x1f353a0, 0xc001eb3500}, 0x40, 0xc001eb3540)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c233e0}, {0x1f353a0, 0xc001eb3500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000c233e0}, {0x1f353a0, 0xc001eb3500}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000c233e0}, {0x1f353a0, 0xc001eb3500})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000c233e0}, {0x1f353a0, 0xc001eb3500}, 0xc000ba15a0, 0xc000899800)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000c233e0}, 0xc0021cbaa0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001855440, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001855440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1054791 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001b69b60, {0x2392228, 0xc001bc6a00})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 846499 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00067ccc0, {0x2392228, 0xc001a62300})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250148 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1a6c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000730840, {0x1f7c860, 0xc0007306e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9c00, 0xc00180f5e8, 0xc00180f5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9c00, {0x1f7c860, 0xc0007306e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00275d080}, {0x20768da, 0x8}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x1c75901, {0xc001881880, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x7f6c191ae5b8, {0xc001881880, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x20, 0xc001856ed0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0xc000da4820, {0xc0017cb7c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x0, 0xc001856ea0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x1e0cfc0, 0xc001856e10, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x4167af, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00275d080}, {0x20768da, 0x203000}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x2e, 0xc001856de0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00275d080}, {0x20768da, 0x46}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc00275d080}, {0x20768da, 0x0}, {0x1f33180, 0xc001856d80}, {0x1f7c860, 0xc0007306e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc00275d080}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc00275d080})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1245812 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008ba7e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1ad0, {0x1f7c860, 0xc001dc1a20}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x4a)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9f00, 0xc0015df5e8, 0xc0015df5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9f00, {0x1f7c860, 0xc001dc1a20})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0021ac180}, {0x20768da, 0x8}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x1c75901, {0xc000fe9220, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x7f6c191ae5b8, {0xc000fe9220, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x20, 0xc0016345a0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0xc000da4820, {0xc000e3d9c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x0, 0xc001634570, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x1e0cfc0, 0xc0016344e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0021ac180}, {0x20768da, 0x203000}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x8, 0xc0016344b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0021ac180}, {0x20768da, 0x46}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, 0x144aacf, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc0021ac180}, {0x20768da, 0x0}, {0x1f33180, 0xc001634450}, {0x1f7c860, 0xc001dc1a20}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0021ac180}, 0xc002252ee0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00003b480, {0x2392298, 0xc0021ac180})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1259465 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00083db00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001646210, {0x1f7b560, 0xc0015c3d60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c9200, 0xc0017e3d18, 0xc0017e3ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c9200, {0x1f7b560, 0xc0015c3d60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e24900}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0x20, {0xc001881c60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e24900}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e24900}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0xc000ec9080, {0xc001881c60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e24900}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0xc00218fb70, 0xc001e24930, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e24900}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e1c380}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0x237ac70, 0xc001e24870, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e1c380}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e1c380}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e1c300}, {0x1f7b560, 0xc0015c3d60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001e1c380}, 0xc002372fc0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001e241b0}, 0xc001e1c300, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001e241b0}, 0xc001e1c300, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0017e45a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0017e4648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21b9570, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001e241b0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e0c470})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0017e4898, {0x235a640, 0xc0000a9fb0}, 0x612d336461342d37)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001e241b0}, 0xc001e24000)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0018c0db8, 0x4c1cb4}, {0xc057f6a2a6f60957, 0x618e78252f}, {0xc0018c0db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001e241b0}, 0xc001e24000)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000}, 0xc0018817e0, 0xc001e0ea68)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000}, 0x15, 0xc001e1c0c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000}, 0x207690b, 0xc001e1c0c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000}, 0x237bc38, 0xc001e1c0c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e241b0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001e24060}, {0x1f33180, 0xc001e24000}, 0x1e5a940, 0xc001e1c0c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e24060}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000}, 0x9, 0xc001e1c0c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000}, 0xc000987768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000987b60, {0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e03fb0}, {0x1f33180, 0xc001e24000}, 0xc0018817e0, 0xc001e0ea68)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001e03fb0}, 0xc0022daea0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001be18c0, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001be18c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242820 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016a9b00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbd130, {0x1f7b560, 0xc00068a140}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016fb00, 0xc0013d2440, 0xc0013d2410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016fb00, {0x1f7b560, 0xc00068a140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001dfb410}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0x20, {0xc000e9b000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001dfb410}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001dfb410}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0xc00162b4a0, {0xc000e9b000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001dfb410}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0xc001fb6920, 0xc001dfb440, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001dfb410}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e8a500}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0x237ac70, 0xc001dfb380, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e8a500}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e8a500}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e8a480}, {0x1f7b560, 0xc00068a140}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc001e8a500}, 0xc002277686, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc001f1ff80}, 0xc001e8a480, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc001f1ff80}, 0xc001e8a480, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc000846240, {0x2392298, 0xc001f1ff80})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000846480, {0x2392298, 0xc001f1ff80}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000ef0d80, {0x23922d0, 0xc001f3b710}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001f3b710}, 0xc0005796f0, 0x1bd58c0, 0x579710)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001f3b530, 0xc001fab480)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001f3b0e0}, 0xc001fab480)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f3b0e0}, {0x1f353a0, 0xc001fab480})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001f3b0e0}, {0x1f353a0, 0xc001fab480}, 0x237bc38, 0xc0020b2810)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3b0e0}, {0x1f353a0, 0xc001fab480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001f3af30, {0x23922d0, 0xc001f3af60}, {0x1f353a0, 0xc001fab480}, 0xc00078bf50, 0xc001fab4c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3af60}, {0x1f353a0, 0xc001fab480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f3af60}, {0x1f353a0, 0xc001fab480}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3af60}, {0x1f353a0, 0xc001fab480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f3aed0}, {0x1f353a0, 0xc001fab480}, 0x40, 0xc001fab4c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3aed0}, {0x1f353a0, 0xc001fab480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f3aed0}, {0x1f353a0, 0xc001fab480}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3aed0}, {0x1f353a0, 0xc001fab480})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f3aed0}, {0x1f353a0, 0xc001fab480}, 0xc0016f6b20, 0xc0020b2810)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001f3aed0}, 0xc001f1ff20, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ec79e0, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ec79e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1371 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0015d8c80)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1372 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003f8460, {0x2392228, 0xc0015fef40})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1373 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001684720, {0x2392228, 0xc0015fef80})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1374 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001684720, {0x2392228, 0xc0015fef80})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1259460 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002166d80, {0x2392298, 0xc0022da960}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0010fc6c0, {0x23922d0, 0xc001e03440}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001e03440}, 0xc001d73680, 0x1bd58c0, 0x1d73690)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001e03230, 0xc001d6d900)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001e02f90}, 0xc001d6d900)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e02f90}, {0x1f353a0, 0xc001d6d900})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001e02f90}, {0x1f353a0, 0xc001d6d900}, 0x237bc38, 0xc001e0e390)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e02f90}, {0x1f353a0, 0xc001d6d900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7, {0x23922d0, 0xc001e02e40}, {0x1f353a0, 0xc001d6d900}, 0xc0009875b0, 0xc001d6d940)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e02e40}, {0x1f353a0, 0xc001d6d900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e02e40}, {0x1f353a0, 0xc001d6d900}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e02e40}, {0x1f353a0, 0xc001d6d900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e02de0}, {0x1f353a0, 0xc001d6d900}, 0x40, 0xc001d6d940)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e02de0}, {0x1f353a0, 0xc001d6d900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e02de0}, {0x1f353a0, 0xc001d6d900}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e02de0}, {0x1f353a0, 0xc001d6d900})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e02de0}, {0x1f353a0, 0xc001d6d900}, 0xc0018814a0, 0xc001e0e390)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001e02de0}, 0xc0022da900, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0008d6ea0, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc0008d6ea0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245919 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001a610e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001badce0, {0x1f7b560, 0xc000cd5d60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036300, 0xc0016dbd18, 0xc0016dbce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036300, {0x1f7b560, 0xc000cd5d60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001fd71a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0x20, {0xc0013e9220, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001fd71a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd71a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0xc000f9dce0, {0xc0013e9220, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001fd71a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0xc00218fb70, 0xc001fd71d0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001fd71a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000f26540}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0x237ac70, 0xc001fd7110, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000f26540}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000f26540}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000f264c0}, {0x1f7b560, 0xc000cd5d60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000f26540}, 0xc0016b6cc0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001fd6a50}, 0xc000f264c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001fd6a50}, 0xc000f264c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0016dc5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016dc648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d49b8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001fd6a50}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e51f10})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016dc898, {0x235a640, 0xc0000a9fb0}, 0x622d623364342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001fd6a50}, 0xc001fd68a0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0017b4db8, 0x4c1cb4}, {0xc057f6a1e94f8fb8, 0x60de014d85}, {0xc0017b4db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001fd6a50}, 0xc001fd68a0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0}, 0xc0013e9160, 0xc002595ba8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0}, 0x15, 0xc000f26280)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0}, 0x207690b, 0xc000f26280)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0}, 0x237bc38, 0xc000f26280)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6a50}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001fd6900}, {0x1f33180, 0xc001fd68a0}, 0x1e5a940, 0xc000f26280)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6900}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0}, 0x9, 0xc000f26280)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0}, 0xc002236768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002236b60, {0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0}, 0x40, 0xc00005ec00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001fd6870}, {0x1f33180, 0xc001fd68a0}, 0xc0013e9160, 0xc002595ba8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001fd6870}, 0xc0025af5c0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001ba5e60, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001ba5e60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1386 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001666500, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0016530e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45

goroutine 1391 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc000ba17e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b770)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b770)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1388 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0016665f0, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001653320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 1389 [select, 7 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000da9080)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1390 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207ae0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00166e180, 0xc001678000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00166e180, {0xc001678000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00166e180, {0xc001678000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f2c8, {0xc001678000, 0x40ce34, 0xc001650b60})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001653140, {0xc00038b540, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001653140}, {0xc00038b540, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038b540, 0x9, 0xc000ff9098}, {0x23585c0, 0xc001653140})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038b500)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000da9080, 0xab3c05, 0xab15af)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0003fbc00, {0x23d1088, 0xc000da9080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1392 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00087b770)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1409 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000ec4f70, 0xc0013414a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000ec4f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1410 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000ec4f70, 0xc001e9c6e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000ec4f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1270871 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0016848a0, {0x2392298, 0xc001c13c80}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001688000, {0x23922d0, 0xc0013b4c00}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002372570, {0x23922d0, 0xc0013b4c00}, 0xc0013a52a0, 0x1, 0x13a52b0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0013b4ba0, 0xc0013b6040)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0013b4930}, 0xc0013b6040)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0013b4930}, {0x1f35540, 0xc0013b6040})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0013b4930}, {0x1f35540, 0xc0013b6040}, 0x237bc38, 0xc001e53458)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013b4930}, {0x1f35540, 0xc0013b6040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0013b47e0}, {0x1f35540, 0xc0013b6040}, 0xc0013a1680, 0xc0013b60c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013b47e0}, {0x1f35540, 0xc0013b6040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013b47e0}, {0x1f35540, 0xc0013b6040}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013b47e0}, {0x1f35540, 0xc0013b6040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0013b47b0}, {0x1f35540, 0xc0013b6040}, 0x40, 0xc0013b60c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013b47b0}, {0x1f35540, 0xc0013b6040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0013b47b0}, {0x1f35540, 0xc0013b6040}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013b47b0}, {0x1f35540, 0xc0013b6040})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0013b47b0}, {0x1f35540, 0xc0013b6040}, 0xc000126300, 0xc001e53458)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0013b47b0}, 0xc001c13c20, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c099e0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c099e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250158 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000408380, {0x2392228, 0xc002475100})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1411 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000ec4f70, 0xc0013783b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000ec4f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1412 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000ec4f70, 0xc001fee6b0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc000ec4f70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1413 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00087b680)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1270894 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016a98c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbc9a0, {0x1f736c0, 0xc00132d8c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8d00, 0xc0016635c0, 0xc001663590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8d00, {0x1f736c0, 0xc00132d8c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000efd320}, {0x2076920, 0x8}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x1c75901, {0xc001b58880, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x7f6c191ae108, {0xc001b58880, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x20, 0xc001347620, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0xc000d3fa00, {0xc001344e40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x0, 0xc0013475f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x0, 0xc001347560, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000efd320}, {0x2076920, 0x203000}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x203000, 0xc001347530, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000efd320}, {0x2076920, 0x46}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc000efd320}, {0x2076920, 0x200010001}, {0x1f33320, 0xc001344e00}, {0x1f736c0, 0xc00132d8c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc000efd320}, 0x2, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000ec4f70, {0x2392298, 0xc000efd320})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1250330 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0025afd40, {0x2392298, 0xc002250480}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a61e60, {0x23922d0, 0xc000c64060}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc000c64060}, 0xc002106050, 0x1bd58c0, 0x2106060)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc000c64000, 0xc001bc7940)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc0018b9d40}, 0xc001bc7940)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0018b9d40}, {0x1f353a0, 0xc001bc7940})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0018b9d40}, {0x1f353a0, 0xc001bc7940}, 0x237bc38, 0xc001effea8)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018b9d40}, {0x1f353a0, 0xc001bc7940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0018b9b00, {0x23922d0, 0xc0018b9b30}, {0x1f353a0, 0xc001bc7940}, 0xc002204000, 0xc001bc7980)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018b9b30}, {0x1f353a0, 0xc001bc7940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0018b9b30}, {0x1f353a0, 0xc001bc7940}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018b9b30}, {0x1f353a0, 0xc001bc7940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0018b9aa0}, {0x1f353a0, 0xc001bc7940}, 0x40, 0xc001bc7980)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018b9aa0}, {0x1f353a0, 0xc001bc7940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0018b9aa0}, {0x1f353a0, 0xc001bc7940}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0018b9aa0}, {0x1f353a0, 0xc001bc7940})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0018b9aa0}, {0x1f353a0, 0xc001bc7940}, 0xc0016f7f40, 0xc001effea8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0018b9aa0}, 0xc002250420, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016c0900, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016c0900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250100 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0017905a0, {0x2392228, 0xc001e33840})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1403 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0015d9950)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1404 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003f8690, {0x2392228, 0xc0016aba80})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1405 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0016d23c0, {0x2392228, 0xc0016abac0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1406 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0016d23c0, {0x2392228, 0xc0016abac0})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250099 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0000b4310, {0x2392228, 0xc001e33800})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1242687 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00261fe60, {0x2392298, 0xc0021bb080}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0015fa6c0, {0x23922d0, 0xc001e6e3c0}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001b9bb00, {0x23922d0, 0xc001e6e3c0}, 0xc001e60700, 0x1, 0x1e60710)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001e6e1b0, 0xc001e5a980)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001e55ef0}, 0xc001e5a980)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e55ef0}, {0x1f35540, 0xc001e5a980})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001e55ef0}, {0x1f35540, 0xc001e5a980}, 0x237bc38, 0xc0023cf068)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e55ef0}, {0x1f35540, 0xc001e5a980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x1ba3c00, {0x23922d0, 0xc001e55da0}, {0x1f35540, 0xc001e5a980}, 0xc0021bae40, 0xc001e5aa00)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e55da0}, {0x1f35540, 0xc001e5a980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e55da0}, {0x1f35540, 0xc001e5a980}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e55da0}, {0x1f35540, 0xc001e5a980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e55d70}, {0x1f35540, 0xc001e5a980}, 0x40, 0xc001e5aa00)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e55d70}, {0x1f35540, 0xc001e5a980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e55d70}, {0x1f35540, 0xc001e5a980}, 0x203000, 0x203000)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e55d70}, {0x1f35540, 0xc001e5a980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e55d70}, {0x1f35540, 0xc001e5a980}, 0xc000ba06a0, 0xc0023cf068)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001e55d70}, 0xc0021bb020, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001d2e900, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001d2e900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250316 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010fc900)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac8f0, {0x1f7b560, 0xc000cd48c0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d400, 0xc001b71d18, 0xc001b71ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d400, {0x1f7b560, 0xc000cd48c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0007c6510}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0x20, {0xc000ba1c60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0007c6510}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0007c6510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0xc002197600, {0xc000ba1c60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0007c6510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0xc00218fb70, 0xc0007c6540, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0007c6510}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001cfa640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0x237ac70, 0xc0007c6450, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001cfa640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001cfa640}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001cfa580}, {0x1f7b560, 0xc000cd48c0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001cfa640}, 0xc0016fffb0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001f71b60}, 0xc001cfa580, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001f71b60}, 0xc001cfa580, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001b725a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b72648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d6910, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001f71b60}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e20f10})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001b72898, {0x235a640, 0xc0000a9fb0}, 0x612d636533342d34)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001f71b60}, 0xc001f71920)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000b9cdb8, 0x4c1cb4}, {0xc057f6a22976b840, 0x6119c34018}, {0xc000b9cdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001f71b60}, 0xc001f71920)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920}, 0xc000ba1b80, 0xc002013d70)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920}, 0x15, 0xc001cfa180)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920}, 0x207690b, 0xc001cfa180)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920}, 0x237bc38, 0xc001cfa180)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f71b60}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f71980}, {0x1f33180, 0xc001f71920}, 0x1e5a940, 0xc001cfa180)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f71980}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920}, 0x9, 0xc001cfa180)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920}, 0xc000980768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000980b60, {0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920}, 0x40, 0xc00005e400)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f718f0}, {0x1f33180, 0xc001f71920}, 0xc000ba1b80, 0xc002013d70)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001f718f0}, 0xc001e86c60, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000797440, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000797440, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242743 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001ee3aa0, {0x2392228, 0xc001ea6d00})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1255011 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002663aa0, {0x2392228, 0xc001fe3900})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1269356 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1242719 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001a718c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc0d10, {0x1f736c0, 0xc001a27d40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x69)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3c400, 0xc0016bb5c0, 0xc0016bb590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3c400, {0x1f736c0, 0xc001a27d40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001ee33e0}, {0x2076920, 0x8}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x1, {0xc0016f7780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x7f6c191ae108, {0xc0016f7780, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x20, 0xc001ea04b0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0xc000d3fa00, {0xc001dfdd00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x0, 0xc001ea0480, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0xc00174e1a0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x1e0cfc0, 0xc001ea03f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x2b0017224e8, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001ee33e0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x203000, 0xc001ea03c0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001ee33e0}, {0x2076920, 0x46}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x468c27, {0x2392298, 0xc001ee33e0}, {0x2076920, 0x100000000}, {0x1f33320, 0xc001dfdcc0}, {0x1f736c0, 0xc001a27d40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001ee33e0}, 0x1fe4e76, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000ec4f70, {0x2392298, 0xc001ee33e0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1269321 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1073601 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf221abb0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001f8b580, 0xc002048000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f8b580, {0xc002048000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001f8b580, {0xc002048000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc002201328, {0xc002048000, 0xff8f, 0x40ce34})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c3cc00, {0xc0007922e0, 0x9, 0x18})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc001c3cc00}, {0xc0007922e0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0007922e0, 0x9, 0xc00003e648}, {0x23585c0, 0xc001c3cc00})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0007922a0)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00258ed80, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000183880, {0x23d1088, 0xc00258ed80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1245777 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc000e2d310)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1270907 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017ce120, {0x2392298, 0xc0017fc420}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0017898c0, {0x23922d0, 0xc0015b0b40}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0000a88d0, {0x23922d0, 0xc0015b0b40}, 0xc001378a70, 0x1, 0x1378a80)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0015b0ae0, 0xc00137a800)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0015b0870}, 0xc00137a800)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0015b0870}, {0x1f35540, 0xc00137a800})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0015b0870}, {0x1f35540, 0xc00137a800}, 0x237bc38, 0xc001f91938)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015b0870}, {0x1f35540, 0xc00137a800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0015b0720}, {0x1f35540, 0xc00137a800}, 0xc00137a200, 0xc00137a880)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015b0720}, {0x1f35540, 0xc00137a800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015b0720}, {0x1f35540, 0xc00137a800}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015b0720}, {0x1f35540, 0xc00137a800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0015b06f0}, {0x1f35540, 0xc00137a800}, 0x40, 0xc00137a880)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015b06f0}, {0x1f35540, 0xc00137a800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0015b06f0}, {0x1f35540, 0xc00137a800}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0015b06f0}, {0x1f35540, 0xc00137a800})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0015b06f0}, {0x1f35540, 0xc00137a800}, 0xc001b58fa0, 0xc001f91938)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0015b06f0}, 0xc0017fc3c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016c07e0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0016c07e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245814 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008117a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1b80, {0x1f7b560, 0xc001769720}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8000, 0xc001bddd18, 0xc001bddce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8000, {0x1f7b560, 0xc001769720})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001635350}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0x20, {0xc000fe94a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001635350}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001635350}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0xc000f34dc0, {0xc000fe94a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001635350}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0xc00218fb70, 0xc001635380, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001635350}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e3de80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0x237ac70, 0xc0016352c0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e3de80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e3de80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e3de00}, {0x1f7b560, 0xc001769720}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000e3de80}, 0xc00079f578, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001634c00}, 0xc000e3de00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001634c00}, 0xc000e3de00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001bde5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001bde648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21c9820, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001634c00}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001742930})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001bde898, {0x235a640, 0xc0000a9fb0}, 0x622d666234342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001634c00}, 0xc001634a50)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0015cadb8, 0x4c1cb4}, {0xc057f6a1e7a73a54, 0x60dc58f82d}, {0xc0015cadb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001634c00}, 0xc001634a50)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50}, 0xc000fe93a0, 0xc002244750)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50}, 0x15, 0xc000e3dbc0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50}, 0x207690b, 0xc000e3dbc0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50}, 0x237bc38, 0xc000e3dbc0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634c00}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001634ab0}, {0x1f33180, 0xc001634a50}, 0x1e5a940, 0xc000e3dbc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634ab0}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50}, 0x9, 0xc000e3dbc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50}, 0xc002239768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002239b60, {0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001634a20}, {0x1f33180, 0xc001634a50}, 0xc000fe93a0, 0xc002244750)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001634a20}, 0xc0021ac3c0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008bb0e0, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008bb0e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1434 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0016fb180)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1435 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003f8930, {0x2392228, 0xc0017ca240})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1436 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001791f80, {0x2392228, 0xc0017ca280})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1437 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001791f80, {0x2392228, 0xc0017ca280})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1245789 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008d6c60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000731600, {0x1f7b560, 0xc0015c3ae0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0008c8200, 0xc002848440, 0xc002848410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008c8200, {0x1f7b560, 0xc0015c3ae0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e02900}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0x20, {0xc001881360, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e02900}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e02900}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0xc002073340, {0xc001881360, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e02900}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0xc001fb6920, 0xc001e02930, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e02900}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d6d680}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0x237ac70, 0xc001e02870, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d6d680}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d6d680}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001d6d600}, {0x1f7b560, 0xc0015c3ae0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc001d6d680}, 0xc0025f3266, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc001f49620}, 0xc001d6d600, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc001f49620}, 0xc001d6d600, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0020c0a20, {0x2392298, 0xc001f49620})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0020c0d20, {0x2392298, 0xc001f49620}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00089d0e0, {0x23922d0, 0xc00236b470}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00236b470}, 0xc001a7d6f0, 0x1bd58c0, 0x1a7d700)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00236b410, 0xc000dd6e40)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc00236b0b0}, 0xc000dd6e40)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00236b0b0}, {0x1f353a0, 0xc000dd6e40})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00236b0b0}, {0x1f353a0, 0xc000dd6e40}, 0x237bc38, 0xc001efbe30)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236b0b0}, {0x1f353a0, 0xc000dd6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00236ac90, {0x23922d0, 0xc00236acc0}, {0x1f353a0, 0xc000dd6e40}, 0xc0016265b0, 0xc000dd6e80)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236acc0}, {0x1f353a0, 0xc000dd6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00236acc0}, {0x1f353a0, 0xc000dd6e40}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236acc0}, {0x1f353a0, 0xc000dd6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00236ac30}, {0x1f353a0, 0xc000dd6e40}, 0x40, 0xc000dd6e80)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236ac30}, {0x1f353a0, 0xc000dd6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00236ac30}, {0x1f353a0, 0xc000dd6e40}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236ac30}, {0x1f353a0, 0xc000dd6e40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00236ac30}, {0x1f353a0, 0xc000dd6e40}, 0xc0013e8880, 0xc001efbe30)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00236ac30}, 0xc001f495c0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00175a900, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00175a900, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242726 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c8cc60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bad810, {0x1f7b7c0, 0xc0015c30e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73a00, 0xc002223bf8, 0xc002223bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73a00, {0x1f7b7c0, 0xc0015c30e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e7e630}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0x20, {0xc000ba11a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e7e630}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e7e630}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0xc001c74b00, {0xc000ba11a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e7e630}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0xc00218fb70, 0xc001e7e660, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e7e630}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e5ba80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0x237ac70, 0xc001e7e5a0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e5ba80}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e5ba80}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001e5ba00}, {0x1f7b7c0, 0xc0015c30e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc001e5ba80}, 0xc002618180, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc001e6fec0}, 0xc001e5ba00, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc001e6fec0}, 0xc001e5ba00, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc002224480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc002224528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf22518a8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001e6fec0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c002a0da10})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc002224778, {0x235a640, 0xc0000a9fb0}, 0x392d663362342d62)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc001e6fec0}, 0xc001e5b700)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001836db8, 0x4c1cb4}, {0xc057f6a1a86da660, 0x60a1849a35}, {0xc001836db8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc001e6fec0}, 0xc001e5b700)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700}, 0xc000ba0f20, 0xc001d46030)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700}, 0x15, 0xc001e5b740)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700}, 0x2076951, 0xc001e5b740)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700}, 0x237bc38, 0xc001e5b740)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fec0}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc0017acd00, {0x23922d0, 0xc001e6fd70}, {0x1f33320, 0xc001e5b700}, 0x203000, 0xc001e5b740)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fd70}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700}, 0x23c27b0, 0xc001e5b740)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00134fb60, {0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700}, 0x40, 0xc00005ec00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e6fd40}, {0x1f33320, 0xc001e5b700}, 0xc000ba0f20, 0xc001d46030)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001e6fd40}, 0xc0021bbce0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000ae1c20, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000ae1c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1059763 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001051270)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1064364 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002205680, {0x2392228, 0xc001119b00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1271009 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001c13020, {0x2392298, 0xc0022919e0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001c1a7e0, {0x23922d0, 0xc0013ef6b0}, 0x31)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002372ba0, {0x23922d0, 0xc0013ef6b0}, 0xc0013f0380, 0x1, 0x13f0390)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0013ef650, 0xc0013e7140)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0013ef3e0}, 0xc0013e7140)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0013ef3e0}, {0x1f35540, 0xc0013e7140})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0013ef3e0}, {0x1f35540, 0xc0013e7140}, 0x237bc38, 0xc001ebd128)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ef3e0}, {0x1f35540, 0xc0013e7140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0013ef290}, {0x1f35540, 0xc0013e7140}, 0xc0013e6b40, 0xc0013e71c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ef290}, {0x1f35540, 0xc0013e7140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0013ef290}, {0x1f35540, 0xc0013e7140}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ef290}, {0x1f35540, 0xc0013e7140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0013ef260}, {0x1f35540, 0xc0013e7140}, 0x40, 0xc0013e71c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ef260}, {0x1f35540, 0xc0013e7140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0013ef260}, {0x1f35540, 0xc0013e7140}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013ef260}, {0x1f35540, 0xc0013e7140})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0013ef260}, {0x1f35540, 0xc0013e7140}, 0xc001cb45e0, 0xc001ebd128)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0013ef260}, 0xc002291980, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00077b680, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00077b680, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1242713 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001e8e4b0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1250102 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001790840, {0x2392298, 0xc001790f00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001c519e0, {0x23922d0, 0xc0004558c0}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001f95980, {0x23922d0, 0xc0004558c0}, 0xc001755ac0, 0x1, 0x1755ad0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc000455860, 0xc0016962c0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0004554d0}, 0xc0016962c0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0004554d0}, {0x1f35540, 0xc0016962c0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0004554d0}, {0x1f35540, 0xc0016962c0}, 0x237bc38, 0xc0019faf60)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0004554d0}, {0x1f35540, 0xc0016962c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000455380}, {0x1f35540, 0xc0016962c0}, 0xc001790ba0, 0xc001696340)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000455380}, {0x1f35540, 0xc0016962c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000455380}, {0x1f35540, 0xc0016962c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000455380}, {0x1f35540, 0xc0016962c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000455350}, {0x1f35540, 0xc0016962c0}, 0x40, 0xc001696340)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000455350}, {0x1f35540, 0xc0016962c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000455350}, {0x1f35540, 0xc0016962c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000455350}, {0x1f35540, 0xc0016962c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000455350}, {0x1f35540, 0xc0016962c0}, 0xc000e9ac20, 0xc0019faf60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc000455350}, 0xc001790ea0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ae06c0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc000ae06c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1073599 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001ff6e60, 0x1)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001c3cea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef

goroutine 1068205 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0021ecd80, {0x2392228, 0xc002375840})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250214 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001be0000)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647760, {0x1f7c860, 0xc0016476b0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72a00, 0xc0016f35e8, 0xc0016f35b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72a00, {0x1f7c860, 0xc0016476b0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001dc4e40}, {0x20768da, 0x8}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x1c75901, {0xc0016f6d20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x7f6c191ae108, {0xc0016f6d20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x20, 0xc000cb6390, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0xc000da4820, {0xc0015ee100, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x0, 0xc000cb6360, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x1e0cfc0, 0xc000cb62d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x7f6bf287cfff, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001dc4e40}, {0x20768da, 0x203000}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x8, 0xc000cb62a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001dc4e40}, {0x20768da, 0x46}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, 0x0, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc001dc4e40}, {0x20768da, 0x0}, {0x1f33180, 0xc000cb6240}, {0x1f7c860, 0xc0016476b0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc001dc4e40}, 0xc001720720, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000daaf80, {0x2392298, 0xc001dc4e40})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1478 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000daaf80, 0xc001845390)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000daaf80)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1479 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000daaf80, 0xc000e7dbb0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000daaf80)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1480 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000daaf80, 0xc00184dd10)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000daaf80)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1481 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc000daaf80, 0xc0011173a0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc000daaf80)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1482 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00087b860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1509 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0017711d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1245917 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ba5d40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001badc30, {0x1f7c860, 0xc001badb80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036200, 0xc001bd55e8, 0xc001bd55b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036200, {0x1f7c860, 0xc001badb80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0025af380}, {0x20768da, 0x8}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x1c75901, {0xc0013e9060, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x7f6c191ae108, {0xc0013e9060, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x20, 0xc001fd64e0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0xc000da4820, {0xc000f26080, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x0, 0xc001fd64b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x1e0cfc0, 0xc001fd6420, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0025af380}, {0x20768da, 0x203000}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x8, 0xc001fd63f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0025af380}, {0x20768da, 0x46}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, 0x144aacf, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc0025af380}, {0x20768da, 0x0}, {0x1f33180, 0xc001fd6390}, {0x1f7c860, 0xc001badb80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0025af380}, 0xc0007e76e0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000e91100, {0x2392298, 0xc0025af380})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1250133 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ae19e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001646790, {0x1f7c860, 0xc0016466e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72200, 0xc0018ad5e8, 0xc0018ad5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72200, {0x1f7c860, 0xc0016466e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001791ce0}, {0x20768da, 0x8}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x1c75901, {0xc000e9afa0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x7f6c191ae108, {0xc000e9afa0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x20, 0xc002431410, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0xc000da4820, {0xc001ca23c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x0, 0xc0024313e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x1e0cfc0, 0xc002431350, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0xc000e83d98, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001791ce0}, {0x20768da, 0x203000}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0x8, 0xc002431320, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001791ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, 0xc001f95920, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc001791ce0}, {0x20768da, 0x0}, {0x1f33180, 0xc0024312c0}, {0x1f7c860, 0xc0016466e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc001791ce0}, 0xc000e83f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000daaf80, {0x2392298, 0xc001791ce0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1270906 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016c06c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcbb0, {0x1f7b7c0, 0xc001a11a40}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9200, 0xc000ebbbf8, 0xc000ebbbc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9200, {0x1f7b7c0, 0xc001a11a40})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0015b0390}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0x20, {0xc001b58ec0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0015b0390}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015b0390}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0x7f6bf211de60, {0xc001b58ec0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0015b0390}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0xc00218fb70, 0xc0015b03c0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0015b0390}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00137a580}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0x237ac70, 0xc0015b0300, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00137a580}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00137a580}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00137a500}, {0x1f7b7c0, 0xc001a11a40}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc00137a580}, 0xc000e28bc0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00136dc80}, 0xc00137a500, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00136dc80}, 0xc00137a500, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000ebc480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000ebc528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d5de8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00136dc80}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001e0c6c0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000ebc778, {0x235a640, 0xc0000a9fb0}, 0x392d336538342d33)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00136dc80}, 0xc00137a200)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0015ccdb8, 0x4c1cb4}, {0xc057f6a3666acbfc, 0x6240bd45c6}, {0xc0015ccdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00136dc80}, 0xc00137a200)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200}, 0xc001b58dc0, 0xc001f91500)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200}, 0x15, 0xc00137a240)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200}, 0x2076951, 0xc00137a240)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200}, 0x237bc38, 0xc00137a240)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136dc80}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00136db30}, {0x1f33320, 0xc00137a200}, 0x1e5a940, 0xc00137a240)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136db30}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200}, 0x23c27b0, 0xc00137a240)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00136db00}, {0x1f33320, 0xc00137a200}, 0xc001b58dc0, 0xc001f91500)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00136db00}, 0xc0017fc180, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000796fc0, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000796fc0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250155 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008d1320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000730fd0, {0x1f7b560, 0xc000deed20}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9f00, 0xc0018c9d18, 0xc0018c9ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9f00, {0x1f7b560, 0xc000deed20})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00178c6c0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0x20, {0xc000ba02e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00178c6c0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00178c6c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0xc001593ce0, {0xc000ba02e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00178c6c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0xc00218fb70, 0xc00178c6f0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00178c6c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002474740}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0x237ac70, 0xc00178c630, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002474740}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002474740}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0024746c0}, {0x1f7b560, 0xc000deed20}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc002474740}, 0xc000ff8af8, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc000f23ce0}, 0xc0024746c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc000f23ce0}, 0xc0024746c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0018ca5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0018ca648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf211d778, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc000f23ce0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00206e210})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0018ca898, {0x235a640, 0xc0000a9fb0}, 0x612d373239342d39)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc000f23ce0}, 0xc000f23860)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000b8adb8, 0x4c1cb4}, {0xc057f6a2272aaa2c, 0x61177731f7}, {0xc000b8adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc000f23ce0}, 0xc000f23860)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860}, 0xc000ba0120, 0xc001669dd0)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860}, 0x15, 0xc002474400)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860}, 0x207690b, 0xc002474400)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860}, 0x237bc38, 0xc002474400)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23ce0}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc000f23980}, {0x1f33180, 0xc000f23860}, 0x1e5a940, 0xc002474400)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23980}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860}, 0x9, 0xc002474400)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860}, 0xc0007f7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc000f23830}, {0x1f33180, 0xc000f23860}, 0xc000ba0120, 0xc001669dd0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc000f23830}, 0xc00275d8c0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000393320, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc000393320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 828621 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207d98, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c3bd80, 0xc000e1a000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000c3bd80, {0xc000e1a000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000c3bd80, {0xc000e1a000, 0x1060100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc002200748, {0xc000e1a000, 0xc0017c6cf8, 0x44d2f4})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00214ad20, {0xc00038a740, 0x9, 0x62426a8499})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc00214ad20}, {0xc00038a740, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00038a740, 0x9, 0xc000d29b80}, {0x23585c0, 0xc00214ad20})
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00038a700)
	/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001737c80, 0xc00084bf98, 0x1418c25)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000183880, {0x23d1088, 0xc001737c80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185

goroutine 1250156 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0016a9440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000731ef0, {0x1f7b560, 0xc000def0e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3c600, 0xc0026d0440, 0xc0026d0410)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3c600, {0x1f7b560, 0xc000def0e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00178de90}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0x20, {0xc000ba0540, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00178de90}, {0x207eb79, 0x20}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00178de90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0xc00182c9a0, {0xc000ba0540, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00178de90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0xc001fb6920, 0xc00178dec0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00178de90}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002475280}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0x237ac70, 0xc00178de00, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002475280}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002475280}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc002475200}, {0x1f7b560, 0xc000def0e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000097128, {0x2392228, 0xc002475280}, 0xc000594e66, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc00068e150, {0x2392298, 0xc0021ca180}, 0xc002475200, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc00062aa20, {0x2392298, 0xc0021ca180}, 0xc002475200, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00275dda0, {0x2392298, 0xc0021ca180})
	/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0021ca060, {0x2392298, 0xc0021ca180}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0016a87e0, {0x23922d0, 0xc00178cf30}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00178cf30}, 0xc0013e2ca0, 0x1bd58c0, 0x13e2cb0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc00178ced0, 0xc002474980)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc00178cc30}, 0xc002474980)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00178cc30}, {0x1f353a0, 0xc002474980})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc00178cc30}, {0x1f353a0, 0xc002474980}, 0x237bc38, 0xc000898288)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00178cc30}, {0x1f353a0, 0xc002474980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00178cab0, {0x23922d0, 0xc00178cae0}, {0x1f353a0, 0xc002474980}, 0xc000f23860, 0xc0024749c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00178cae0}, {0x1f353a0, 0xc002474980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00178cae0}, {0x1f353a0, 0xc002474980}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00178cae0}, {0x1f353a0, 0xc002474980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00178ca50}, {0x1f353a0, 0xc002474980}, 0x40, 0xc0024749c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00178ca50}, {0x1f353a0, 0xc002474980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00178ca50}, {0x1f353a0, 0xc002474980}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00178ca50}, {0x1f353a0, 0xc002474980})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00178ca50}, {0x1f353a0, 0xc002474980}, 0xc000ba03c0, 0xc000898288)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc00178ca50}, 0xc00275db00, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008d1560, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008d1560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1508 [select, 7 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc001880620)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017711d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017711d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1250101 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0017905a0, {0x2392228, 0xc001e33840})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1250262 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0023aa050)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1510 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0018a4a90, 0xc00062cc90)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0018a4a90)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1511 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0018a4a90, 0xc001379d70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0018a4a90)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1512 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0018a4a90, 0xc0005bbe20)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0018a4a90)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1513 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0018a4a90, 0xc00161cf70)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc0018a4a90)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1514 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0017710e0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1270923 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0010fd560)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcd10, {0x1f736c0, 0xc00136f830}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e9a00, 0xc0018e55c0, 0xc0018e5590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e9a00, {0x1f736c0, 0xc00136f830})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0017fd740}, {0x2076920, 0x8}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x1c75901, {0xc001b593c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x7f6c191ae108, {0xc001b593c0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x20, 0xc0015beae0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0xc000d3fa00, {0xc00137bb00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x0, 0xc0015beab0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x0, 0xc0015bea20, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0017fd740}, {0x2076920, 0x203000}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x203000, 0xc0015be9f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0017fd740}, {0x2076920, 0x46}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc0017fd740}, {0x2076920, 0x200010001}, {0x1f33320, 0xc00137bac0}, {0x1f736c0, 0xc00136f830}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc0017fd740}, 0x2, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0018a4a90, {0x2392298, 0xc0017fd740})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1386530 [IO wait]:
internal/poll.runtime_pollWait(0x7f6bf2207bc8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0022a7900, 0xc001a3d301, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0022a7900, {0xc001a3d301, 0x1, 0x1})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0022a7900, {0xc001a3d301, 0xc000ba5800, 0x0})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000097038, {0xc001a3d301, 0x0, 0x0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc001a3d2f0)
	/usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
	/usr/local/go/src/net/http/server.go:668 +0xcf

goroutine 1242742 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc001ee3aa0, {0x2392228, 0xc001ea6d00})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1245959 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000830240)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001646000, {0x1f7c860, 0xc001badef0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x54)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036500, 0xc00186b5e8, 0xc00186b5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036500, {0x1f7c860, 0xc001badef0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00247ad80}, {0x20768da, 0x8}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x1c75901, {0xc0013e97e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x7f6c191ae108, {0xc0013e97e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x20, 0xc00132ef90, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0xc000da4820, {0xc000f38880, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x0, 0xc00132ef60, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x1e0cfc0, 0xc00132eed0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x60, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00247ad80}, {0x20768da, 0x203000}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x8, 0xc00132eea0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00247ad80}, {0x20768da, 0x46}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, 0x4f9949, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc00247ad80}, {0x20768da, 0x0}, {0x1f33180, 0xc00132ee40}, {0x1f7c860, 0xc001badef0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc00247ad80}, 0x34c4800, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc000daaf80, {0x2392298, 0xc00247ad80})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1242759 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00077f9e0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac000, {0x1f736c0, 0xc001e075f0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x75)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73200, 0xc00186d5c0, 0xc00186d590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73200, {0x1f736c0, 0xc001e075f0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001f1e780}, {0x2076920, 0x8}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x80, {0xc0016f6080, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x7f6c191ae108, {0xc0016f6080, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x20, 0xc000795530, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0xc000d3fa00, {0xc00130d000, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x0, 0xc000794090, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x1e0cfc0, 0xc00050cdb0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0xc06ebb8156, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001f1e780}, {0x2076920, 0x203000}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x203000, 0xc00050ccf0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001f1e780}, {0x2076920, 0x46}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001376e50, {0x2392298, 0xc001f1e780}, {0x2076920, 0x100554c30}, {0x1f33320, 0xc00130cfc0}, {0x1f736c0, 0xc001e075f0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc001f1e780}, 0xc000584840, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0018a4a90, {0x2392298, 0xc001f1e780})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1054788 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001b8afa0)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1539 [select, 2 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0018f6d20)
	/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
	/temporal/service/matching/liveness.go:80 +0x70

goroutine 1540 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0001a15e0, {0x2392228, 0xc0019e2440})
	/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
	/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
	/temporal/internal/goro/goro.go:52 +0x112

goroutine 1541 [select, 7 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0018f1bc0, {0x2392228, 0xc0019e2480})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1542 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0018f1bc0, {0x2392228, 0xc0019e2480})
	/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1270994 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00175b200)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbcf20, {0x1f736c0, 0xc001644240}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a72100, 0xc0019df5c0, 0xc0019df590)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a72100, {0x1f736c0, 0xc001644240})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000ebfda0}, {0x2076920, 0x8}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x1c75901, {0xc001b59e20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x7f6c191ae108, {0xc001b59e20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x20, 0xc0016561e0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0xc000d3fa00, {0xc00161eec0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x0, 0xc0016561b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x0, 0xc001656120, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x40ce34, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000ebfda0}, {0x2076920, 0x203000}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x203000, 0xc0016560f0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000ebfda0}, {0x2076920, 0x46}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, 0x7f6c191ae108, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002994e50, {0x2392298, 0xc000ebfda0}, {0x2076920, 0x2008eb858}, {0x1f33320, 0xc00161ee80}, {0x1f736c0, 0xc001644240}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e340, {0x2392298, 0xc000ebfda0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc0018a4a90, {0x2392298, 0xc000ebfda0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1544 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00166f600, 0xc000578860)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00166f600)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1545 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00166f600, 0xc001e83430)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00166f600)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1546 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00166f600, 0xc00117c780)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00166f600)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1547 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00166f600, 0xc0011f48d0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00166f600)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c

goroutine 1548 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc0017712c0)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c

goroutine 1259513 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0007965a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbd340, {0x1f7c860, 0xc000dbd290}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016fc00, 0xc0019d95e8, 0xc0019d95b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016fc00, {0x1f7c860, 0xc000dbd290})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00240dce0}, {0x20768da, 0x8}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x1c75901, {0xc000e9b2e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x7f6c191ae5b8, {0xc000e9b2e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x20, 0xc001eba8a0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0xc000da4820, {0xc001e8adc0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x0, 0xc001eba870, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x1e0cfc0, 0xc001eba7e0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0xc001724d98, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00240dce0}, {0x20768da, 0x203000}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x8, 0xc001eba7b0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00240dce0}, {0x20768da, 0x46}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, 0x7, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae5b8, {0x2392298, 0xc00240dce0}, {0x20768da, 0x0}, {0x1f33180, 0xc001eba750}, {0x1f7c860, 0xc000dbd290}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc00240dce0}, 0x0, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00166f600, {0x2392298, 0xc00240dce0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1270842 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c1ad80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ee82c0, {0x1f7b7c0, 0xc0015c3b80}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001f3d000, 0xc000ac9bf8, 0xc000ac9bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001f3d000, {0x1f7b7c0, 0xc0015c3b80})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00139c750}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0x20, {0xc000ba16a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00139c750}, {0x207ebc6, 0x20}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00139c750}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0xc000ec9a20, {0xc000ba16a0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00139c750}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0xc00218fb70, 0xc00139c780, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00139c750}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0013a0080}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0x237ac70, 0xc00139c6c0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0013a0080}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0013a0080}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc0013a0000}, {0x1f7b7c0, 0xc0015c3b80}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc0013a0080}, 0xc000f00240, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00138be00}, 0xc0013a0000, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00138be00}, 0xc0013a0000, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0x4e800, 0x64})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000aca528, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf21d0e50, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00138be00}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001ec8920})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000aca778, {0x235a640, 0xc0000a9fb0}, 0x622d626138342d30)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00138be00}, 0xc0012f9d00)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001cccdb8, 0x4c1cb4}, {0xc057f6a36605c552, 0x6240583f2a}, {0xc001cccdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00138be00}, 0xc0012f9d00)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00}, 0xc000ba15a0, 0xc00257bba8)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00}, 0x15, 0xc0012f9d40)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00}, 0x2076951, 0xc0012f9d40)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00}, 0x237bc38, 0xc0012f9d40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138be00}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00138bcb0}, {0x1f33320, 0xc0012f9d00}, 0x1e5a940, 0xc0012f9d40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138bcb0}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00}, 0x23c27b0, 0xc0012f9d40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0017a3b60, {0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00138bc80}, {0x1f33320, 0xc0012f9d00}, 0xc000ba15a0, 0xc00257bba8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00138bc80}, 0xc001c12240, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00089ca20, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc00089ca20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1073654 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00258ef00)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837

goroutine 1245898 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001c3fd40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1ce0, {0x1f7b560, 0xc001769ea0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8200, 0xc000fe5d18, 0xc000fe5ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8200, {0x1f7b560, 0xc001769ea0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00209a0f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0x20, {0xc000fe99e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00209a0f0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00209a0f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0xc00176eb00, {0xc000fe99e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00209a0f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0xc00218fb70, 0xc00209a120, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00209a0f0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000e5b8c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0x237ac70, 0xc00209a060, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000e5b8c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000e5b8c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000e5b840}, {0x1f7b560, 0xc001769ea0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000e5b8c0}, 0xc0023086c0, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001edb980}, 0xc000e5b840, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001edb980}, 0xc000e5b840, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000fe65a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000fe6648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2127478, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001edb980}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c0016183e0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc000fe6898, {0x235a640, 0xc0000a9fb0}, 0x382d313031342d65)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001edb980}, 0xc001edb7d0)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc00180adb8, 0x4c1cb4}, {0xc057f6a1e92cbdbc, 0x60ddde7b86}, {0xc00180adb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001edb980}, 0xc001edb7d0)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0}, 0xc000fe9920, 0xc001cfed20)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0}, 0x15, 0xc000e5b600)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0}, 0x207690b, 0xc000e5b600)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0}, 0x237bc38, 0xc000e5b600)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb980}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001edb830}, {0x1f33180, 0xc001edb7d0}, 0x1e5a940, 0xc000e5b600)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb830}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0}, 0x7f6bf217ff68, 0xc000e5b600)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0}, 0xc002239768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002239b60, {0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001edb7a0}, {0x1f33180, 0xc001edb7d0}, 0xc000fe9920, 0xc001cfed20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001edb7a0}, 0xc0021adc80, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001c3ec60, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001c3ec60, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1269318 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
	/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105

goroutine 1245909 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ba5440)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bada20, {0x1f7c860, 0xc001bad970}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x60)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000036000, 0xc00181d5e8, 0xc00181d5b8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000036000, {0x1f7c860, 0xc001bad970})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0025aeea0}, {0x20768da, 0x8}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x1c75901, {0xc0013e8ee0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0xbfbf4f, 0x20a9218, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x7f6c191ae108, {0xc0013e8ee0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x20, 0xc0020574d0, ...)
	/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0xc000da4820, {0xc000f13800, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x0, 0xc0020574a0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x1e0cfc0, 0xc002057320, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x0, {0xc000574620, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0025aeea0}, {0x20768da, 0x203000}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0x8, 0xc0020571d0, ...)
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0025aeea0}, {0x20768da, 0x46}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, 0xc000e80e50, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7f6c191ae108, {0x2392298, 0xc0025aeea0}, {0x20768da, 0x0}, {0x1f33180, 0xc002057170}, {0x1f7c860, 0xc001bad970}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e340, {0x2392298, 0xc0025aeea0}, 0xc000e80f20, {0x0, 0x0, 0x0})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00166f600, {0x2392298, 0xc0025aeea0})
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
	/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef

goroutine 1250159 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00275dce0, {0x2392228, 0xc002475140})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1242819 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000ec78c0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bac420, {0x1f7b560, 0xc000cd4820}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73f00, 0xc001611d18, 0xc001611ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73f00, {0x1f7b560, 0xc000cd4820})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f3aab0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0x20, {0xc0016f69e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f3aab0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f3aab0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0xc001c8e580, {0xc0016f69e0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f3aab0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0xc00218fb70, 0xc001f3aae0, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f3aab0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001fab040}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0x237ac70, 0xc001f3a8a0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001fab040}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001fab040}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001faafc0}, {0x1f7b560, 0xc000cd4820}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001fab040}, 0xc002619200, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001f3a150}, 0xc001faafc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001f3a150}, 0xc001faafc0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0016125a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001612648, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20db5d8, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001f3a150}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c002a0dbd0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001612898, {0x235a640, 0xc0000a9fb0}, 0x392d613230342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001f3a150}, 0xc00078bf50)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0019dadb8, 0x4c1cb4}, {0xc057f6a1aa32c576, 0x60a349b940}, {0xc0019dadb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001f3a150}, 0xc00078bf50)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50}, 0xc0016f68c0, 0xc0020b2420)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50}, 0x15, 0xc001faac40)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50}, 0x207690b, 0xc001faac40)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50}, 0x237bc38, 0xc001faac40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f3a150}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00078bfb0}, {0x1f33180, 0xc00078bf50}, 0x1e5a940, 0xc001faac40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00078bfb0}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50}, 0x9, 0xc001faac40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50}, 0xc0007f7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0007f7b60, {0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50}, 0x40, 0xc00005ec00)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00078bf20}, {0x1f33180, 0xc00078bf50}, 0xc0016f68c0, 0xc0020b2420)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00078bf20}, 0xc001f1fce0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008e9c20, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0008e9c20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245790 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0016d3a40, {0x2392298, 0xc002167e00}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0016c0fc0, {0x23922d0, 0xc001c1d920}, 0x1e)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001a6f1a0, {0x23922d0, 0xc001c1d920}, 0xc001fef570, 0x1, 0x1fef580)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001c1d8c0, 0xc000e3d100)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001c1d620}, 0xc000e3d100)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001c1d620}, {0x1f35540, 0xc000e3d100})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001c1d620}, {0x1f35540, 0xc000e3d100}, 0x237bc38, 0xc001739f80)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1d620}, {0x1f35540, 0xc000e3d100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001c1d3e0}, {0x1f35540, 0xc000e3d100}, 0xc000e3cb00, 0xc000e3d180)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1d3e0}, {0x1f35540, 0xc000e3d100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c1d3e0}, {0x1f35540, 0xc000e3d100}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1d3e0}, {0x1f35540, 0xc000e3d100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001c1d3b0}, {0x1f35540, 0xc000e3d100}, 0x40, 0xc000e3d180)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1d3b0}, {0x1f35540, 0xc000e3d100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001c1d3b0}, {0x1f35540, 0xc000e3d100}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c1d3b0}, {0x1f35540, 0xc000e3d100})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001c1d3b0}, {0x1f35540, 0xc000e3d100}, 0xc000fe8fc0, 0xc001739f80)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001c1d3b0}, 0xc002167da0, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00175aa20, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc00175aa20, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250097 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001790840, {0x2392298, 0xc001790900}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001c519e0, {0x23922d0, 0xc000454600}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc001f95620, {0x23922d0, 0xc000454600}, 0xc0017552b0, 0x1, 0x17552c0)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc0004543c0, 0xc001e32cc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc0004540f0}, 0xc001e32cc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0004540f0}, {0x1f35540, 0xc001e32cc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc0004540f0}, {0x1f35540, 0xc001e32cc0}, 0x237bc38, 0xc0019faab0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0004540f0}, {0x1f35540, 0xc001e32cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x1ba3c00, {0x23922d0, 0xc0017e9f50}, {0x1f35540, 0xc001e32cc0}, 0x197, 0xc001e32d40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017e9f50}, {0x1f35540, 0xc001e32cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0017e9f50}, {0x1f35540, 0xc001e32cc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017e9f50}, {0x1f35540, 0xc001e32cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0017e9f20}, {0x1f35540, 0xc001e32cc0}, 0x40, 0xc001e32d40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017e9f20}, {0x1f35540, 0xc001e32cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0017e9f20}, {0x1f35540, 0xc001e32cc0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0017e9f20}, {0x1f35540, 0xc001e32cc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0017e9f20}, {0x1f35540, 0xc001e32cc0}, 0xc000e9aa40, 0xc0019faab0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0017e9f20}, 0xc001790360, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001c506c0, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000da9080}, 0xc001c506c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245815 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002016c00, {0x2392298, 0xc0021ac6c0}, 0x0)
	/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0005f37a0, {0x23922d0, 0xc001635c80}, 0x1bd3d80)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001635c80}, 0xc00203c330, 0x1bd58c0, 0x203c340)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00038aee0, 0xc001635b30, 0xc000e480c0)
	/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000590540, {0x23922d0, 0xc001635890}, 0xc000e480c0)
	/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001635890}, {0x1f353a0, 0xc000e480c0})
	/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001635890}, {0x1f353a0, 0xc000e480c0}, 0x237bc38, 0xc002244b40)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001635890}, {0x1f353a0, 0xc000e480c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001635710, {0x23922d0, 0xc001635740}, {0x1f353a0, 0xc000e480c0}, 0xc001634a50, 0xc000e48100)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001635740}, {0x1f353a0, 0xc000e480c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001635740}, {0x1f353a0, 0xc000e480c0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001635740}, {0x1f353a0, 0xc000e480c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0016356b0}, {0x1f353a0, 0xc000e480c0}, 0x40, 0xc000e48100)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016356b0}, {0x1f353a0, 0xc000e480c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0016356b0}, {0x1f353a0, 0xc000e480c0}, 0x1e78c20, 0x1)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0016356b0}, {0x1f353a0, 0xc000e480c0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0016356b0}, {0x1f353a0, 0xc000e480c0}, 0xc000fe9580, 0xc002244b40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc0016356b0}, 0xc0021ac660, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008119e0, 0xc00070a240, 0x34776d8, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc0008119e0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1245987 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000444ea0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001dc1ef0, {0x1f7b560, 0xc001a101e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb9)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8400, 0xc001c87d18, 0xc001c87ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8400, {0x1f7b560, 0xc001a101e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001383a10}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0x20, {0xc000fe9f00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001383a10}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001383a10}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0xc0017ec2c0, {0xc000fe9f00, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001383a10}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0xc00218fb70, 0xc001383a40, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001383a10}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000fa7780}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0x237ac70, 0xc001383980, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000fa7780}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000fa7780}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000fa7700}, {0x1f7b560, 0xc001a101e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc000fa7780}, 0xc002308f90, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc0013832c0}, 0xc000fa7700, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc0013832c0}, 0xc000fa7700, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001c885a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c88648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2134378, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc0013832c0}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001619120})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001c88898, {0x235a640, 0xc0000a9fb0}, 0x382d396438342d36)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc0013832c0}, 0xc001383110)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc0015aadb8, 0x4c1cb4}, {0xc057f6a1ea8d3c01, 0x60df3ef9d9}, {0xc0015aadb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc0013832c0}, 0xc001383110)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110}, 0xc000fe9e40, 0xc0017d0690)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110}, 0x15, 0xc000fa74c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110}, 0x207690b, 0xc000fa74c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110}, 0x237bc38, 0xc000fa74c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013832c0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001383170}, {0x1f33180, 0xc001383110}, 0x1e5a940, 0xc000fa74c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001383170}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110}, 0x9, 0xc000fa74c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110}, 0xc00296e768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00296eb60, {0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0013830e0}, {0x1f33180, 0xc001383110}, 0xc000fe9e40, 0xc0017d0690)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc0013830e0}, 0xc000ba5020, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001a53560, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001a53560, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1270863 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008d6a20)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bbc790, {0x1f7b7c0, 0xc001a10aa0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019e8900, 0xc0016c5bf8, 0xc0016c5bc8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019e8900, {0x1f7b7c0, 0xc001a10aa0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00133c360}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0x20, {0xc001b58200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00133c360}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00133c360}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0xc0017ed080, {0xc001b58200, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00133c360}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0xc00218fb70, 0xc00133c390, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00133c360}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001327400}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0x237ac70, 0xc00133c2d0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001327400}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001327400}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001327380}, {0x1f7b7c0, 0xc001a10aa0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000096e80, {0x2392228, 0xc001327400}, 0xc000e28440, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000b15c80, {0x23922d0, 0xc00132bc50}, 0xc001327380, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000575300, {0x23922d0, 0xc00132bc50}, 0xc001327380, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0016c6480})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016c6528, {0x235a640, 0xc0006dc210}, 0x0)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf211d118, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00132bc50}, 0x101000001888a8f, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001fbf880})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0016c6778, {0x235a640, 0xc0000a9fb0}, 0x622d366631342d34)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0007cd4e0, {0x235a640, 0xc0000a9fb0}, 0x20)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001702a0, {0x23922d0, 0xc00132bc50}, 0xc001327080)
	/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc001cbcdb8, 0x4c1cb4}, {0xc057f6a366365939, 0x624088d308}, {0xc001cbcdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc000768360, {0x23922d0, 0xc00132bc50}, 0xc001327080)
	/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080}, 0xc001b58120, 0xc001eed038)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080}, 0x15, 0xc0013270c0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080}, 0x2076951, 0xc0013270c0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080}, 0x237bc38, 0xc0013270c0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bc50}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00132bb00}, {0x1f33320, 0xc001327080}, 0x1e5a940, 0xc0013270c0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bb00}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080}, 0x23c27b0, 0xc0013270c0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080}, 0x0, 0x49d300)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c002994b60, {0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080}, 0x40, 0xc00005f800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00132bad0}, {0x1f33320, 0xc001327080}, 0xc001b58120, 0xc001eed038)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc00132bad0}, 0xc0021bbce0, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001789320, 0xc000cf2e10, 0x347a528, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc001789320, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1259515 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000797d40)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000dbd3f0, {0x1f7b560, 0xc00068b0e0}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xae)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016fe00, 0xc0018c5d18, 0xc0018c5ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016fe00, {0x1f7b560, 0xc00068b0e0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ebb560}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0x20, {0xc000e9b660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ebb560}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ebb560}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0xc0015c7a20, {0xc000e9b660, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ebb560}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0xc00218fb70, 0xc001ebb590, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ebb560}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e8b280}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0x237ac70, 0xc001ebb4d0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e8b280}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e8b280}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e8b200}, {0x1f7b560, 0xc00068b0e0}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc001e8b280}, 0xc001f5e540, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc001ebae10}, 0xc001e8b200, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc001ebae10}, 0xc001e8b200, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0018c65a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0018c6648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf2123830, {0x235a640, 0xc0006dc210}, 0x7f6c191ae5b8)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc001ebae10}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c001def100})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc0018c6898, {0x235a640, 0xc0000a9fb0}, 0x392d346335342d63)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000a9fb0}, 0x15)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001702a0, {0x23922d0, 0xc001ebae10}, 0xc001ebac60)
	/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc000556e40, 0x6})
	/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc000768360, {0xc000c0cdb8, 0x4c1cb4}, {0xc057f6a2a7ed1e21, 0x618f6f39f7}, {0xc000c0cdb8, 0x93ebb2}, 0x34c4660)
	/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc000768360, {0x23922d0, 0xc001ebae10}, 0xc001ebac60)
	/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc0006a4b90, {0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60}, 0xc000e9b520, 0xc001e92b10)
	/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000ae75c0, {0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60}, 0x15, 0xc001e8afc0)
	/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000ae7410, {0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60}, 0x207690b, 0xc001e8afc0)
	/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006c9080, {0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60}, 0x237bc38, 0xc001e8afc0)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebae10}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ebacc0}, {0x1f33180, 0xc001ebac60}, 0x1e5a940, 0xc001e8afc0)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebacc0}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60}, 0x9, 0xc001e8afc0)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60}, 0xc0017c7768, 0x988353)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0017c7b60, {0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60}, 0x40, 0xc00005e800)
	/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ebac30}, {0x1f33180, 0xc001ebac60}, 0xc000e9b520, 0xc001e92b10)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc000768360}, {0x23922d0, 0xc001ebac30}, 0xc00240df20, 0xc0005754c0)
	/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0007966c0, 0xc000cf2e10, 0x347a570, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000183880, {0x23d1088, 0xc001737c80}, 0xc0007966c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1054838 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc0024a1f20, {0x2392228, 0xc001cfbdc0})
	/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
	/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
	/temporal/internal/goro/group.go:55 +0xbe

goroutine 1242727 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000e5c900)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001bad290, {0x1f7b7c0, 0xc000cd4a00}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00016ef00, 0xc000d664d0, 0xc000d664a0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00016ef00, {0x1f7b7c0, 0xc000cd4a00})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002348750}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0x20, {0xc0013e8600, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002348750}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002348750}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0xc0007151e0, {0xc0013e8600, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002348750}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0xc001fb6920, 0xc002348780, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002348750}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0x8, {0xc000ed4b60, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000d2b880}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0x237ac70, 0xc0023486c0, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000d2b880}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000d2b880}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000d2b7c0}, {0x1f7b7c0, 0xc000cd4a00}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000097128, {0x2392228, 0xc000d2b880}, 0xc0011e1e66, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc00068e150, {0x2392298, 0xc001f1e0c0}, 0xc000d2b7c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc00062aa20, {0x2392298, 0xc001f1e0c0}, 0xc000d2b7c0, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00221db00, {0x2392298, 0xc001f1e0c0})
	/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00221dd40, {0x2392298, 0xc001f1e0c0}, 0x0)
	/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
	/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00083d680, {0x23922d0, 0xc001e7ee40}, 0x23)
	/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0026183c0, {0x23922d0, 0xc001e7ee40}, 0xc001e61700, 0x1, 0x1e61710)
	/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00038aee0, 0xc001e7ede0, 0xc001e5bcc0)
	/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000590540, {0x23922d0, 0xc001e7eb40}, 0xc001e5bcc0)
	/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001e7eb40}, {0x1f35540, 0xc001e5bcc0})
	/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00086dd28, {0x23922d0, 0xc001e7eb40}, {0x1f35540, 0xc001e5bcc0}, 0x237bc38, 0xc001d46480)
	/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e7eb40}, {0x1f35540, 0xc001e5bcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001e7e9f0}, {0x1f35540, 0xc001e5bcc0}, 0xc001e5b700, 0xc001e5bd40)
	/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e7e9f0}, {0x1f35540, 0xc001e5bcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e7e9f0}, {0x1f35540, 0xc001e5bcc0}, 0x1cc5b40, 0x34c4b38)
	/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e7e9f0}, {0x1f35540, 0xc001e5bcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e7e9c0}, {0x1f35540, 0xc001e5bcc0}, 0x40, 0xc001e5bd40)
	/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e7e9c0}, {0x1f35540, 0xc001e5bcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e7e9c0}, {0x1f35540, 0xc001e5bcc0}, 0x203000, 0x203000)
	/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e7e9c0}, {0x1f35540, 0xc001e5bcc0})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e7e9c0}, {0x1f35540, 0xc001e5bcc0}, 0xc000ba1300, 0xc001d46480)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000590540}, {0x23922d0, 0xc001e7e9c0}, 0xc001f1e060, 0xc00062aae0)
	/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c8cd80, 0xc00070a240, 0x34776c0, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0003fbc00, {0x23d1088, 0xc000817e00}, 0xc001c8cd80, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294

goroutine 1250215 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001be1320)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001647810, {0x1f7b560, 0xc001a11d60}, 0x0)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaa)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001a73100, 0xc001a39d18, 0xc001a39ce8)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001a73100, {0x1f7b560, 0xc001a11d60})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc000cb7bf0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0x20, {0xc0016f6f20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc000cb7bf0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0x10, 0x20a9218, ...)
	/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000cb7bf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0xc000714580, {0xc0016f6f20, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc000cb7bf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0xc00218fb70, 0xc000cb7c20, ...)
	/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc000cb7bf0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0x8, {0xc000756a40, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0015eeac0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0x237ac70, 0xc000cb7b60, ...)
	/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0015eeac0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, 0x3493960, 0x20a9218, ...)
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0015eeac0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0015eea40}, {0x1f7b560, 0xc001a11d60}, {0x0, ...})
	/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000096e80, {0x2392228, 0xc0015eeac0}, 0xc001ba8880, {0x0, 0x0, 0x0})
	/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000b15c80, {0x23922d0, 0xc000cb6f30}, 0xc0015eea40, {0x0, 0x0, 0x0})
	/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000575300, {0x23922d0, 0xc000cb6f30}, 0xc0015eea40, {0x0, 0x0, 0x0})
	/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
	/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001a3a5a0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001a3a648, {0x235a640, 0xc0006dc210}, 0x403751)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f6bf20d4990, {0x235a640, 0xc0006dc210}, 0x7f6c191ae108)
	/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001383c20, {0x23922d0, 0xc000cb6f30}, 0x101000000000000, {0x0, 0x0, 0x0})
	/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
	/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010a870, 0x10100c00218fea0})
	/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc000040040}, 0xc001a3a898, {0x235a640, 0xc0000a9fb0}, 0x392d313936342d62)
	/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backo