Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
This file has been truncated, but you can view the full file.
goroutine 1358679 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc000828a80})
/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc000828a80}, 0x0)
/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc000828a80}, 0xc)
/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc001c38161, 0x34934e0}, {0x2386300, 0xc000828a80}, 0xc001c38154)
/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc000828a80}, 0xc000734900)
/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc000828a80}, 0x0)
/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc001c3816b, {0x2386300, 0xc000828a80}, 0xc000734900)
/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc00247f1a0}, {0x2386300, 0xc000828a80}, 0xc000734900)
/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc00156e000, {0x23922d0, 0xc0006eb5c0})
/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3033 +0x4e8
goroutine 1 [chan receive, 13 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc00073c000)
/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc00009b340)
/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc0001270e0, 0xc00009b300)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc00053b1e0, {0x2392260, 0xc0000b0000}, {0xc0000ae000, 0x8, 0x8})
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
/temporal/cmd/server/main.go:50 +0x45
goroutine 27 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00003a380)
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92
goroutine 3 [syscall, 13 minutes]:
os/signal.signal_recv()
/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x2c
goroutine 4 [chan receive, 13 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 5 [chan receive, 13 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 65 [select]:
database/sql.(*DB).connectionCleaner(0xc000533380, 0x0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 122 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc0005796c0, {0x2392228, 0xc000c696c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 68 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef1830, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003bc00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003bc00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003bc00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000d890)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000d890)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0000859e0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790000)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 76 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00056b4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 58 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 51 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc000533380, {0x2392228, 0xc000753d00})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 1266200 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 49 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000088a50)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 16 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef1a00, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003ab00, 0x416f26, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003ab00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003ab00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000c1c8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000c1c8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc0003280e0, {0x23860f0, 0xc00000c1c8})
/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc0003280e0)
/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
/temporal/common/pprof/pprof.go:72 +0x9c
goroutine 1266153 [select]:
go.temporal.io/server/service/history.(*timerQueueProcessorBase).internalProcessor(0xc00127b080)
/temporal/service/history/timerQueueProcessorBase.go:271 +0x37f
go.temporal.io/server/service/history.(*timerQueueProcessorBase).processorPump(0xc00127b080)
/temporal/service/history/timerQueueProcessorBase.go:196 +0x9e
created by go.temporal.io/server/service/history.(*timerQueueProcessorBase).Start
/temporal/service/history/timerQueueProcessorBase.go:163 +0xaf
goroutine 584 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060ec60)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 1282761 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001ba4ae0, {0x2392298, 0xc0008a9f80}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000640a20, {0x23922d0, 0xc001e99ad0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001e99ad0}, 0xc001e96cb0, 0x1bd58c0, 0x1e96cc0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001e99a70, 0xc001e91580)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001e997d0}, 0xc001e91580)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001e997d0}, {0x1f353a0, 0xc001e91580})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001e997d0}, {0x1f353a0, 0xc001e91580}, 0x237bc38, 0xc001e89818)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e997d0}, {0x1f353a0, 0xc001e91580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001e99650, {0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580}, 0xc001e98990, 0xc001e915c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e99680}, {0x1f353a0, 0xc001e91580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580}, 0x40, 0xc001e915c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001e995f0}, {0x1f353a0, 0xc001e91580}, 0xc0019a5620, 0xc001e89818)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001e995f0}, 0xc0008a9f20, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000681c20, 0xc000eeee10, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000681c20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 69 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc0006c0d00, {0x2392228, 0xc0007966c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 75 [IO wait, 12 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef1748, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00079a080, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00079a080)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00079a080)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000dbd8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000dbd8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0007501c0, {0x23860f0, 0xc00000dbd8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc0007137d0)
/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc00056b360}, 0xc00072001c)
/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
/temporal/service/history/fx.go:168 +0x9a
goroutine 1266165 [select, 1 minutes]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001be0bb0)
/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
/temporal/service/history/visibilityQueueProcessor.go:203 +0x85
goroutine 78 [select]:
database/sql.(*DB).connectionOpener(0xc000aa20d0, {0x2392228, 0xc000a948c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 59 [sleep]:
time.Sleep(0x306dc4200)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 85 [select]:
database/sql.(*DB).connectionCleaner(0xc000aa20d0, 0xc00056b4a0)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 1286354 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0023bd440)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205f1e0, {0x1f7b560, 0xc000630dc0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa600, 0xc00160e440, 0xc00160e410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa600, {0x1f7b560, 0xc000630dc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00212f920}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x20, {0xc001fa1520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00212f920}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00212f920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0xc000a2f8c0, {0xc001fa1520, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00212f920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0xc0021b2800, 0xc00212f950, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00212f920}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x8, {0xc00062b440, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001e90100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x33, 0xc00212f890, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001e90100}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001e90100}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001e90040}, {0x1f7b560, 0xc000630dc0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001e90100}, 0xc0011d09c6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc00286bec0}, 0xc001e90040, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc00286bec0}, 0xc001e90040, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc00286bbc0, {0x2392298, 0xc00286bec0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00286be60, {0x2392298, 0xc00286bec0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0023bcea0, {0x23922d0, 0xc00212ec90}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00212ec90}, 0xc001e1e740, 0x1bd58c0, 0x1e1e750)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc00212ec30, 0xc001d73480)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc00212e990}, 0xc001d73480)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00212e990}, {0x1f353a0, 0xc001d73480})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc00212e990}, {0x1f353a0, 0xc001d73480}, 0x237bc38, 0xc0020c1260)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e990}, {0x1f353a0, 0xc001d73480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00212e720, {0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480}, 0xc0021152c0, 0xc001d734c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e750}, {0x1f353a0, 0xc001d73480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480}, 0x40, 0xc001d734c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00212e6c0}, {0x1f353a0, 0xc001d73480}, 0xc001fa1220, 0xc0020c1260)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc00212e6c0}, 0xc00286b920, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0008cfe60, 0xc000eeee10, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0008cfe60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1174605 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002572360, {0x2392228, 0xc002576640})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 88 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8
goroutine 89 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 90 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 120 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 1300594 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002481e60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5760, {0x1f7b560, 0xc0000203c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xaf)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000734600, 0xc002789d18, 0xc002789ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000734600, {0x1f7b560, 0xc0000203c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x20, {0xc0021d7760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x7fb42acf4f50, {0xc0021d7760, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0xc0017985b0, 0xc001f1af00, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1aed0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001f14b40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x237ac70, 0xc001f1ae40, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001f14b40}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001f14b40}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001f14ac0}, {0x1f7b560, 0xc0000203c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001f14b40}, 0xc001d19f20, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001f1a780}, 0xc001f14ac0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001f1a780}, 0xc001f14ac0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00278a5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00278a648, {0x235a640, 0xc0000a8b10}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad28cc0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001f1a780}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00251b5d0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc00278a898, {0x235a640, 0xc000045ad0}, 0x392d353831342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001f1a780}, 0xc001f1a5d0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001a2adb8, 0x4c1cb4}, {0xc057031378e9340c, 0xb4d300a989}, {0xc001a2adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001f1a780}, 0xc001f1a5d0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0xc0021d75e0, 0xc0026e73c8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0x15, 0xc001f14880)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0x207690b, 0xc001f14880)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0}, 0x237bc38, 0xc001f14880)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a780}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001f1a630}, {0x1f33180, 0xc001f1a5d0}, 0x1e5a940, 0xc001f14880)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a630}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0x9, 0xc001f14880)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0xc001c8f768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001c8fb60, {0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0x40, 0xc0007c4800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001f1a5a0}, {0x1f33180, 0xc001f1a5d0}, 0xc0021d75e0, 0xc0026e73c8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001f1a5a0}, 0xc0017dce40, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002480900, 0xc0009e06f0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc002480900, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1282766 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00242f320, {0x2392298, 0xc001ffe6c0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002465200, {0x23922d0, 0xc001ea3920}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001ea3920}, 0xc001e97880, 0x1bd58c0, 0x1e97890)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001ea38c0, 0xc001ea6140)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001ea3620}, 0xc001ea6140)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ea3620}, {0x1f353a0, 0xc001ea6140})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001ea3620}, {0x1f353a0, 0xc001ea6140}, 0x237bc38, 0xc001ea4228)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3620}, {0x1f353a0, 0xc001ea6140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001ea34a0, {0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140}, 0xc001ea27e0, 0xc001ea6180)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea34d0}, {0x1f353a0, 0xc001ea6140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140}, 0x40, 0xc001ea6180)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ea3440}, {0x1f353a0, 0xc001ea6140}, 0xc0019a59e0, 0xc001ea4228)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001ea3440}, 0xc001ffe660, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00196b9e0, 0xc000eeee10, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc00196b9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1286004 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001588f00, {0x2392228, 0xc002232840})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1319418 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 667 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000fbc3a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 128 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 116 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef12c0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00076ce00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00076ce00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00076ce00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000a0ab70)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000a0ab70)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc00002d340, {0x23860f0, 0xc000a0ab70})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc0007137d0)
/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc000a36720}, 0x0)
/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
/temporal/service/matching/fx.go:129 +0x9a
goroutine 1318871 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001961560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001838c60, {0x1f736c0, 0xc00105c2d0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6c00, 0xc00151b5c0, 0xc00151b590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6c00, {0x1f736c0, 0xc00105c2d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002036cc0}, {0x2076920, 0x8}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x1c75901, {0xc00247d660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x7fb451d0c5b8, {0xc00247d660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x20, 0xc001b75230, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0xc0006ca340, {0xc001052580, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x0, 0xc001b75200, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x0, 0xc001b750e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x40ce34, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002036cc0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x203000, 0xc001b74ff0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002036cc0}, {0x2076920, 0x46}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, 0x7fb451d0c5b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc00186be50, {0x2392298, 0xc002036cc0}, {0x2076920, 0x22ac7a3d0}, {0x1f33320, 0xc001052540}, {0x1f736c0, 0xc00105c2d0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002036cc0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc002036cc0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 174 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00034edc0, 0x1fef77d)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1266199 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0f20, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1266214 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc00177b2c0)
/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
/temporal/service/history/queueProcessor.go:158 +0x23b
goroutine 127 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 1318718 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00032d320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018388f0, {0x1f736c0, 0xc000fbe120}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968500, 0xc0008915c0, 0xc000891590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968500, {0x1f736c0, 0xc000fbe120})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0015bb200}, {0x2076920, 0x8}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x1c75901, {0xc00247cbc0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x7fb451d0c5b8, {0xc00247cbc0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x20, 0xc0022037d0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0xc0006ca340, {0xc000fb3380, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x0, 0xc0022037a0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x0, 0xc002203710, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x40ce34, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0015bb200}, {0x2076920, 0x203000}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x203000, 0xc0022036e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0015bb200}, {0x2076920, 0x46}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, 0x7fb451d0c5b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc00186be50, {0x2392298, 0xc0015bb200}, {0x2076920, 0x200a74410}, {0x1f33320, 0xc000fb3340}, {0x1f736c0, 0xc000fbe120}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0015bb200}, 0xc0006ef8c0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc0015bb200})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1282661 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000645e60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb46e0, {0x1f736c0, 0xc001d62900}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968600, 0xc00152d5c0, 0xc00152d590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968600, {0x1f736c0, 0xc001d62900})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002282720}, {0x2076920, 0x8}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x1c75901, {0xc0008e7f40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x7fb451d0c108, {0xc0008e7f40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x20, 0xc001d66d50, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0xc0006ca340, {0xc001d5ecc0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x0, 0xc001d66d20, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x1e0cfc0, 0xc001d66c90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0xc051a60d0c, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002282720}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x203000, 0xc001d66c60, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002282720}, {0x2076920, 0x46}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, 0x7fb451d0c108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000fa0e50, {0x2392298, 0xc002282720}, {0x2076920, 0x1010013d8}, {0x1f33320, 0xc001d5ec80}, {0x1f736c0, 0xc001d62900}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002282720}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc002282720})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1270840 [select, 1 minutes]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00211f200)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e8f0, {0x1f736c0, 0xc0024c42d0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969000, 0xc0015195c0, 0xc001519590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969000, {0x1f736c0, 0xc0024c42d0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x8}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x1c75901, {0xc0019a45a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x7fb451d0c108, {0xc0019a45a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x20, 0xc002136090, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0xc0006ca340, {0xc002446780, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x0, 0xc002136060, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x1e0cfc0, 0xc002117fb0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x4b400cb94e8, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x203000, 0xc002117f80, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc002c5a0c0}, {0x2076920, 0x46}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, 0x7fb451d0c108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000cb9650, {0x2392298, 0xc002c5a0c0}, {0x2076920, 0x100cb96d0}, {0x1f33320, 0xc002446740}, {0x1f736c0, 0xc0024c42d0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc002c5a0c0}, 0x1fe4e76, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc002c5a0c0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 835 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0978, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000aba680, 0xc0010f6000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000aba680, {0xc0010f6000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000aba680, {0xc0010f6000, 0x60100000000, 0x8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa528, {0xc0010f6000, 0x4383e0, 0xc001264ea0})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0008a6ba0, {0xc000828660, 0x9, 0xc000a46d18})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc0008a6ba0}, {0xc000828660, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000828660, 0x9, 0x18}, {0x23585c0, 0xc0008a6ba0})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000828620)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000ad20)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 1266069 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc00185fd58)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 358 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e790)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 357 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e6e0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 173 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef0b80, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000af780, 0xc0007dcef0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000af780, {0xc0007dcef0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000af780, {0xc0007dcef0, 0xc000015f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa068, {0xc0007dcef0, 0x46b213, 0x617b94f5})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa068}, {0xc0007dcef0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00034edc0, 0x7ff9f0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 359 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e840)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 117 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000a9b1d0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 130 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42aef1490, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00076cb00, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00076cb00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00076cb00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000a0a870)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000a0a870)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc00053e880)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000352600)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 131 [sleep]:
time.Sleep(0x306dc4200)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 191 [select]:
database/sql.(*DB).connectionCleaner(0xc0007c0750, 0x1c)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 1272206 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00211c360)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e210, {0x1f7b560, 0xc00204abe0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000734c00, 0xc002585d18, 0xc002585ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000734c00, {0x1f7b560, 0xc00204abe0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x20, {0xc001fa04e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x7fb42ac74c70, {0xc001fa04e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0xc0023ec2f0, 0xc001ef3f20, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ef3ef0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000c35b00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x237ac70, 0xc001ef3e60, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000c35b00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000c35b00}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc000c35a80}, {0x1f7b560, 0xc00204abe0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc000c35b00}, 0xc0027bcb70, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001ef3770}, 0xc000c35a80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001ef3770}, 0xc000c35a80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc0025865a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002586648, {0x235a640, 0xc0000a8b10}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad664f0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001a31140, {0x23922d0, 0xc001ef3770}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0010bc9e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002586898, {0x235a640, 0xc000045ad0}, 0x612d346530342d64)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001ef3770}, 0xc001ef35c0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00180cdb8, 0x4c1cb4}, {0xc0570311b98c93a7, 0xb33268832b}, {0xc00180cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001ef3770}, 0xc001ef35c0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0xc001fa0400, 0xc0019663c0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0x15, 0xc000c35780)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0x207690b, 0xc000c35780)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0}, 0x237bc38, 0xc000c35780)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3770}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ef3620}, {0x1f33180, 0xc001ef35c0}, 0x1e5a940, 0xc000c35780)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3620}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0x9, 0xc000c35780)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0xc0026da768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c0026dab60, {0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ef3590}, {0x1f33180, 0xc001ef35c0}, 0xc001fa0400, 0xc0019663c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001ef3590}, 0xc00283fb00, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00107cc60, 0xc0009e06f0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00107cc60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 356 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 201 [sleep]:
time.Sleep(0x37e11d600)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 354 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 855 [select, 13 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00075d860)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 179 [select]:
database/sql.(*DB).connectionCleaner(0xc0005796c0, 0x1fef77d)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 583 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060ebb0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 175 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42aef0c68, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000aec00, 0xc0007dd260, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000aec00, {0xc0007dd260, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000aec00, {0xc0007dd260, 0xc000ab1c20, 0xc001b7b230})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa060, {0xc0007dd260, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa060}, {0xc0007dd260, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00060c580, 0x1fef77d)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 176 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00060c580, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 360 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e8f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 186 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc0007c0750, {0x2392228, 0xc0009fb680})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 192 [sleep]:
time.Sleep(0x28fa6ae00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 190 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 851 [select]:
go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:258 +0x129
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:211 +0x11c
goroutine 848 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc002200ac0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1270842 [select, 1 minutes]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00213a240)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e9a0, {0x1f7b7c0, 0xc00204b9a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001969200, 0xc00186fbf8, 0xc00186fbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001969200, {0x1f7b7c0, 0xc00204b9a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x20, {0xc0019a4860, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0xc0026454a0, {0xc0019a4860, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0xc0023ec2f0, 0xc002136d50, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002136d20}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002446d00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x237ac70, 0xc002136c90, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002446d00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002446d00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc002446c80}, {0x1f7b7c0, 0xc00204b9a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc002446d00}, 0xc002672750, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0021365d0}, 0xc002446c80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0021365d0}, 0xc002446c80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001870480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001870528, {0x235a640, 0xc0000a8b10}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ace38b0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0021365d0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c000756ae0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001870778, {0x235a640, 0xc000045ad0}, 0x382d643438342d37)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0021365d0}, 0xc002446980)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00197adb8, 0x4c1cb4}, {0xc05703118058bf28, 0xb2f934aea6}, {0xc00197adb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0021365d0}, 0xc002446980)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0xc0019a4720, 0xc001b8d530)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0x15, 0xc0024469c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0x2076951, 0xc0024469c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980}, 0x237bc38, 0xc0024469c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0021365d0}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002136480}, {0x1f33320, 0xc002446980}, 0x1e5a940, 0xc0024469c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136480}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0x23c27b0, 0xc0024469c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a04b60, {0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0x40, 0xc0007c4800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002136450}, {0x1f33320, 0xc002446980}, 0xc0019a4720, 0xc001b8d530)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002136450}, 0xc002c5a300, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00211f320, 0xc0009e06f0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00211f320, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 213 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 850 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc00112d9c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1285974 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001766360)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bac60, {0x1f7c860, 0xc0022babb0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000735400, 0xc0005bf5e8, 0xc0005bf5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000735400, {0x1f7c860, 0xc0022babb0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x8}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x1c75901, {0xc0019a58a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x7fb451d0c5b8, {0xc0019a58a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x20, 0xc00256fb90, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0xc0007c0000, {0xc0021e6780, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, 0xc00256fb30, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x1e0cfc0, 0xc00256fa70, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x203000}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x8, 0xc00256f9e0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc000fe9ce0}, {0x20768da, 0x46}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, 0x0, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc000fe9ce0}, {0x20768da, 0x0}, {0x1f33180, 0xc00256f980}, {0x1f7c860, 0xc0022babb0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc000fe9ce0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc000fe9ce0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 671 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc001d68250)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 450 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005426e0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 136 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42ada69d8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c21700, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c21700)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000c21700)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000c7c7f8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000c7c7f8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc00053e560)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 137 [select, 13 minutes]:
database/sql.(*DB).connectionOpener(0xc000aa2270, {0x2392228, 0xc0006e6a80})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 183 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7fb42ada68f0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c21a00, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c21a00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000c21a00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000c7caf8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000c7caf8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000750380, {0x23860f0, 0xc000c7caf8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0xc0008217d0)
/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc00003a680}, 0x0)
/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
/temporal/service/frontend/fx.go:282 +0x9a
goroutine 184 [select, 3 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00056a6e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 1266067 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc000edac80)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1266071 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69e40, 0xc00185fb18)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 199 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 196 [IO wait, 13 minutes]:
internal/poll.runtime_pollWait(0x7fb42ada6468, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000afb00, 0x40d187, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0000afb00)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0000afb00)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000569488)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000569488)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007991c0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000790400)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 262 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 198 [chan receive, 13 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0xc000497fd0)
/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc000c73220}, 0x0)
/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
/temporal/service/worker/fx.go:78 +0x9a
goroutine 214 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1289988 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0024667e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001556370, {0x1f7c860, 0xc0015562c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x59)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aae00, 0xc0005bd5e8, 0xc0005bd5b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aae00, {0x1f7c860, 0xc0015562c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001da3680}, {0x20768da, 0x8}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x1c75901, {0xc000084420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x7fb451d0c5b8, {0xc000084420, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x20, 0xc0021be900, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0xc0007c0000, {0xc0009db040, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x0, 0xc0021be8d0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x1e0cfc0, 0xc0021be840, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001da3680}, {0x20768da, 0x203000}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x8, 0xc0021be810, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001da3680}, {0x20768da, 0x46}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, 0x7, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x7fb451d0c5b8, {0x2392298, 0xc001da3680}, {0x20768da, 0x0}, {0x1f33180, 0xc0021be7b0}, {0x1f7c860, 0xc0015562c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollActivityTaskQueue(0xc00000e2f0, {0x2392298, 0xc001da3680}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:406 +0xc9
go.temporal.io/sdk/internal.(*activityTaskPoller).poll(0xc00085a800, {0x2392298, 0xc001da3680})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:812 +0x1df
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 250 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5e10, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000888a80, 0xc000757350, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000888a80, {0xc000757350, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000888a80, {0xc000757350, 0xc0007b9f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa5b8, {0xc000757350, 0x46b213, 0x617b94f5})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa5b8}, {0xc000757350, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00075eb00, 0x7202b0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 254 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0b48, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000888980, 0xc0007574f0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000888980, {0xc0007574f0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000888980, {0xc0007574f0, 0xc0007bdf50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa590, {0xc0007574f0, 0x46b213, 0x617b94f5})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa590}, {0xc0007574f0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006eedc0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 355 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 451 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000542790)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 447 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 669 [select, 1 minutes]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000df12b0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 670 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc001d166e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 263 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 1266148 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001568160, 0xc001ba74b8)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 578 [select]:
github.com/temporalio/ringpop-go/swim.sendPingWithChanges(0xc0008bf040, {0xc001248bf0, 0x10}, {0x34c4b38, 0x0, 0x0}, 0x0)
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/ping_sender.go:97 +0x6f5
github.com/temporalio/ringpop-go/swim.sendPing(0xc0008bf040, {0xc001248bf0, 0x10}, 0xc000bf9f74)
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/ping_sender.go:47 +0x6b
github.com/temporalio/ringpop-go/swim.(*Node).pingNextMember(0xc0008bf040)
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/node.go:527 +0x108
github.com/temporalio/ringpop-go/swim.(*gossip).ProtocolPeriod(0xc00077b3f0)
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/gossip.go:177 +0x47
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:30 +0xb2
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 252 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5fe0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000aff80, 0xc0007574e0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000aff80, {0xc0007574e0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000aff80, {0xc0007574e0, 0xc0028a8750, 0xc000ed9460})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa5b0, {0xc0007574e0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa5b0}, {0xc0007574e0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006eeb00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 1282222 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006685a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022baf20, {0x1f7b560, 0xc00156e5a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb6)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7100, 0xc000882440, 0xc000882410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7100, {0x1f7b560, 0xc00156e5a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x20, {0xc0021d7ac0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0xc0020d7b80, {0xc0021d7ac0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0xc0021b2800, 0xc001d8c960, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d8c930}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x8, {0xc00062b440, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001cdff80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x237ac70, 0xc001d8c8a0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001cdff80}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001cdff80}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001cdff00}, {0x1f7b560, 0xc00156e5a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc0000ab0f8, {0x2392228, 0xc001cdff80}, 0xc0023b2ae6, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000ab1ce0, {0x2392298, 0xc0025d9a40}, 0xc001cdff00, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000799920, {0x2392298, 0xc0025d9a40}, 0xc001cdff00, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0024adaa0, {0x2392298, 0xc0025d9a40})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024adf80, {0x2392298, 0xc0025d9a40}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001082360, {0x23922d0, 0xc001c523f0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001c523f0}, 0xc001c50560, 0x1bd58c0, 0x1c50570)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001c52390, 0xc001c32680)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001c520f0}, 0xc001c32680)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001c520f0}, {0x1f353a0, 0xc001c32680})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001c520f0}, {0x1f353a0, 0xc001c32680}, 0x237bc38, 0xc0021dede0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c520f0}, {0x1f353a0, 0xc001c32680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001b75f50, {0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680}, 0xc00186d5b0, 0xc001c326c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75f80}, {0x1f353a0, 0xc001c32680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680}, 0x40, 0xc001c326c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001b75ef0}, {0x1f353a0, 0xc001c32680}, 0xc0021d7280, 0xc0021dede0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001b75ef0}, 0xc0025d99e0, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024805a0, 0xc000eeee10, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc0024805a0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 251 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00075eb00, 0x790000)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 253 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006eeb00, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 255 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006eedc0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 580 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 302 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0320, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889a80, 0xc000c66ad0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889a80, {0xc000c66ad0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889a80, {0xc000c66ad0, 0xc000ab1c20, 0xc001b7ad20})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aace8, {0xc000c66ad0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aace8}, {0xc000c66ad0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef8c0, 0xa99ae0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 573 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 452 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e630)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 453 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060e9a0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 448 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 449 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 1286320 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc002216120, {0x2392228, 0xc001dc2440})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1318843 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006457a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001838bb0, {0x1f7b7c0, 0xc000021180}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6a00, 0xc002347bf8, 0xc002347bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6a00, {0x1f7b7c0, 0xc000021180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x20, {0xc00247d480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0xc0020966e0, {0xc00247d480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0xc0017985b0, 0xc001d31320, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d31290}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001047600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x237ac70, 0xc001d31200, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001047600}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001047600}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001047580}, {0x1f7b7c0, 0xc000021180}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc001047600}, 0xc00104f140, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001d30990}, 0xc001047580, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001d30990}, 0xc001047580, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc002348480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002348528, {0x235a640, 0xc0000a8b10}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acf1d80, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001d30990}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c00289baa0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc002348778, {0x235a640, 0xc000045ad0}, 0x392d653239342d32)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001d30990}, 0xc001047280)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001b9cdb8, 0x4c1cb4}, {0xc0570314bb0103eb, 0xb5ff1e6b73}, {0xc001b9cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001d30990}, 0xc001047280)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0xc00247d380, 0xc0025baaf8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0x15, 0xc0010472c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0x2076951, 0xc0010472c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280}, 0x237bc38, 0xc0010472c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30990}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001d30840}, {0x1f33320, 0xc001047280}, 0x1e5a940, 0xc0010472c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30840}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0xc00104e940, 0xc0010472c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00186bb60, {0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0x40, 0xc0003e1400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d30810}, {0x1f33320, 0xc001047280}, 0xc00247d380, 0xc0025baaf8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001d30810}, 0xc001971aa0, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0017677a0, 0xc0009e06f0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0017677a0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 849 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc00104baa0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 672 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000f7cab0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 298 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0d18, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889b80, 0xc000c66960, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889b80, {0xc000c66960, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889b80, {0xc000c66960, 0xc000898f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aad10, {0xc000c66960, 0x46b213, 0x617b94f5})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aad10}, {0xc000c66960, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef340, 0x9e6580)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 247 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0a60, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000888a00, 0xc000757340, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000888a00, {0xc000757340, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000888a00, {0xc000757340, 0xc000c65d10, 0xc001b7a610})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aa598, {0xc000757340, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aa598}, {0xc000757340, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00075e840, 0xab8720)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 248 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00075e840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1277170 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc00222b4a0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 300 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42ada5c40, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889100, 0xc000c66ac0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889100, {0xc000c66ac0, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889100, {0xc000c66ac0, 0xc000c65d10, 0xc001b7ab40})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aad08, {0xc000c66ac0, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aad08}, {0xc000c66ac0, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef600, 0x5d09f0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 299 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef340, 0x1d8150)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 301 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef600, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1318923 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000127e60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9080, {0x1f736c0, 0xc001107b90}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aaf00, 0xc001c095c0, 0xc001c09590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aaf00, {0x1f736c0, 0xc001107b90})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0028c46c0}, {0x2076920, 0x8}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x1c75901, {0xc001fa0b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x7fb451d0c108, {0xc001fa0b20, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x20, 0xc001d38030, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0xc0006ca340, {0xc001146440, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x0, 0xc001d38000, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x0, 0xc001d67ec0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x40ce34, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0028c46c0}, {0x2076920, 0x203000}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x203000, 0xc001d67e90, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0028c46c0}, {0x2076920, 0x46}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, 0x7fb451d0c108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc0028c46c0}, {0x2076920, 0x200e03640}, {0x1f33320, 0xc001146400}, {0x1f736c0, 0xc001107b90}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc0028c46c0}, 0xc000790400, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc0028c46c0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1266177 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc000de6c30)
/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
/temporal/service/history/queueProcessor.go:158 +0x23b
goroutine 1266116 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0790, 0xc000a0b440)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 303 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef8c0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1318849 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00067b7a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce8fd0, {0x1f736c0, 0xc000f7abd0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x7c)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aac00, 0xc00152f5c0, 0xc00152f590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aac00, {0x1f736c0, 0xc000f7abd0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001cc76e0}, {0x2076920, 0x8}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x1c75901, {0xc001fa0820, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x7fb451d0c108, {0xc001fa0820, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x20, 0xc001d8d830, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0xc0006ca340, {0xc000f75ac0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x0, 0xc001d8d800, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x0, 0xc001d8d770, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x40ce34, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001cc76e0}, {0x2076920, 0x203000}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x203000, 0xc001d8d740, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001cc76e0}, {0x2076920, 0x46}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, 0x7fb451d0c108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc002471e50, {0x2392298, 0xc001cc76e0}, {0x2076920, 0x266343752}, {0x1f33320, 0xc000f75a80}, {0x1f736c0, 0xc000f7abd0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001cc76e0}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc001cc76e0})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1272371 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006777a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00205e420, {0x1f736c0, 0xc0007719e0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0010aa100, 0xc000aab5c0, 0xc000aab590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0010aa100, {0x1f736c0, 0xc0007719e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00230d920}, {0x2076920, 0x8}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x1c75901, {0xc001fa1260, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x7fb451d0c108, {0xc001fa1260, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x20, 0xc0009d7f20, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0xc0006ca340, {0xc000a944c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x0, 0xc0009d7ef0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x1e0cfc0, 0xc0009d7e60, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x4b700cb74e8, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00230d920}, {0x2076920, 0x203000}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x203000, 0xc0009d7e30, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00230d920}, {0x2076920, 0x46}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, 0x7fb451d0c108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000cb7650, {0x2392298, 0xc00230d920}, {0x2076920, 0x100000000}, {0x1f33320, 0xc000a94480}, {0x1f736c0, 0xc0007719e0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc00230d920}, 0x4802f6, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001240f70, {0x2392298, 0xc00230d920})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 668 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001240f70, 0xc000e5a9a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001240f70)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a3c0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 581 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060ea50)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 582 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc00060eb00)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 925 [select, 13 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc0007fd380)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0008dbef0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0008dbef0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1002155 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000946820, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0024594a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 579 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 403 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00016ae40, {0x2392228, 0xc000edabc0})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 404 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 295 [IO wait]:
internal/poll.runtime_pollWait(0x7fb42adb0238, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000889b00, 0xc000c66950, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000889b00, {0xc000c66950, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000889b00, {0xc000c66950, 0xc00089df50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0000aacf0, {0xc000c66950, 0x46b213, 0x617b94f5})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc0000aacf0}, {0xc000c66950, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc0006ef080, 0x9e6250)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 296 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc0006ef080, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 1266204 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015682c0, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1282558 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0019618c0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0022bb340, {0x1f7b560, 0xc00156ec80}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f7700, 0xc001611d18, 0xc001611ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f7700, {0x1f7b560, 0xc00156ec80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x20, {0xc0021d7f40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0xc002644f20, {0xc0021d7f40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0xc0023ec2f0, 0xc001d994d0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001d994a0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001d93580}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x237ac70, 0xc001d99410, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001d93580}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001d93580}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc001d93500}, {0x1f7b560, 0xc00156ec80}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc00000e5f8, {0x2392228, 0xc001d93580}, 0xc00003e900, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006aa120, {0x23922d0, 0xc001d98d50}, 0xc001d93500, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0007982c0, {0x23922d0, 0xc001d98d50}, 0xc001d93500, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0016125a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001612648, {0x235a640, 0xc0000a8b10}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad221e0, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc000c4d050, {0x23922d0, 0xc001d98d50}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0024ba260})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001612898, {0x235a640, 0xc000045ad0}, 0x382d613065342d39)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc000045ad0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc00013e150, {0x23922d0, 0xc001d98d50}, 0xc001d98ba0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc0005c6db8, 0x4c1cb4}, {0xc0570312400e0741, 0xb3abba54c0}, {0xc0005c6db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0003ef440, {0x23922d0, 0xc001d98d50}, 0xc001d98ba0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0xc0021d7e80, 0xc002652e28)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0x15, 0xc001d932c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0x207690b, 0xc001d932c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0}, 0x237bc38, 0xc001d932c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98d50}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001d98c00}, {0x1f33180, 0xc001d98ba0}, 0x1e5a940, 0xc001d932c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98c00}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0x9, 0xc001d932c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0xc001867768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001867b60, {0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0x40, 0xc00005fc00)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001d98b70}, {0x1f33180, 0xc001d98ba0}, 0xc0021d7e80, 0xc002652e28)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc001d98b70}, 0xc00250bb00, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001960480, 0xc0009e06f0, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc001960480, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1286340 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0023cb4a0, {0x2392298, 0xc0028f9a40}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a12b40, {0x23922d0, 0xc0020fa900}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0020fa900}, 0xc001d687e0, 0x1bd58c0, 0x1d687f0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc0020fa540, 0xc001d5eb40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc0020fa270}, 0xc001d5eb40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0020fa270}, {0x1f353a0, 0xc001d5eb40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc0020fa270}, {0x1f353a0, 0xc001d5eb40}, 0x237bc38, 0xc0022e3920)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa270}, {0x1f353a0, 0xc001d5eb40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40}, 0x203000, 0xc001d5eb80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa120}, {0x1f353a0, 0xc001d5eb40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40}, 0x40, 0xc001d5eb80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0020fa090}, {0x1f353a0, 0xc001d5eb40}, 0xc001fa02c0, 0xc0022e3920)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc0020fa090}, 0xc0028f99e0, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0017f9320, 0xc000eeee10, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0017f9320, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1286313 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002339320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00060ef20, {0x1f736c0, 0xc000daccf0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x5f)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f6e00, 0xc00158b5c0, 0xc00158b590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f6e00, {0x1f736c0, 0xc000daccf0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc001b67980}, {0x2076920, 0x8}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x1c75901, {0xc0008e6660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x7fb451d0c5b8, {0xc0008e6660, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x20, 0xc0017fe1b0, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0xc0006ca340, {0xc001d47480, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x0, 0xc0017fe180, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x0, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x1e0cfc0, 0xc0017fe0f0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x4cc47c577b3, {0xc0000849c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc001b67980}, {0x2076920, 0x203000}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x203000, 0xc0017fe0c0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc001b67980}, {0x2076920, 0x46}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, 0x7fb451d0c5b8, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000e03e50, {0x2392298, 0xc001b67980}, {0x2076920, 0x14f6a69fe}, {0x1f33320, 0xc001d47440}, {0x1f736c0, 0xc000daccf0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000e2f0, {0x2392298, 0xc001b67980}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc000e57c70, {0x2392298, 0xc001b67980})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 1266215 [select, 1 minutes]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001568210)
/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
/temporal/service/history/visibilityQueueProcessor.go:203 +0x85
goroutine 1266101 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc0015680b0, 0xc0000a3770)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1286281 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00110fda0, {0x2392228, 0xc001d2f340})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 1266056 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001f69d90, 0xc00185fd58)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1266051 [select, 1 minutes]:
go.temporal.io/server/service/history.NewLocalTimerGate.func1()
/temporal/service/history/timerGate.go:114 +0xe5
created by go.temporal.io/server/service/history.NewLocalTimerGate
/temporal/service/history/timerGate.go:109 +0x15e
goroutine 1318874 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000a923c0, {0x2392298, 0xc002037260}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000378b40, {0x23922d0, 0xc001db4960}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0006ccc60, {0x23922d0, 0xc001db4960}, 0xc00105e150, 0x1, 0x105e160)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc000828460, 0xc001db4900, 0xc001052d00)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000abef60, {0x23922d0, 0xc001db4690}, 0xc001052d00)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001db4690}, {0x1f35540, 0xc001052d00})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001db4690}, {0x1f35540, 0xc001052d00}, 0x237bc38, 0xc0025bbcc8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db4690}, {0x1f35540, 0xc001052d00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00}, 0xc001052780, 0xc001052d80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db4510}, {0x1f35540, 0xc001052d00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00}, 0x40, 0xc001052d80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001db44e0}, {0x1f35540, 0xc001052d00}, 0xc00247dbe0, 0xc0025bbcc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001db44e0}, 0xc002037200, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c13c20, 0xc000eeee10, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc00084fb00}, 0xc000c13c20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1276962 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0006bec60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001cb5ce0, {0x1f7b7c0, 0xc000020e60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb7)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00084a800, 0xc001751bf8, 0xc001751bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00084a800, {0x1f7b7c0, 0xc000020e60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x20, {0xc000799540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0xc001581760, {0xc000799540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0xc0023ec2f0, 0xc002077e30, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc002077e00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00207e9c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x237ac70, 0xc002077d70, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00207e9c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00207e9c0}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00207e940}, {0x1f7b7c0, 0xc000020e60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc00207e9c0}, 0xc00074c5e0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc0020776b0}, 0xc00207e940, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc0020776b0}, 0xc00207e940, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001752480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001752528, {0x235a640, 0xc0000a8b10}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad35118, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc0020776b0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002061090})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001752778, {0x235a640, 0xc000045ad0}, 0x392d363266342d33)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc0020776b0}, 0xc00207e640)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc001c64db8, 0x4c1cb4}, {0xc05703120130b699, 0xb371423a15}, {0xc001c64db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc0020776b0}, 0xc00207e640)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0xc000799260, 0xc00205d230)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0x15, 0xc00207e680)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0x2076951, 0xc00207e680)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640}, 0x237bc38, 0xc00207e680)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0020776b0}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002077560}, {0x1f33320, 0xc00207e640}, 0x1e5a940, 0xc00207e680)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077560}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0x23c27b0, 0xc00207e680)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000a05b60, {0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002077530}, {0x1f33320, 0xc00207e640}, 0xc000799260, 0xc00205d230)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002077530}, 0xc001ccc120, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006810e0, 0xc0009e06f0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc0006810e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1282217 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00230c1e0, {0x2392298, 0xc0025d9500}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00211d320, {0x23922d0, 0xc001b75770}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001b75770}, 0xc001b55e10, 0x1bd58c0, 0x1b55e20)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc000828460, 0xc001b75560, 0xc001c320c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000abef60, {0x23922d0, 0xc001b752c0}, 0xc001c320c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001b752c0}, {0x1f353a0, 0xc001c320c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000a9ee58, {0x23922d0, 0xc001b752c0}, {0x1f353a0, 0xc001c320c0}, 0x237bc38, 0xc0021de8d0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b752c0}, {0x1f353a0, 0xc001c320c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x7, {0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0}, 0xc00186d5b0, 0xc001c32100)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75170}, {0x1f353a0, 0xc001c320c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0}, 0x40, 0xc001c32100)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001b75110}, {0x1f353a0, 0xc001c320c0}, 0xc0021d7120, 0xc0021de8d0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000abef60}, {0x23922d0, 0xc001b75110}, 0xc0025d94a0, 0xc000799a00)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0006765a0, 0xc000eeee10, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00002d340, {0x23d1088, 0xc000ad2d80}, 0xc0006765a0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 869 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc001d90440)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 870 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc00085a800, 0xc002122630)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc00085a800)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc00125a5a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc00125a5a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 1266142 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0c60, 0xc00101cf00)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1318720 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc002466480)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0018389a0, {0x1f7b7c0, 0xc0000205a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001968d00, 0xc001755bf8, 0xc001755bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001968d00, {0x1f7b7c0, 0xc0000205a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x20, {0xc00247cda0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x20}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0xc001581600, {0xc00247cda0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0xc0017985b0, 0xc0021ca690, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0021ca660}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc000fb3900}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x237ac70, 0xc0021ca5d0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc000fb3900}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc000fb3900}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc000fb3880}, {0x1f7b7c0, 0xc0000205a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc000fb3900}, 0xc001d49b40, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc002203d10}, 0xc000fb3880, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc002203d10}, 0xc000fb3880, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc001756480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001756528, {0x235a640, 0xc0000a8b10}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42acc4140, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c5b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002203d10}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c002104ee0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001756778, {0x235a640, 0xc000045ad0}, 0x622d626262342d39)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc002203d10}, 0xc000fb3580)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00158cdb8, 0x4c1cb4}, {0xc0570314b85e7290, 0xb5fc7bda76}, {0xc00158cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc002203d10}, 0xc000fb3580)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0xc00247ccc0, 0xc0025e5278)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0x15, 0xc000fb35c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0x2076951, 0xc000fb35c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580}, 0x237bc38, 0xc000fb35c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203d10}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002203bc0}, {0x1f33320, 0xc000fb3580}, 0x1e5a940, 0xc000fb35c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203bc0}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0x23c27b0, 0xc000fb35c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c00186bb60, {0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0x40, 0xc00005e800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002203b90}, {0x1f33320, 0xc000fb3580}, 0xc00247ccc0, 0xc0025e5278)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0003ef440}, {0x23922d0, 0xc002203b90}, 0xc0015bb680, 0xc0007987c0)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00032d440, 0xc0009e06f0, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc000750380, {0x23d1088, 0xc000c08d80}, 0xc00032d440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 1266132 [select, 1 minutes]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001be0b00, 0xc00185f770)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 1174603 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc002574140)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 1319106 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b845a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001ce9a20, {0x1f7b7c0, 0xc0019808c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000ee6800, 0xc001ca5bf8, 0xc001ca5bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000ee6800, {0x1f7b7c0, 0xc0019808c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x20, {0xc0016521c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0xc00199e9a0, {0xc0016521c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0xc0017985b0, 0xc001f1a960, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001f1a930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x8, {0xc000c5f1f0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00117f640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x237ac70, 0xc001f1a870, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00117f640}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00117f640}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00117f5c0}, {0x1f7b7c0, 0xc0019808c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc00000e5f8, {0x2392228, 0xc00117f640}, 0xc00203d5c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006aa120, {0x23922d0, 0xc001ea3ec0}, 0xc00117f5c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0007982c0, {0x23922d0, 0xc001ea3ec0}, 0xc00117f5c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001ca6480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001ca6528, {0x235a640, 0xc0000a8b10}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7fb42ad24468, {0x235a640, 0xc0000a8b10}, 0x7fb451d0c108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001ea3ec0}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc000096870, 0x10100c0027af7c0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc0000b0000}, 0xc001ca6778, {0x235a640, 0xc000045ad0}, 0x612d633434342d38)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc000c5fbc0, {0x235a640, 0xc000045ad0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc00013e150, {0x23922d0, 0xc001ea3ec0}, 0xc00117f2c0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc0006d65c0, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0003ef440, {0xc00155edb8, 0x4c1cb4}, {0xc0570314c2e5511a, 0xb6029d8298}, {0xc00155edb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0003ef440, {0x23922d0, 0xc001ea3ec0}, 0xc00117f2c0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc000754410, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0xc0016520a0, 0xc001fb8cc0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc0006aa840, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0x15, 0xc00117f300)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc0006aa660, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0x2076951, 0xc00117f300)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc000568be8, {0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0}, 0x237bc38, 0xc00117f300)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3ec0}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001ea3d70}, {0x1f33320, 0xc00117f2c0}, 0x1e5a940, 0xc00117f300)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3d70}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0}, 0x23c27b0, 0xc00117f300)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ea3d40}, {0x1f33320, 0xc00117f2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/serve