Skip to content

Instantly share code, notes, and snippets.

@ngoclamtran
Created Nov 1, 2021
Embed
What would you like to do?
This file has been truncated, but you can view the full file.
goroutine 968584 [running]:
runtime/pprof.writeGoroutineStacks({0x235d7a0, 0xc00074ed20})
/usr/local/go/src/runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x235d7a0, 0xc00074ed20}, 0x0)
/usr/local/go/src/runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x1e1e9c0, {0x235d7a0, 0xc00074ed20}, 0xc)
/usr/local/go/src/runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc002201361, 0x34934e0}, {0x2386300, 0xc00074ed20}, 0xc002201354)
/usr/local/go/src/net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x2386300, 0xc00074ed20}, 0xc000175700)
/usr/local/go/src/net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x2386300, 0xc00074ed20}, 0x0)
/usr/local/go/src/net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc00220136b, {0x2386300, 0xc00074ed20}, 0xc000175700)
/usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc001c0fbc0}, {0x2386300, 0xc00074ed20}, 0xc000175700)
/usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc001bba000, {0x23922d0, 0xc0007405a0})
/usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3033 +0x4e8
goroutine 1 [chan receive, 6 minutes]:
go.temporal.io/server/temporal.(*Server).Start(0xc00017e7e0)
/temporal/temporal/server.go:272 +0xe9c
main.buildCLI.func2(0xc0005ce640)
/temporal/cmd/server/main.go:168 +0xc4d
github.com/urfave/cli/v2.(*Command).Run(0xc0003f8ea0, 0xc0005ce600)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/command.go:163 +0x64a
github.com/urfave/cli/v2.(*App).RunContext(0xc0003e9860, {0x2392260, 0xc00012c000}, {0xc00012a000, 0x8, 0x8})
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:313 +0x81e
github.com/urfave/cli/v2.(*App).Run(...)
/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
/temporal/cmd/server/main.go:50 +0x45
goroutine 21 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00012a300)
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
/go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92
goroutine 22 [syscall, 6 minutes]:
os/signal.signal_recv()
/usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
/usr/local/go/src/os/signal/signal.go:151 +0x2c
goroutine 10 [chan receive, 6 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 11 [chan receive, 6 minutes]:
go.temporal.io/server/temporal.InterruptCh.func1()
/temporal/temporal/interrupt.go:39 +0x31
created by go.temporal.io/server/temporal.InterruptCh
/temporal/temporal/interrupt.go:38 +0xef
goroutine 55 [select]:
database/sql.(*DB).connectionCleaner(0xc0001ff450, 0xc000090370)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 631595 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001b4bb00, {0x2392228, 0xc00129da40})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 941373 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00181c6c0, {0x2392298, 0xc002064840}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00181e360, {0x23922d0, 0xc00222cab0}, 0x31)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0027c6120, {0x23922d0, 0xc00222cab0}, 0xc002505100, 0x1, 0x2505110)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00074e0e0, 0xc00222c9c0, 0xc001833980)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000084240, {0x23922d0, 0xc00222c750}, 0xc001833980)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00222c750}, {0x1f35540, 0xc001833980})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc00222c750}, {0x1f35540, 0xc001833980}, 0x237bc38, 0xc001f1c5d0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222c750}, {0x1f35540, 0xc001833980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00222c600}, {0x1f35540, 0xc001833980}, 0xc001833400, 0xc001833a00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222c600}, {0x1f35540, 0xc001833980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00222c600}, {0x1f35540, 0xc001833980}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222c600}, {0x1f35540, 0xc001833980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00222c5d0}, {0x1f35540, 0xc001833980}, 0x40, 0xc001833a00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222c5d0}, {0x1f35540, 0xc001833980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00222c5d0}, {0x1f35540, 0xc001833980}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222c5d0}, {0x1f35540, 0xc001833980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00222c5d0}, {0x1f35540, 0xc001833980}, 0xc001aa2320, 0xc001f1c5d0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc00222c5d0}, 0xc0020647e0, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc001a3ec60, 0xc000753950, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc001a3ec60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 72 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000100370)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 38 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e9f9a30, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000744380, 0x416f26, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000744380)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000744380)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00000c6a8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00000c6a8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc00074e000, {0x23860f0, 0xc00000c6a8})
/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc00074e000)
/usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
/usr/local/go/src/net/http/server.go:3184
go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start.func1()
/temporal/common/pprof/pprof.go:74 +0x245
created by go.temporal.io/server/common/pprof.(*PProfInitializerImpl).Start
/temporal/common/pprof/pprof.go:72 +0x9c
goroutine 62 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 58 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f561e9f9b18, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00003bd80, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00003bd80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00003bd80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000900618)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000900618)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000612420)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000914000)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 938336 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 71 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f561e9f9948, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000920200, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000920200)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000920200)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000900948)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000900948)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc00044ce00, {0x23860f0, 0xc000900948})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/history.(*Service).Start(0xc0007327d0)
/temporal/service/history/service.go:99 +0x185
go.temporal.io/server/service/history.ServiceLifetimeHooks.func1.1({0x237c7c8, 0xc000100230}, 0x0)
/temporal/service/history/fx.go:170 +0x2c
created by go.temporal.io/server/service/history.ServiceLifetimeHooks.func1
/temporal/service/history/fx.go:168 +0x9a
goroutine 938419 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8a50, 0xc000711c38)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 59 [select, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc0001ffc70, {0x2392228, 0xc0009243c0})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 83 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f561e9f9690, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872c80, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000872c80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000872c80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00011c4b0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00011c4b0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc000612d80)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc00044a200)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 28 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000090370)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 30 [select, 3 minutes]:
database/sql.(*DB).connectionOpener(0xc0001ff450, {0x2392228, 0xc0008ba900})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 872 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc0007f2a20)
/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1748 +0x1e65
goroutine 74 [select]:
database/sql.(*DB).connectionOpener(0xc0008dcea0, {0x2392228, 0xc0008ae080})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 78 [select]:
database/sql.(*DB).connectionCleaner(0xc0008dcea0, 0xc0005c001c)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 938343 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de80b0, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 80 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 86 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000090c30)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 85 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f561e9f95a8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000872f80, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000872f80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000872f80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00011c7b0)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00011c7b0)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0005d0380, {0x23860f0, 0xc00011c7b0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/matching.(*Service).Start(0xc00072e7d0)
/temporal/service/matching/service.go:128 +0x154
go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1.1({0x237c840, 0xc0009570b0}, 0x0)
/temporal/service/matching/fx.go:131 +0x2c
created by go.temporal.io/server/service/matching.ServiceLifetimeHooks.func1
/temporal/service/matching/fx.go:129 +0x9a
goroutine 996 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc001161860, 0xc00231a880)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*workflowTaskPoller).PollTask(0xc001161860)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:241 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc0007614a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc0007614a0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 99 [sleep]:
time.Sleep(0x342770c00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 938408 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de89a0, 0xc0020b4428)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 104 [select]:
database/sql.(*DB).connectionCleaner(0xc0005f4ea0, 0xc00009001c)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 938324 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c276b0, 0xc0021256b0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 88 [select, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc0005f4ea0, {0x2392228, 0xc00065a540})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 938393 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8840, 0xc00209f2c0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 161 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 92 [chan receive]:
github.com/rcrowley/go-metrics.(*meterArbiter).tick(0x34906a0)
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:239 +0x2a
created by github.com/rcrowley/go-metrics.NewMeter
/go/pkg/mod/github.com/rcrowley/go-metrics@v0.0.0-20201227073835-cf1acfcdf475/meter.go:46 +0xd8
goroutine 93 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 94 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 144 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 202 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005a9600)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 159 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000625290)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 814370 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 200 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 201 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 271 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 273 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 105 [sleep]:
time.Sleep(0x342770c00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 938431 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8dc0, 0xc001e38830)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 194 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00001a000, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 193 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e9f8e68, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006a9100, 0xc00072d780, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0006a9100, {0xc00072d780, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0006a9100, {0xc00072d780, 0xc000cebef0, 0xc0029bb1c0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f670, {0xc00072d780, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f670}, {0xc00072d780, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00001a000, 0x8a9360)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 204 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005a93f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 261 [select]:
database/sql.(*DB).connectionCleaner(0xc000cc6750, 0xc00078b008)
/usr/local/go/src/database/sql/sql.go:1068 +0xbd
created by database/sql.(*DB).startCleanerLocked
/usr/local/go/src/database/sql/sql.go:1055 +0x105
goroutine 310 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00009a580, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 309 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e81c4d0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00012b580, 0xc00091b700, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00012b580, {0xc00091b700, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00012b580, {0xc00091b700, 0xc002143f20, 0xc0029a6330})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e6b8, {0xc00091b700, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000e6b8}, {0xc00091b700, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00009a580, 0x3d2960)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 337 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0006253f0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 199 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 195 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e9f8f50, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006a8680, 0xc00072d980, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0006a8680, {0xc00072d980, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0006a8680, {0xc00072d980, 0xc0006b3f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000f668, {0xc00072d980, 0x46b213, 0x617f62ee})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000f668}, {0xc00072d980, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00002e840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 196 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00002e840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 203 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005a9970)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 205 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0005a94a0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 160 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000625340)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 267 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f561e9f8d80, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000c7d400, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000c7d400)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000c7d400)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000c74af8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000c74af8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0007e42a0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc00044a600)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 269 [chan receive, 6 minutes]:
go.temporal.io/server/service/worker.(*Service).Start(0x0)
/temporal/service/worker/service.go:171 +0x228
go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1.1({0x237c890, 0xc000100280}, 0xc00087b240)
/temporal/service/worker/fx.go:80 +0x2c
created by go.temporal.io/server/service/worker.ServiceLifetimeHooks.func1
/temporal/service/worker/fx.go:78 +0x9a
goroutine 274 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 314 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00009ab00, 0x8a001c)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 148 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00074a180, {0x2392228, 0xc0006acac0})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 259 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 110 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f561e9f9120, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00012be80, 0xc00004c000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00012be80)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00012be80)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0008371e8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0008371e8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
github.com/uber/tchannel-go/tnet.(*listener).Accept(0xc0006133e0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/tnet/listener.go:80 +0x77
github.com/uber/tchannel-go.(*Channel).serve(0xc000914a00)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:518 +0x46
created by github.com/uber/tchannel-go.(*Channel).Serve
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/channel.go:389 +0x2ef
goroutine 111 [select, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc0008dde10, {0x2392228, 0xc0006ac640})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 255 [select]:
go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start.func1()
/temporal/common/metrics/runtime.go:145 +0x86
created by go.temporal.io/server/common/metrics.(*RuntimeMetricsReporter).Start
/temporal/common/metrics/runtime.go:142 +0x75
goroutine 938377 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8370, 0x8)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 262 [sleep]:
time.Sleep(0x28fa6ae00)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 251 [select, 6 minutes]:
database/sql.(*DB).connectionOpener(0xc000cc6750, {0x2392228, 0xc000cd1440})
/usr/local/go/src/database/sql/sql.go:1196 +0x93
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:794 +0x188
goroutine 938297 [select]:
go.temporal.io/server/service/history.NewLocalTimerGate.func1()
/temporal/service/history/timerGate.go:114 +0xe5
created by go.temporal.io/server/service/history.NewLocalTimerGate
/temporal/service/history/timerGate.go:109 +0x15e
goroutine 248 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e9f9038, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006a8180, 0xc00004e800, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0006a8180)
/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0006a8180)
/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc0008377e8)
/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc0008377e8)
/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc00044d180, {0x23860f0, 0xc0008377e8})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:779 +0x362
go.temporal.io/server/service/frontend.(*Service).Start(0x0)
/temporal/service/frontend/service.go:238 +0x230
go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1.1({0x237c750, 0xc0006a9480}, 0xc00078b050)
/temporal/service/frontend/fx.go:284 +0x2c
created by go.temporal.io/server/service/frontend.ServiceLifetimeHooks.func1
/temporal/service/frontend/fx.go:282 +0x9a
goroutine 249 [select, 1 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0007dec80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:77 +0xa7
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.41.0/balancer_conn_wrappers.go:67 +0x246
goroutine 836 [select]:
go.temporal.io/server/service/history/shard.(*ControllerImpl).shardManagementPump(0xc000cabea0)
/temporal/service/history/shard/controller_impl.go:312 +0x148
created by go.temporal.io/server/service/history/shard.(*ControllerImpl).Start
/temporal/service/history/shard/controller_impl.go:145 +0xaf
goroutine 327 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000624fd0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 938439 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc0013ee690)
/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
/temporal/service/history/queueProcessor.go:158 +0x23b
goroutine 938371 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8370, 0xc002146a00)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 461 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 392 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001e9680, {0x2392228, 0xc0005d9800})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 155 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 158 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc0006251e0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 157 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 311 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e81bca8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00012b280, 0xc00091b710, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00012b280, {0xc00091b710, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00012b280, {0xc00091b710, 0xc000660f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e600, {0xc00091b710, 0x46b213, 0x617f62ee})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000e600}, {0xc00091b710, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00009a840, 0x3d2b80)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 312 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00009a840, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 315 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e81c5b8, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000921500, 0xc00091b960, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000921500, {0xc00091b960, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000921500, {0xc00091b960, 0xc00068df50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e6b0, {0xc00091b960, 0x46b213, 0x617f62ee})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000e6b0}, {0xc00091b960, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00009adc0, 0x78b008)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 316 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00009adc0, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 329 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000625130)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 328 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000625080)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 326 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000624f20)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 325 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 450 [select, 6 minutes]:
go.temporal.io/server/service/history/events.(*NotifierImpl).dequeueHistoryEventNotifications(0xc000091220)
/temporal/service/history/events/notifier.go:234 +0x105
created by go.temporal.io/server/service/history/events.(*NotifierImpl).Start
/temporal/service/history/events/notifier.go:253 +0x71
goroutine 449 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc0001e92c0, {0x2392228, 0xc0005d9dc0})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 941902 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000819560)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000d1fd90, {0x1f7b7c0, 0xc0013c1d60}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000799500, 0xc0017cbbf8, 0xc0017cbbc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000799500, {0x1f7b7c0, 0xc0013c1d60})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001c70690}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0x20, {0xc001aa3e40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001c70690}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c70690}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0xc000bbfa20, {0xc001aa3e40, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c70690}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0xc00276d380, 0xc001c706c0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001c70690}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001c69480}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0x237ac70, 0xc001c70600, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001c69480}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001c69480}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001c69400}, {0x1f7b7c0, 0xc0013c1d60}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000cbaaa0, {0x2392228, 0xc001c69480}, 0xc0027116c0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006778c0, {0x23922d0, 0xc001c61f80}, 0xc001c69400, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0001437a0, {0x23922d0, 0xc001c61f80}, 0xc001c69400, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc0017cc480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc0017cc528, {0x235a640, 0xc000044390}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e7dc318, {0x235a640, 0xc000044390}, 0x7f56458145b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001c61f80}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c002481360})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc0017cc778, {0x235a640, 0xc0000458c0}, 0x382d353033342d39)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0008802f0, {0x235a640, 0xc0000458c0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001471f0, {0x23922d0, 0xc001c61f80}, 0xc001c69100)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc0007b2db8, 0x4c1cb4}, {0xc057f6994a0f9c2e, 0x5426be21c9}, {0xc0007b2db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0008a6540, {0x23922d0, 0xc001c61f80}, 0xc001c69100)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100}, 0xc001aa3d60, 0xc001fe24c8)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100}, 0x15, 0xc001c69140)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100}, 0x2076951, 0xc001c69140)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100}, 0x237bc38, 0xc001c69140)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61f80}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001c61e30}, {0x1f33320, 0xc001c69100}, 0x1e5a940, 0xc001c69140)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61e30}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100}, 0x23c27b0, 0xc001c69140)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001df8b60, {0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100}, 0x40, 0xc00005f400)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001c61e00}, {0x1f33320, 0xc001c69100}, 0xc001aa3d60, 0xc001fe24c8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc001c61e00}, 0xc0021b4d20, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc000776a20, 0xc000ceab10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc000776a20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 394 [select, 6 minutes]:
go.temporal.io/server/service/frontend.(*VersionChecker).versionCheckLoop(0xc0007deb40)
/temporal/service/frontend/versionChecker.go:88 +0xe5
created by go.temporal.io/server/service/frontend.(*VersionChecker).Start.func1
/temporal/service/frontend/versionChecker.go:70 +0x5f
goroutine 393 [select, 1 minutes]:
go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).purgeProcessor(0xc0005b6070)
/temporal/common/persistence/namespaceReplicationQueue.go:404 +0xed
created by go.temporal.io/server/common/persistence.(*namespaceReplicationQueueImpl).Start
/temporal/common/persistence/namespaceReplicationQueue.go:123 +0x71
goroutine 399 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 743326 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001a389b0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 156 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 313 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e81bd90, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00012b200, 0xc00091b950, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00012b200, {0xc00091b950, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00012b200, {0xc00091b950, 0xc002143f20, 0xc001e931e0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e5f8, {0xc00091b950, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000e5f8}, {0xc00091b950, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00009ab00, 0x880001)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 323 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 324 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 938399 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8840, 0xc000d4e4b8)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938313 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27550, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 922432 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003e4930, {0x2392228, 0xc001d48b00})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 938312 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27550, 0x1c)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938279 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27340, 0xc0013cb340)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938370 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8370, 0xc0020e2ca8)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 928628 [select]:
database/sql.(*DB).conn(0xc0008dcea0, {0x2392298, 0xc0017c9920}, 0x1)
/usr/local/go/src/database/sql/sql.go:1312 +0x413
database/sql.(*DB).query(0x1b11548, {0x2392298, 0xc0017c9920}, {0x208f76a, 0x94}, {0xc002466320, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1695 +0x5d
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0017c9920}, {0x208f76a, 0x94}, {0xc002466320, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc0008b5890, {0x2392298, 0xc0017c9920}, {0x208f76a, 0xc00004c000}, {0xc002466320, 0xc001b116c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0017c9920}, {0x2380120, 0xc0008b5890}, {0x1b5d8e0, 0xc0018ce870}, {0x208f76a, 0xc002201100}, {0xc002466320, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2b, {0x2392298, 0xc0017c9920}, {0x1b5d8e0, 0xc0018ce870}, {0x208f76a, 0xc001b117e0}, {0xc002466320, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc0008f8be0, {0x2392298, 0xc0017c9920}, {0xdb6a293a, {0xc00260c880, 0x3c, 0x3c}, 0x0, 0xc000cd0a28, 0xc002683900, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0008b5c50, 0xc000cd0a00)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000142360, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0008b5c80, 0xc00005f400)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0008b5cb0, 0x2b)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc0011de840, 0xd0c2620, 0xd0dacc0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc00045d000, 0xc0022b16e0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc001b11c50, {0x235a640, 0xc0005f2240}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0005f2240}, 0xc001b11d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001b11de2, 0xc0008e9040)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001b11d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc0011dea20)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0011dea20, {0x2392228, 0xc00235f380})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 935513 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008cc5a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001fa0420, {0x1f7b560, 0xc002452c80}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0011fbc00, 0xc000b71d18, 0xc000b71ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0011fbc00, {0x1f7b560, 0xc002452c80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0023bcf00}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0x20, {0xc001cef4a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0023bcf00}, {0x207eb79, 0x20}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023bcf00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0xc00233c000, {0xc001cef4a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0023bcf00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0xc00276d380, 0xc0023bcf30, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0023bcf00}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00153d240}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0x237ac70, 0xc0023bce70, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00153d240}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00153d240}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc00153d1c0}, {0x1f7b560, 0xc002452c80}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000cbaaa0, {0x2392228, 0xc00153d240}, 0xc00047a7b0, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006778c0, {0x23922d0, 0xc0023bc7b0}, 0xc00153d1c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0001437a0, {0x23922d0, 0xc0023bc7b0}, 0xc00153d1c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000b725a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000b72648, {0x235a640, 0xc000044390}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e78ea38, {0x235a640, 0xc000044390}, 0x7f56458145b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001dc8360, {0x23922d0, 0xc0023bc7b0}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c002481a00})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000b72898, {0x235a640, 0xc0000458c0}, 0x382d326666342d37)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000458c0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001471f0, {0x23922d0, 0xc0023bc7b0}, 0xc0023bc600)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc001b98db8, 0x4c1cb4}, {0xc057f698cb4ef426, 0x53b0c7e5c1}, {0xc001b98db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0008a6540, {0x23922d0, 0xc0023bc7b0}, 0xc0023bc600)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600}, 0xc001cef2e0, 0xc001c5da58)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600}, 0x15, 0xc00153cf80)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600}, 0x207690b, 0xc00153cf80)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600}, 0x237bc38, 0xc00153cf80)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc7b0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc0023bc660}, {0x1f33180, 0xc0023bc600}, 0x1e5a940, 0xc00153cf80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc660}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600}, 0x9, 0xc00153cf80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600}, 0xc001f0c768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001f0cb60, {0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600}, 0x40, 0xc0011f7800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0023bc5d0}, {0x1f33180, 0xc0023bc600}, 0xc001cef2e0, 0xc001c5da58)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc0023bc5d0}, 0xc0021f4b40, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc0001fd440, 0xc000ceab10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc0001fd440, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 938429 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8dc0, 0xc001da2bd0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 743124 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc000700b60, {0x2392228, 0xc002714d40})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 735269 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001ce6280, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000a78d80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 938383 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8580, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 925365 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002544f00, {0x2392228, 0xc00224b400})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 743125 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000c58f60, {0x2392228, 0xc002714d80})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 938440 [select]:
go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).completeTaskLoop(0xc001de8b00)
/temporal/service/history/visibilityQueueProcessor.go:229 +0x125
created by go.temporal.io/server/service/history.(*visibilityQueueProcessorImpl).Start
/temporal/service/history/visibilityQueueProcessor.go:203 +0x85
goroutine 922638 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e86d680, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001898a00, 0xc001add000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001898a00, {0xc001add000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001898a00, {0xc001add000, 0xc000c10fd8, 0x6625c9})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000125a08, {0xc001add000, 0xc000c11020, 0x6738a5})
/usr/local/go/src/net/net.go:183 +0x45
github.com/go-sql-driver/mysql.(*buffer).fill(0xc001a3f8c0, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:90 +0x23a
github.com/go-sql-driver/mysql.(*buffer).readNext(0xc001a3f8c0, 0x4)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/buffer.go:119 +0x2d
github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc001a3f8c0)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:31 +0x85
github.com/go-sql-driver/mysql.(*mysqlStmt).readPrepareResultPacket(0xc00078aee8)
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/packets.go:829 +0x28
github.com/go-sql-driver/mysql.(*mysqlConn).Prepare(0xc001a3f8c0, {0x208f76a, 0xc0028a30e0})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:167 +0x18f
github.com/go-sql-driver/mysql.(*mysqlConn).PrepareContext(0xc0028a30e0, {0x2392298, 0xc0028a30e0}, {0x208f76a, 0x94})
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:540 +0x53
database/sql.ctxDriverPrepare({0x2392298, 0xc0028a30e0}, {0x237efb0, 0xc001a3f8c0}, {0x208f76a, 0x94})
/usr/local/go/src/database/sql/ctxutil.go:15 +0x11b
database/sql.(*DB).queryDC.func2()
/usr/local/go/src/database/sql/sql.go:1744 +0x4b
database/sql.withLock({0x237b030, 0xc001954000}, 0xc000c113d0)
/usr/local/go/src/database/sql/sql.go:3396 +0x8c
database/sql.(*DB).queryDC(0x203001, {0x2392298, 0xc0028a30e0}, {0x0, 0x0}, 0xc001954000, 0xc001e93dd0, {0x208f76a, 0x94}, {0xc0023b9ae0, ...})
/usr/local/go/src/database/sql/sql.go:1743 +0x30f
database/sql.(*DB).query(0xc11548, {0x2392298, 0xc0028a30e0}, {0x208f76a, 0x94}, {0xc0023b9ae0, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1700 +0xfd
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0028a30e0}, {0x208f76a, 0x94}, {0xc0023b9ae0, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc0008b5890, {0x2392298, 0xc0028a30e0}, {0x208f76a, 0xc00004e800}, {0xc0023b9ae0, 0xc000c116c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0028a30e0}, {0x2380120, 0xc0008b5890}, {0x1b5d8e0, 0xc00078aa20}, {0x208f76a, 0xc0022eac00}, {0xc0023b9ae0, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x26, {0x2392298, 0xc0028a30e0}, {0x1b5d8e0, 0xc00078aa20}, {0x208f76a, 0xc000c117e0}, {0xc0023b9ae0, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc0008f8be0, {0x2392298, 0xc0028a30e0}, {0xe8572d79, {0xc00072b500, 0x37, 0x37}, 0x0, 0xc001ca47a8, 0xc00073efe8, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0008b5c50, 0xc001ca4780)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000142360, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0008b5c80, 0xc00005f400)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0008b5cb0, 0x26)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc001a5d680, 0xe297d00, 0xe2b03a0, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0x490000, 0xc0014e3d10})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000c11c50, {0x235a640, 0xc0005f2240}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0005f2240}, 0xc000c11d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc000c11de2, 0xc0008f7a00)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc000c11d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc001a5d860)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001a5d860, {0x2392228, 0xc001054c00})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 938372 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8370, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 922530 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001ee17a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000794e70, {0x1f7b7c0, 0xc00067c460}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xb7)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001166600, 0xc000ba1bf8, 0xc000ba1bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001166600, {0x1f7b7c0, 0xc00067c460})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001e48930}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0x20, {0xc0018126a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001e48930}, {0x207ebc6, 0x20}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e48930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0xc00166adc0, {0xc0018126a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001e48930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0xc00276d380, 0xc001e48960, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001e48930}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc001daf100}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0x33, 0xc001e488a0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc001daf100}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc001daf100}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc001daf080}, {0x1f7b7c0, 0xc00067c460}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000cbaaa0, {0x2392228, 0xc001daf100}, 0xc000747340, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006778c0, {0x23922d0, 0xc001cabd40}, 0xc001daf080, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0001437a0, {0x23922d0, 0xc001cabd40}, 0xc001daf080, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc000ba2480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000ba2528, {0x235a640, 0xc000044390}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e79de38, {0x235a640, 0xc000044390}, 0x7f56458145b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc001cabd40}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c0019c3540})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000ba2778, {0x235a640, 0xc0000458c0}, 0x612d333135342d36)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0008802f0, {0x235a640, 0xc0000458c0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001471f0, {0x23922d0, 0xc001cabd40}, 0xc001daed80)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc001b0edb8, 0x4c1cb4}, {0xc057f697c93f8ead, 0x52c04d5848}, {0xc001b0edb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0008a6540, {0x23922d0, 0xc001cabd40}, 0xc001daed80)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80}, 0xc0018125c0, 0xc0023e5110)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80}, 0x15, 0xc001daedc0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80}, 0x2076951, 0xc001daedc0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80}, 0x237bc38, 0xc001daedc0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabd40}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001cabb90}, {0x1f33320, 0xc001daed80}, 0x1e5a940, 0xc001daedc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabb90}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80}, 0x23c27b0, 0xc001daedc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c000e72b60, {0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80}, 0x40, 0xc0011f7800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001cabb60}, {0x1f33320, 0xc001daed80}, 0xc0018125c0, 0xc0023e5110)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc001cabb60}, 0xc001c41f20, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc001ee0240, 0xc000ceab10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc001ee0240, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 938328 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc0011fe0f0)
/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
/temporal/service/history/queueProcessor.go:158 +0x23b
goroutine 938305 [select]:
go.temporal.io/server/service/history.(*transferQueueProcessorImpl).completeTransferLoop(0xc00156ae40)
/temporal/service/history/transferQueueProcessor.go:270 +0x125
created by go.temporal.io/server/service/history.(*transferQueueProcessorImpl).Start
/temporal/service/history/transferQueueProcessor.go:166 +0xfb
goroutine 922449 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc00145e120, {0x2392228, 0xc001d48b40})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 938483 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0012cef50, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0021c6900)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:405 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:403 +0x1f45
goroutine 938344 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de80b0, 0xc00251ddc0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 928568 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00214fe00, {0x2392298, 0xc001a5cfc0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00017d8c0, {0x23922d0, 0xc001ed54d0}, 0x14)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0027c7dd0, {0x23922d0, 0xc001ed54d0}, 0xc00228a0d0, 0x1, 0x228a0e0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00074e0e0, 0xc001ed52c0, 0xc00231e080)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000084240, {0x23922d0, 0xc001ed4fc0}, 0xc00231e080)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001ed4fc0}, {0x1f35540, 0xc00231e080})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc001ed4fc0}, {0x1f35540, 0xc00231e080}, 0x237bc38, 0xc002404fd8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed4fc0}, {0x1f35540, 0xc00231e080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001ed4e70}, {0x1f35540, 0xc00231e080}, 0xc0023008c0, 0xc00231e100)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed4e70}, {0x1f35540, 0xc00231e080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ed4e70}, {0x1f35540, 0xc00231e080}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed4e70}, {0x1f35540, 0xc00231e080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ed4e40}, {0x1f35540, 0xc00231e080}, 0x40, 0xc00231e100)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed4e40}, {0x1f35540, 0xc00231e080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ed4e40}, {0x1f35540, 0xc00231e080}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed4e40}, {0x1f35540, 0xc00231e080})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ed4e40}, {0x1f35540, 0xc00231e080}, 0xc001cefde0, 0xc002404fd8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc001ed4e40}, 0xc001a5cf60, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc0015a2780}, 0xc001b93c20, 0xc000753950, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc0015a2780}, 0xc001b93c20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 438 [sleep]:
time.Sleep(0x306dc4200)
/usr/local/go/src/runtime/time.go:193 +0x12e
go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop.func1()
/temporal/common/membership/rpMonitor.go:280 +0x5f
created by go.temporal.io/server/common/membership.(*ringpopMonitor).startHeartbeatUpsertLoop
/temporal/common/membership/rpMonitor.go:284 +0x77
goroutine 922220 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008ccd80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001d6d130, {0x1f7b560, 0xc002453540}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0000a5b00, 0xc000c61d18, 0xc000c61ce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000a5b00, {0x1f7b560, 0xc002453540})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001b96810}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0x20, {0xc001ed75e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001b96810}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b96810}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0xc002698580, {0xc001ed75e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b96810}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0xc00276d380, 0xc001b96840, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001b96810}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0017ec7c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0x237ac70, 0xc001b96780, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0017ec7c0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0017ec7c0}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0017ec740}, {0x1f7b560, 0xc002453540}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000cbaaa0, {0x2392228, 0xc0017ec7c0}, 0xc00047a330, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006778c0, {0x23922d0, 0xc00205dd40}, 0xc0017ec740, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0001437a0, {0x23922d0, 0xc00205dd40}, 0xc0017ec740, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000c625a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000c62648, {0x235a640, 0xc000044390}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e7d2948, {0x235a640, 0xc000044390}, 0x7f5645814108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001dc8360, {0x23922d0, 0xc00205dd40}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c0017e8300})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000c62898, {0x235a640, 0xc0000458c0}, 0x382d363166342d35)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000458c0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001471f0, {0x23922d0, 0xc00205dd40}, 0xc00205dad0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc001b9cdb8, 0x4c1cb4}, {0xc057f697c33d4e8b, 0x52ba4b1833}, {0xc001b9cdb8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0008a6540, {0x23922d0, 0xc00205dd40}, 0xc00205dad0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0}, 0xc001ed74a0, 0xc001b25f80)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0}, 0x15, 0xc0017ec500)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0}, 0x207690b, 0xc0017ec500)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0}, 0x237bc38, 0xc0017ec500)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205dd40}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00205db90}, {0x1f33180, 0xc00205dad0}, 0x1e5a940, 0xc0017ec500)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205db90}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0}, 0xc00205cde0, 0xc0017ec500)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0}, 0xc001a70768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001a70b60, {0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0}, 0x40, 0xc000490000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00205daa0}, {0x1f33180, 0xc00205dad0}, 0xc001ed74a0, 0xc001b25f80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc00205daa0}, 0xc001d938c0, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc001a1b9e0, 0xc000ceab10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc001a1b9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 922641 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc002403ec0, {0x2392298, 0xc0011de2a0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0009530e0, {0x23922d0, 0xc001c7b2f0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001c7b2f0}, 0xc0019c6090, 0x1bd58c0, 0x19c60a0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc001c7b0e0, 0xc001055200)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc001c7ae40}, 0xc001055200)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001c7ae40}, {0x1f353a0, 0xc001055200})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc001c7ae40}, {0x1f353a0, 0xc001055200}, 0x237bc38, 0xc0024053f8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7ae40}, {0x1f353a0, 0xc001055200})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc001c7acf0}, {0x1f353a0, 0xc001055200}, 0x203000, 0xc001055240)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7acf0}, {0x1f353a0, 0xc001055200})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c7acf0}, {0x1f353a0, 0xc001055200}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7acf0}, {0x1f353a0, 0xc001055200})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001c7ac90}, {0x1f353a0, 0xc001055200}, 0x40, 0xc001055240)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7ac90}, {0x1f353a0, 0xc001055200})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001c7ac90}, {0x1f353a0, 0xc001055200}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7ac90}, {0x1f353a0, 0xc001055200})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001c7ac90}, {0x1f353a0, 0xc001055200}, 0xc0006138c0, 0xc0024053f8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc001c7ac90}, 0xc0011de1e0, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc0015a2780}, 0xc000941b00, 0xc000753950, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc0015a2780}, 0xc000941b00, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 925297 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc00017f810, {0x2392228, 0xc002291740})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 938374 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8370, 0xc001f6d140)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938367 [select]:
go.temporal.io/server/service/history.NewLocalTimerGate.func1()
/temporal/service/history/timerGate.go:114 +0xe5
created by go.temporal.io/server/service/history.NewLocalTimerGate
/temporal/service/history/timerGate.go:109 +0x15e
goroutine 743150 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003e5c00, {0x2392228, 0xc0017f3dc0})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 1026 [select]:
go.temporal.io/sdk/internal.(*basePoller).doPoll(0xc0007aba00, 0xc0023335e0)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:212 +0x249
go.temporal.io/sdk/internal.(*activityTaskPoller).PollTask(0xc0007aba00)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:835 +0x5e
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000761680)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000761680)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 871 [select, 6 minutes]:
net/http.(*persistConn).readLoop(0xc0007f2a20)
/usr/local/go/src/net/http/transport.go:2207 +0xd8a
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1747 +0x1e05
goroutine 989 [select, 6 minutes]:
go.temporal.io/sdk/internal.(*localActivityTunnel).getTask(0xffffffffffffffff)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:161 +0x6c
go.temporal.io/sdk/internal.(*localActivityTaskPoller).PollTask(0xc001164220)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:436 +0x1d
go.temporal.io/sdk/internal.(*baseWorker).pollTask(0xc000761590)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:280 +0xa5
go.temporal.io/sdk/internal.(*baseWorker).runPoller(0xc000761590)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:244 +0x97
created by go.temporal.io/sdk/internal.(*baseWorker).Start
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_worker_base.go:207 +0x6c
goroutine 925341 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001a7f7a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000c27c30, {0x1f736c0, 0xc002242b40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0018b6400, 0xc00006b5c0, 0xc00006b590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0018b6400, {0x1f736c0, 0xc002242b40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc00239bf80}, {0x2076920, 0x8}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x1c75901, {0xc001cee060, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x7f5645814108, {0xc001cee060, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x20, 0xc00223fe30, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0xc00144d2b0, {0xc00224a2c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x0, 0xc00223fe00, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x0, {0xc0007e4540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x1e0cfc0, 0xc00223fd70, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x25145814108, {0xc0007e4540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc00239bf80}, {0x2076920, 0x203000}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x203000, 0xc00223fd40, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc00239bf80}, {0x2076920, 0x46}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, 0x7f5645814108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc001173e50, {0x2392298, 0xc00239bf80}, {0x2076920, 0x100000000}, {0x1f33320, 0xc00224a280}, {0x1f736c0, 0xc002242b40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000f888, {0x2392298, 0xc00239bf80}, 0x1173ea0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001161860, {0x2392298, 0xc00239bf80})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 478 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:233 +0xd2
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:232 +0x105
goroutine 479 [chan receive]:
github.com/temporalio/ringpop-go.(*Ringpop).startTimers.func2()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:247 +0x5b
created by github.com/temporalio/ringpop-go.(*Ringpop).startTimers
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/ringpop.go:246 +0x1a8
goroutine 938307 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27550, 0xc002589fc8)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 928567 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b93b00)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0008bfe40, {0x1f7b7c0, 0xc001bbb2c0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x98)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0019cc400, 0xc00274e4d0, 0xc00274e4a0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0019cc400, {0x1f7b7c0, 0xc001bbb2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc001ed4ae0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0x20, {0xc001cefd00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc001ed4ae0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ed4ae0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0xc001a3b8c0, {0xc001cefd00, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ed4ae0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0xc00218f770, 0xc001ed4b10, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc001ed4ae0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0x8, {0xc001489e90, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002301e40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0x237ac70, 0xc001ed4a50, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002301e40}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002301e40}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc002301d40}, {0x1f7b7c0, 0xc001bbb2c0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000cbabe8, {0x2392228, 0xc002301e40}, 0xc00110f926, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc000956870, {0x2392298, 0xc001a5ccc0}, 0xc002301d40, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc000083060, {0x2392298, 0xc001a5ccc0}, 0xc002301d40, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc0024865a0, {0x2392298, 0xc001a5ccc0})
/temporal/service/matching/forwarder.go:224 +0x59a
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0024867e0, {0x2392298, 0xc001a5ccc0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000c3fb00, {0x23922d0, 0xc001ed4840}, 0x14)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0027c7bc0, {0x23922d0, 0xc001ed4840}, 0xc002221b80, 0x1, 0x2221b90)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00074e0e0, 0xc001ed47e0, 0xc002301b00)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000084240, {0x23922d0, 0xc001ed4540}, 0xc002301b00)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc001ed4540}, {0x1f35540, 0xc002301b00})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc001ed4540}, {0x1f35540, 0xc002301b00}, 0x237bc38, 0xc002404738)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed4540}, {0x1f35540, 0xc002301b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc001ed43f0}, {0x1f35540, 0xc002301b00}, 0xc0023008c0, 0xc002301b80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed43f0}, {0x1f35540, 0xc002301b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001ed43f0}, {0x1f35540, 0xc002301b00}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed43f0}, {0x1f35540, 0xc002301b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001ed43c0}, {0x1f35540, 0xc002301b00}, 0x40, 0xc002301b80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed43c0}, {0x1f35540, 0xc002301b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001ed43c0}, {0x1f35540, 0xc002301b00}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001ed43c0}, {0x1f35540, 0xc002301b00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001ed43c0}, {0x1f35540, 0xc002301b00}, 0xc001cefc60, 0xc002404738)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc001ed43c0}, 0xc001a5cc60, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc001b92ea0, 0xc000753950, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc001b92ea0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 938532 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e86d850, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001b1e280, 0xc000cf8000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001b1e280, {0xc000cf8000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001b1e280, {0xc000cf8000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000cba6c0, {0xc000cf8000, 0x0, 0x7f561e83fd80})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002317c80, {0xc00074ec80, 0x9, 0xc000490000})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc002317c80}, {0xc00074ec80, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00074ec80, 0x9, 0xc000ca6f60}, {0x23585c0, 0xc002317c80})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00074ec40)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000765860)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:1491 +0x41f
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_client.go:355 +0x18ef
goroutine 743500 [select, 1 minutes]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc000a790e0, {0x2392228, 0xc001163a80})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 925229 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00127e1e0, {0x2392298, 0xc00239a300}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002077560, {0x23922d0, 0xc00222dfb0}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc002201140, {0x23922d0, 0xc00222dfb0}, 0xc002221970, 0x1, 0x2221980)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00074e0e0, 0xc00222df50, 0xc00221fa80)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000084240, {0x23922d0, 0xc00222dcb0}, 0xc00221fa80)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00222dcb0}, {0x1f35540, 0xc00221fa80})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc00222dcb0}, {0x1f35540, 0xc00221fa80}, 0x237bc38, 0xc0022302b8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222dcb0}, {0x1f35540, 0xc00221fa80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc00222db60}, {0x1f35540, 0xc00221fa80}, 0xc00221f4c0, 0xc00221fb00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222db60}, {0x1f35540, 0xc00221fa80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00222db60}, {0x1f35540, 0xc00221fa80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222db60}, {0x1f35540, 0xc00221fa80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00222db30}, {0x1f35540, 0xc00221fa80}, 0x40, 0xc00221fb00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222db30}, {0x1f35540, 0xc00221fa80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00222db30}, {0x1f35540, 0xc00221fa80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00222db30}, {0x1f35540, 0xc00221fa80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00222db30}, {0x1f35540, 0xc00221fa80}, 0xc001165e60, 0xc0022302b8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc00222db30}, 0xc00239a2a0, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc00086bc20, 0xc000753950, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc00086bc20, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 506 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e81bbc0, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0007aa500, 0xc000897050, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0007aa500, {0xc000897050, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0007aa500, {0xc000897050, 0xc0014b8f60, 0xc0011f1dc0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e9d0, {0xc000897050, 0x46b213, 0x3493960})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000e9d0}, {0xc000897050, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00009b340, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 925208 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00083f320)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000c27970, {0x1f7b560, 0xc0013c0f00}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xa7)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000798c00, 0xc001b5bd18, 0xc001b5bce8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000798c00, {0x1f7b560, 0xc0013c0f00})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022119e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0x20, {0xc001165260, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022119e0}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022119e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0xc0000a9760, {0xc001165260, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022119e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0xc00276d380, 0xc002211a10, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022119e0}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002207640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0x237ac70, 0xc002211950, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002207640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002207640}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0022075c0}, {0x1f7b560, 0xc0013c0f00}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000cbaaa0, {0x2392228, 0xc002207640}, 0xc002200600, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc0006778c0, {0x23922d0, 0xc002211290}, 0xc0022075c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc0001437a0, {0x23922d0, 0xc002211290}, 0xc0022075c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue.func1()
/temporal/client/matching/retryableClient.go:93 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc001b5c5a0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc001b5c648, {0x235a640, 0xc000044390}, 0x403751)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e7b1ec0, {0x235a640, 0xc000044390}, 0x7f5645814108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollActivityTaskQueue(0xc001dc8360, {0x23922d0, 0xc002211290}, 0x101000000000000, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:97 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:1055 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c002173500})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc001b5c898, {0x235a640, 0xc0000458c0}, 0x622d643931342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x0, {0x235a640, 0xc0000458c0}, 0x15)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollActivityTaskQueue(0xc0001471f0, {0x23922d0, 0xc002211290}, 0xc0022110e0)
/temporal/service/frontend/workflowHandler.go:1063 +0x451
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:502 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc001b72db8, 0x4c1cb4}, {0xc057f697fa7b5037, 0x52f18919d4}, {0xc001b72db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollActivityTaskQueue(0xc0008a6540, {0x23922d0, 0xc002211290}, 0xc0022110e0)
/temporal/service/frontend/dcRedirectionHandler.go:498 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1142 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0}, 0xc0011651a0, 0xc00220e7e0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0}, 0x15, 0xc002207380)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0}, 0x207690b, 0xc002207380)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0}, 0x237bc38, 0xc002207380)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211290}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002211140}, {0x1f33180, 0xc0022110e0}, 0x1e5a940, 0xc002207380)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211140}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0}, 0x9, 0xc002207380)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0}, 0xc001a70768, 0x988353)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001a70b60, {0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0}, 0x40, 0xc00045d000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022110b0}, {0x1f33180, 0xc0022110e0}, 0xc0011651a0, 0xc00220e7e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollActivityTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc0022110b0}, 0xc0022f2b40, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1144 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc000953e60, 0xc000ceab10, 0x347a570, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc000953e60, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 519 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 520 [select]:
github.com/temporalio/ringpop-go/swim.scheduleRepeaditly.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:31 +0x118
created by github.com/temporalio/ringpop-go/swim.scheduleRepeaditly
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/schedule.go:26 +0x132
goroutine 521 [select]:
github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start.func1()
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:78 +0x86
created by github.com/temporalio/ringpop-go/swim.(*discoverProviderHealer).Start
/go/pkg/mod/github.com/temporalio/ringpop-go@v0.0.0-20210913194348-a2bfad448548/swim/heal_via_discover_provider.go:75 +0x70
goroutine 932227 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001a5db60, {0x2392298, 0xc0008a70e0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000941440, {0x23922d0, 0xc00088b5f0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00088b5f0}, 0xc0013c7420, 0x1bd58c0, 0x13c7430)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc00088b590, 0xc000638d40)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc00088b2c0}, 0xc000638d40)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00088b2c0}, {0x1f353a0, 0xc000638d40})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc00088b2c0}, {0x1f353a0, 0xc000638d40}, 0x237bc38, 0xc00194f6e0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00088b2c0}, {0x1f353a0, 0xc000638d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00088b140, {0x23922d0, 0xc00088b170}, {0x1f353a0, 0xc000638d40}, 0xc0008a1b60, 0xc000638d80)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00088b170}, {0x1f353a0, 0xc000638d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00088b170}, {0x1f353a0, 0xc000638d40}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00088b170}, {0x1f353a0, 0xc000638d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00088b0e0}, {0x1f353a0, 0xc000638d40}, 0x40, 0xc000638d80)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00088b0e0}, {0x1f353a0, 0xc000638d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00088b0e0}, {0x1f353a0, 0xc000638d40}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00088b0e0}, {0x1f353a0, 0xc000638d40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00088b0e0}, {0x1f353a0, 0xc000638d40}, 0xc000613d00, 0xc00194f6e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc00088b0e0}, 0xc0008a7080, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc0001a78c0, 0xc000753950, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc0001a78c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 938349 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de80b0, 0xc00251dc40)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938284 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27340, 0xc0005d9340)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 925282 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0017ab0e0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc002457ef0, {0x1f7b7c0, 0xc00067da40}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xcb)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001166c00, 0xc001009bf8, 0xc001009bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001166c00, {0x1f7b7c0, 0xc00067da40})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0022975c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0x20, {0xc001fae460, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0022975c0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022975c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0xc000c2ef20, {0xc001fae460, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022975c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0xc00276d380, 0xc0022975f0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0022975c0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc002290d00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0x237ac70, 0xc002297530, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc002290d00}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc002290d00}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc002290c80}, {0x1f7b7c0, 0xc00067da40}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000cbaaa0, {0x2392228, 0xc002290d00}, 0xc0022eb710, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006778c0, {0x23922d0, 0xc002296e70}, 0xc002290c80, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0001437a0, {0x23922d0, 0xc002296e70}, 0xc002290c80, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004e800, 0xc00100a480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc00100a528, {0x235a640, 0xc000044390}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e79cf40, {0x235a640, 0xc000044390}, 0x7f56458145b8)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc002296e70}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c002286610})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc00100a778, {0x235a640, 0xc0000458c0}, 0x622d373461342d33)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0008802f0, {0x235a640, 0xc0000458c0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001471f0, {0x23922d0, 0xc002296e70}, 0xc002290980)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc001926db8, 0x4c1cb4}, {0xc057f698001d3fb9, 0x52f2c5d353}, {0xc001926db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0008a6540, {0x23922d0, 0xc002296e70}, 0xc002290980)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980}, 0xc001fae300, 0xc0022923f0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980}, 0x15, 0xc0022909c0)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980}, 0x2076951, 0xc0022909c0)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980}, 0x237bc38, 0xc0022909c0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296e70}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc002296d20}, {0x1f33320, 0xc002290980}, 0x1e5a940, 0xc0022909c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296d20}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980}, 0x18a, 0xc0022909c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001dffb60, {0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980}, 0x40, 0xc0011f7800)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002296cf0}, {0x1f33320, 0xc002290980}, 0xc001fae300, 0xc0022923f0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc002296cf0}, 0xc0024934a0, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc000134000, 0xc000ceab10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc000134000, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 922207 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0017f9920, {0x2392298, 0xc001d93aa0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc00013a7e0, {0x23922d0, 0xc001b97110}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001b97110}, 0xc001c1a980, 0x1bd58c0, 0x1c1a990)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc001b970b0, 0xc0017ec940)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc001b96d80}, 0xc0017ec940)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001b96d80}, {0x1f353a0, 0xc0017ec940})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc001b96d80}, {0x1f353a0, 0xc0017ec940}, 0x237bc38, 0xc001f2a270)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b96d80}, {0x1f353a0, 0xc0017ec940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001b96b10, {0x23922d0, 0xc001b96ba0}, {0x1f353a0, 0xc0017ec940}, 0xc00205dad0, 0xc0017ec980)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b96ba0}, {0x1f353a0, 0xc0017ec940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001b96ba0}, {0x1f353a0, 0xc0017ec940}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b96ba0}, {0x1f353a0, 0xc0017ec940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001b96ab0}, {0x1f353a0, 0xc0017ec940}, 0x40, 0xc0017ec980)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b96ab0}, {0x1f353a0, 0xc0017ec940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001b96ab0}, {0x1f353a0, 0xc0017ec940}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001b96ab0}, {0x1f353a0, 0xc0017ec940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001b96ab0}, {0x1f353a0, 0xc0017ec940}, 0xc001ed7740, 0xc001f2a270)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc001b96ab0}, 0xc001d93a40, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc000916fc0, 0xc000753950, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc000916fc0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 507 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00009b340, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 938410 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de89a0, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 534 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000625600)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 531 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000624dc0)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 533 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000625550)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 504 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e86da20, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0007aa580, 0xc000896f80, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0007aa580, {0xc000896f80, 0x10, 0x10})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0007aa580, {0xc000896f80, 0xc000e74f50, 0x1449ef8})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00000e9d8, {0xc000896f80, 0x46b213, 0x617f62ee})
/usr/local/go/src/net/net.go:183 +0x45
io.ReadAtLeast({0x235d4e0, 0xc00000e9d8}, {0xc000896f80, 0x10, 0x10}, 0x10)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
github.com/uber/tchannel-go.(*Connection).readFrames(0xc00009b080, 0x881d60)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:660 +0x88
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:374 +0x12f4
goroutine 505 [select]:
github.com/uber/tchannel-go.(*Connection).writeFrames(0xc00009b080, 0x0)
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:737 +0x89
created by github.com/uber/tchannel-go.(*Channel).newConnection
/go/pkg/mod/github.com/uber/tchannel-go@v1.22.0/connection.go:375 +0x1348
goroutine 532 [select]:
go.temporal.io/server/common/membership.(*ringpopServiceResolver).refreshRingWorker(0xc000624e70)
/temporal/common/membership/rpServiceResolver.go:340 +0x11c
created by go.temporal.io/server/common/membership.(*ringpopServiceResolver).Start
/temporal/common/membership/rpServiceResolver.go:127 +0x27b
goroutine 938347 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de80b0, 0xc00251ddc0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938278 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27340, 0xc002696800)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 922266 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0001fcc60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001fa1ce0, {0x1f7b560, 0xc0024eb400}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xca)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0011fbd00, 0xc001a24440, 0xc001a24410)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0011fbd00, {0x1f7b560, 0xc0024eb400})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc0020ea630}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0x20, {0xc0024a38e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc0020ea630}, {0x207eb79, 0x20}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020ea630}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0xc000f56c60, {0xc0024a38e0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0020ea630}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0xc00218f770, 0xc0020ea660, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc0020ea630}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0x8, {0xc001489e90, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc0019ae640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0x237ac70, 0xc0020ea5a0, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc0019ae640}, {0x207eb79, 0x4d}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc0019ae640}, {0x207eb79, 0x4f94ca}, {0x1f353a0, 0xc0019ae5c0}, {0x1f7b560, 0xc0024eb400}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollActivityTaskQueue(0xc000cbabe8, {0x2392228, 0xc0019ae640}, 0xc000879a86, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:151 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollActivityTaskQueue(0xc000956870, {0x2392298, 0xc000cac4e0}, 0xc0019ae5c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:126 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollActivityTaskQueue(0xc000083060, {0x2392298, 0xc000cac4e0}, 0xc0019ae5c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:117 +0xfc
go.temporal.io/server/service/matching.(*Forwarder).ForwardPoll(0xc001645da0, {0x2392298, 0xc000cac4e0})
/temporal/service/matching/forwarder.go:241 +0x323
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc000cac480, {0x2392298, 0xc000cac4e0}, 0x0)
/temporal/service/matching/matcher.go:380 +0x4c5
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0001fc000, {0x23922d0, 0xc001c7de00}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc001c7de00}, 0xc0011f2f00, 0x1bd58c0, 0x11f2f10)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc001c7dda0, 0xc0014a9940)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc001c7d9b0}, 0xc0014a9940)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc001c7d9b0}, {0x1f353a0, 0xc0014a9940})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc001c7d9b0}, {0x1f353a0, 0xc0014a9940}, 0x237bc38, 0xc001f22ae0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7d9b0}, {0x1f353a0, 0xc0014a9940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc001c7d4d0, {0x23922d0, 0xc001c7d6e0}, {0x1f353a0, 0xc0014a9940}, 0xc001c7c0f0, 0xc0014a99c0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7d6e0}, {0x1f353a0, 0xc0014a9940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc001c7d6e0}, {0x1f353a0, 0xc0014a9940}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7d6e0}, {0x1f353a0, 0xc0014a9940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc001c7d470}, {0x1f353a0, 0xc0014a9940}, 0x40, 0xc0014a99c0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7d470}, {0x1f353a0, 0xc0014a9940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc001c7d470}, {0x1f353a0, 0xc0014a9940}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc001c7d470}, {0x1f353a0, 0xc0014a9940})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc001c7d470}, {0x1f353a0, 0xc0014a9940}, 0xc0024a3700, 0xc001f22ae0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc001c7d470}, 0xc001645b00, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc0006cc360, 0xc000753950, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc0006cc360, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 938421 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8a50, 0xc002057030)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938360 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de82c0, 0xc0021466c0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 931632 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0011ded20, {0x2392298, 0xc00246dce0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc000975680, {0x23922d0, 0xc00236acf0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00236acf0}, 0xc002333b50, 0x1bd58c0, 0x2333b60)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc00236ac90, 0xc00232dc80)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc00236a9f0}, 0xc00232dc80)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00236a9f0}, {0x1f353a0, 0xc00232dc80})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc00236a9f0}, {0x1f353a0, 0xc00232dc80}, 0x237bc38, 0xc002275ce0)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236a9f0}, {0x1f353a0, 0xc00232dc80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc00236a870, {0x23922d0, 0xc00236a8a0}, {0x1f353a0, 0xc00232dc80}, 0xc02be2, 0xc00232dcc0)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236a8a0}, {0x1f353a0, 0xc00232dc80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00236a8a0}, {0x1f353a0, 0xc00232dc80}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236a8a0}, {0x1f353a0, 0xc00232dc80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00236a810}, {0x1f353a0, 0xc00232dc80}, 0x40, 0xc00232dcc0)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236a810}, {0x1f353a0, 0xc00232dc80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00236a810}, {0x1f353a0, 0xc00232dc80}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00236a810}, {0x1f353a0, 0xc00232dc80})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00236a810}, {0x1f353a0, 0xc00232dc80}, 0xc001812660, 0xc002275ce0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc00236a810}, 0xc00246dc20, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc000847320, 0xc000753950, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc000847320, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 925436 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc00127e1e0, {0x2392298, 0xc002311320}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc002077560, {0x23922d0, 0xc002323380}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc0022cb140, {0x23922d0, 0xc002323380}, 0xc00231afa0, 0x1, 0x231afb0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00074e0e0, 0xc002323320, 0xc00231ecc0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000084240, {0x23922d0, 0xc002323080}, 0xc00231ecc0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc002323080}, {0x1f35540, 0xc00231ecc0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc002323080}, {0x1f35540, 0xc00231ecc0}, 0x237bc38, 0xc00230d3c8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002323080}, {0x1f35540, 0xc00231ecc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc002322f30}, {0x1f35540, 0xc00231ecc0}, 0xc00231e700, 0xc00231ed40)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002322f30}, {0x1f35540, 0xc00231ecc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002322f30}, {0x1f35540, 0xc00231ecc0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002322f30}, {0x1f35540, 0xc00231ecc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002322f00}, {0x1f35540, 0xc00231ecc0}, 0x40, 0xc00231ed40)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002322f00}, {0x1f35540, 0xc00231ecc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002322f00}, {0x1f35540, 0xc00231ecc0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002322f00}, {0x1f35540, 0xc00231ecc0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002322f00}, {0x1f35540, 0xc00231ecc0}, 0xc000bf0780, 0xc00230d3c8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc002322f00}, 0xc0023112c0, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc00082c7e0, 0xc000753950, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc00082c7e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 938306 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc000c27550, 0xc002589fc8)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938390 [select]:
go.temporal.io/server/service/history.(*timerQueueProcessorBase).internalProcessor(0xc00191bc80)
/temporal/service/history/timerQueueProcessorBase.go:271 +0x37f
go.temporal.io/server/service/history.(*timerQueueProcessorBase).processorPump(0xc00191bc80)
/temporal/service/history/timerQueueProcessorBase.go:196 +0x9e
created by go.temporal.io/server/service/history.(*timerQueueProcessorBase).Start
/temporal/service/history/timerQueueProcessorBase.go:163 +0xaf
goroutine 928959 [select]:
database/sql.(*DB).conn(0xc0008dcea0, {0x2392298, 0xc0028a2de0}, 0x1)
/usr/local/go/src/database/sql/sql.go:1312 +0x413
database/sql.(*DB).query(0x1bf1548, {0x2392298, 0xc0028a2de0}, {0x208f76a, 0x94}, {0xc0023b9540, 0x5, 0x5}, 0xc0)
/usr/local/go/src/database/sql/sql.go:1695 +0x5d
database/sql.(*DB).QueryContext(0x8, {0x2392298, 0xc0028a2de0}, {0x208f76a, 0x94}, {0xc0023b9540, 0x5, 0x5})
/usr/local/go/src/database/sql/sql.go:1674 +0xdf
github.com/jmoiron/sqlx.(*DB).QueryxContext(0xc0008b5890, {0x2392298, 0xc0028a2de0}, {0x208f76a, 0xc00004e800}, {0xc0023b9540, 0xc001bf16c8, 0x40ce34})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:161 +0x45
github.com/jmoiron/sqlx.SelectContext({0x2392298, 0xc0028a2de0}, {0x2380120, 0xc0008b5890}, {0x1b5d8e0, 0xc00078a7e0}, {0x208f76a, 0xc0022eac00}, {0xc0023b9540, 0x5, ...})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:55 +0x94
github.com/jmoiron/sqlx.(*DB).SelectContext(0x2a, {0x2392298, 0xc0028a2de0}, {0x1b5d8e0, 0xc00078a7e0}, {0x208f76a, 0xc001bf17e0}, {0xc0023b9540, 0x5, 0x5})
/go/pkg/mod/github.com/jmoiron/sqlx@v1.3.4/sqlx_context.go:140 +0x7a
go.temporal.io/server/common/persistence/sql/sqlplugin/mysql.(*db).SelectFromTasks(0xc0008f8be0, {0x2392298, 0xc0028a2de0}, {0x64aa5e25, {0xc00072b480, 0x3b, 0x3b}, 0x0, 0xc001ca4568, 0xc00073ee30, ...})
/temporal/common/persistence/sql/sqlplugin/mysql/task.go:112 +0x1e3
go.temporal.io/server/common/persistence/sql.(*sqlTaskManager).GetTasks(0xc0008b5c50, 0xc001ca4540)
/temporal/common/persistence/sql/task.go:433 +0x285
go.temporal.io/server/common/persistence.(*taskManagerImpl).GetTasks(0xc000142360, 0x2004617)
/temporal/common/persistence/task_manager.go:236 +0x30
go.temporal.io/server/common/persistence.(*taskRateLimitedPersistenceClient).GetTasks(0xc0008b5c80, 0xc00005f400)
/temporal/common/persistence/persistenceRateLimitedClients.go:465 +0x4a
go.temporal.io/server/common/persistence.(*taskPersistenceClient).GetTasks(0xc0008b5cb0, 0x2a)
/temporal/common/persistence/persistenceMetricClients.go:641 +0x7d
go.temporal.io/server/service/matching.(*taskQueueDB).GetTasks(0xc001fcc3c0, 0xca463a0, 0xca5ea40, 0x3e8)
/temporal/service/matching/db.go:147 +0xea
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange.func1()
/temporal/service/matching/taskReader.go:184 +0x65
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry.func1()
/temporal/service/matching/taskQueueManager.go:499 +0x32
go.temporal.io/server/common/backoff.Retry.func1({0xc0011f7800, 0xc00247c8c0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc001bf1c50, {0x235a640, 0xc0005f2240}, 0x3)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x3, {0x235a640, 0xc0005f2240}, 0xc001bf1d10)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).executeWithRetry(0xc001bf1de2, 0xc000c9f1e0)
/temporal/service/matching/taskQueueManager.go:503 +0x74
go.temporal.io/server/service/matching.(*taskReader).getTaskBatchWithRange(0xc001bf1d60, 0x1a55400, 0x1bbc140)
/temporal/service/matching/taskReader.go:183 +0x4d
go.temporal.io/server/service/matching.(*taskReader).getTaskBatch(0xc001fcc540)
/temporal/service/matching/taskReader.go:206 +0x9f
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc001fcc540, {0x2392228, 0xc001195200})
/temporal/service/matching/taskReader.go:145 +0x165
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 925343 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc001b34d80)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0003ba580, {0x1f7b7c0, 0xc0019c80a0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc4)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0018b6500, 0xc000b91bf8, 0xc000b91bc8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0018b6500, {0x1f7b7c0, 0xc0019c80a0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x23922d0, 0xc00224eab0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0x20, {0xc001cee240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/server/common/rpc.errorInterceptor({0x23922d0, 0xc00224eab0}, {0x207ebc6, 0x20}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0x10, 0x20a9218, ...)
/temporal/common/rpc/grpc.go:108 +0x7e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00224eab0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0xc0014874a0, {0xc001cee240, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/metrics.NewClientMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc00224eab0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0xc00276d380, 0xc00224eae0, ...)
/temporal/common/metrics/grpc.go:79 +0x189
google.golang.org/grpc.getChainUnaryInvoker.func1({0x23922d0, 0xc00224eab0}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0x8, {0xc001467540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/server/common/rpc.versionHeadersInterceptor({0x2392228, 0xc00224a840}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0x237ac70, 0xc00224ea20, ...)
/temporal/common/rpc/grpc.go:122 +0xce
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392228, 0xc00224a840}, {0x207ebc6, 0x4d}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, 0x3493960, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0x4c1997, {0x2392228, 0xc00224a840}, {0x207ebc6, 0x4f94ca}, {0x1f35540, 0xc00224a7c0}, {0x1f7b7c0, 0xc0019c80a0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/server/api/matchingservice/v1.(*matchingServiceClient).PollWorkflowTaskQueue(0xc000cbaaa0, {0x2392228, 0xc00224a840}, 0xc002201b90, {0x0, 0x0, 0x0})
/temporal/api/matchingservice/v1/service.pb.go:142 +0xc9
go.temporal.io/server/client/matching.(*clientImpl).PollWorkflowTaskQueue(0xc0006778c0, {0x23922d0, 0xc00224e360}, 0xc00224a7c0, {0x0, 0x0, 0x0})
/temporal/client/matching/client.go:146 +0x242
go.temporal.io/server/client/matching.(*metricClient).PollWorkflowTaskQueue(0xc0001437a0, {0x23922d0, 0xc00224e360}, 0xc00224a7c0, {0x0, 0x0, 0x0})
/temporal/client/matching/metricClient.go:143 +0xfc
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue.func1()
/temporal/client/matching/retryableClient.go:109 +0x46
go.temporal.io/server/common/backoff.Retry.func1({0xc00004c000, 0xc000b92480})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000b92528, {0x235a640, 0xc000044390}, 0x0)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0x7f561e7d1d98, {0x235a640, 0xc000044390}, 0x7f5645814108)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/client/matching.(*retryableClient).PollWorkflowTaskQueue(0x1888bc6, {0x23922d0, 0xc00224e360}, 0x101000001888a8f, {0x0, 0x0, 0x0})
/temporal/client/matching/retryableClient.go:113 +0xb5
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue.func1()
/temporal/service/frontend/workflowHandler.go:790 +0x12c
go.temporal.io/server/common/backoff.Retry.func1({0xc00010e870, 0x10100c00222f2b0})
/temporal/common/backoff/retry.go:104 +0x1b
go.temporal.io/server/common/backoff.RetryContext({0x2392260, 0xc00012c000}, 0xc000b92778, {0x235a640, 0xc0000458c0}, 0x622d303735342d31)
/temporal/common/backoff/retry.go:125 +0x133
go.temporal.io/server/common/backoff.Retry(0xc0008802f0, {0x235a640, 0xc0000458c0}, 0x20)
/temporal/common/backoff/retry.go:105 +0x59
go.temporal.io/server/service/frontend.(*WorkflowHandler).PollWorkflowTaskQueue(0xc0001471f0, {0x23922d0, 0xc00224e360}, 0xc00224a4c0)
/temporal/service/frontend/workflowHandler.go:798 +0x7c7
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue.func2({0xc00072c810, 0x6})
/temporal/service/frontend/dcRedirectionHandler.go:532 +0x14c
go.temporal.io/server/service/frontend.(*NoopRedirectionPolicy).WithNamespaceRedirect(0xc0008a6540, {0xc001838db8, 0x4c1cb4}, {0xc057f698015e87de, 0x52f4071b79}, {0xc001838db8, 0x93ebb2}, 0x34c4660)
/temporal/service/frontend/dcRedirectionPolicy.go:118 +0x26
go.temporal.io/server/service/frontend.(*DCRedirectionHandlerImpl).PollWorkflowTaskQueue(0xc0008a6540, {0x23922d0, 0xc00224e360}, 0xc00224a4c0)
/temporal/service/frontend/dcRedirectionHandler.go:528 +0x2d9
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1088 +0x7b
go.temporal.io/server/common/authorization.(*interceptor).Interceptor(0xc00066ef00, {0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0}, 0xc001cee160, 0xc002231fe0)
/temporal/common/authorization/interceptor.go:152 +0x637
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*NamespaceCountLimitInterceptor).Intercept(0xc000677f20, {0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0}, 0x15, 0xc00224a500)
/temporal/common/rpc/interceptor/namespace_count_limit.go:89 +0x1a3
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceRateLimitInterceptor).Intercept(0xc000677d70, {0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0}, 0x2076951, 0xc00224a500)
/temporal/common/rpc/interceptor/namespace_rate_limit.go:87 +0x1fa
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc00064f938, {0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0}, 0x237bc38, 0xc00224a500)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e360}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x0, {0x23922d0, 0xc00224e210}, {0x1f33320, 0xc00224a4c0}, 0x1e5a940, 0xc00224a500)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e210}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0}, 0x23c27b0, 0xc00224a500)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0}, 0x0, 0x49d300)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc/interceptor.(*NamespaceLogInterceptor).Intercept(0x10000c001a70b60, {0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0}, 0x40, 0xc00045d000)
/temporal/common/rpc/interceptor/namespace_logger.go:83 +0x74f
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc00224e1e0}, {0x1f33320, 0xc00224a4c0}, 0xc001cee160, 0xc002231fe0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/api/workflowservice/v1._WorkflowService_PollWorkflowTaskQueue_Handler({0x1fd7900, 0xc0008a6540}, {0x23922d0, 0xc00224e1e0}, 0xc0025443c0, 0xc000143960)
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:1090 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc001a7f8c0, 0xc000ceab10, 0x347a528, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc00044d180, {0x23d1088, 0xc000fff980}, 0xc001a7f8c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 925425 [select]:
go.temporal.io/server/service/matching.(*taskReader).dispatchBufferedTasks(0xc002310000, {0x2392228, 0xc0023014c0})
/temporal/service/matching/taskReader.go:106 +0xd7
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 938378 [select]:
go.temporal.io/server/service/history.(*queueProcessorBase).processorPump(0xc0007602d0)
/temporal/service/history/queueProcessor.go:212 +0x347
created by go.temporal.io/server/service/history.(*queueProcessorBase).Start
/temporal/service/history/queueProcessor.go:158 +0x23b
goroutine 938386 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8580, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938534 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000d06d20, 0x1)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002317e60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:321 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:318 +0x17ef
goroutine 938536 [IO wait]:
internal/poll.runtime_pollWait(0x7f561e86d938, 0x72)
/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0006a8900, 0xc000dcc000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0006a8900, {0xc000dcc000, 0x8000, 0x8000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0006a8900, {0xc000dcc000, 0x1040100000000, 0x0})
/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0027744b0, {0xc000dcc000, 0xff8e, 0xc05cdf05ac})
/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002317e00, {0xc00074f000, 0x9, 0xc0011f7800})
/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x23585c0, 0xc002317e00}, {0xc00074f000, 0x9, 0x9}, 0x9)
/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00074f000, 0x9, 0xc00261c5d0}, {0x23585c0, 0xc002317e00})
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00074efc0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20211015210444-4f30a5c0130f/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001fca180, 0xc0007a1f98, 0x1418c25)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:558 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc00044d180, {0x23d1088, 0xc001fca180})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:907 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:846 +0x185
goroutine 938299 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8000, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 929010 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc001fbae40, {0x2392298, 0xc001fbaea0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0007fb9e0, {0x23922d0, 0xc0022a3170}, 0x23)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0xc00243c0c0, {0x23922d0, 0xc0022a3170}, 0xc0011b3c70, 0x1, 0x11b3c80)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollWorkflowTaskQueue(0xc00074e0e0, 0xc0022a2e10, 0xc0011ba2c0)
/temporal/service/matching/matchingEngine.go:352 +0x3cd
go.temporal.io/server/service/matching.(*Handler).PollWorkflowTaskQueue(0xc000084240, {0x23922d0, 0xc0022a29c0}, 0xc0011ba2c0)
/temporal/service/matching/handler.go:247 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler.func1({0x23922d0, 0xc0022a29c0}, {0x1f35540, 0xc0011ba2c0})
/temporal/api/matchingservice/v1/service.pb.go:303 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc0022a29c0}, {0x1f35540, 0xc0011ba2c0}, 0x237bc38, 0xc0022d27f8)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a29c0}, {0x1f35540, 0xc0011ba2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0x18d16bb, {0x23922d0, 0xc0022a2720}, {0x1f35540, 0xc0011ba2c0}, 0xc0011ab640, 0xc0011ba340)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a2720}, {0x1f35540, 0xc0011ba2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc0022a2720}, {0x1f35540, 0xc0011ba2c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a2720}, {0x1f35540, 0xc0011ba2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc0022a26f0}, {0x1f35540, 0xc0011ba2c0}, 0x40, 0xc0011ba340)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a26f0}, {0x1f35540, 0xc0011ba2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc0022a26f0}, {0x1f35540, 0xc0011ba2c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc0022a26f0}, {0x1f35540, 0xc0011ba2c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc0022a26f0}, {0x1f35540, 0xc0011ba2c0}, 0xc001ed6c00, 0xc0022d27f8)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollWorkflowTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc0022a26f0}, 0xc001fba9c0, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:305 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc0015a2780}, 0xc00181fb00, 0xc000753950, 0x34776c0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc0015a2780}, 0xc00181fb00, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 925210 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0026e8900, {0x2392298, 0xc0022f2ea0}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc001a7e240, {0x23922d0, 0xc00221a390}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc00221a390}, 0xc00220b9b0, 0x1bd58c0, 0x220b9c0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc00221a330, 0xc0022079c0)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc00221a090}, 0xc0022079c0)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc00221a090}, {0x1f353a0, 0xc0022079c0})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc00221a090}, {0x1f353a0, 0xc0022079c0}, 0x237bc38, 0xc00220ec60)
/temporal/common/rpc/interceptor/rate_limit.go:83 +0x1a5
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc00221a090}, {0x1f353a0, 0xc0022079c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1113 +0x63
go.temporal.io/server/common/rpc/interceptor.(*TelemetryInterceptor).Intercept(0xc002211ef0, {0x23922d0, 0xc002211f20}, {0x1f353a0, 0xc0022079c0}, 0xc0022110e0, 0xc002207a00)
/temporal/common/rpc/interceptor/telemetry.go:108 +0x22e
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211f20}, {0x1f353a0, 0xc0022079c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1({0x23922d0, 0xc002211f20}, {0x1f353a0, 0xc0022079c0}, 0x1cc5b40, 0x34c4b38)
/temporal/common/metrics/grpc.go:113 +0x64
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211f20}, {0x1f353a0, 0xc0022079c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1({0x23922d0, 0xc002211e90}, {0x1f353a0, 0xc0022079c0}, 0x40, 0xc002207a00)
/temporal/common/metrics/grpc.go:66 +0x7c
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211e90}, {0x1f353a0, 0xc0022079c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x23922d0, 0xc002211e90}, {0x1f353a0, 0xc0022079c0}, 0x1e78c20, 0x1)
/temporal/common/rpc/grpc.go:131 +0x34
google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x23922d0, 0xc002211e90}, {0x1f353a0, 0xc0022079c0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1116 +0x87
google.golang.org/grpc.chainUnaryInterceptors.func1({0x23922d0, 0xc002211e90}, {0x1f353a0, 0xc0022079c0}, 0xc001165340, 0xc00220ec60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1118 +0x159
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler({0x1fac440, 0xc000084240}, {0x23922d0, 0xc002211e90}, 0xc0022f2e40, 0xc000083120)
/temporal/api/matchingservice/v1/service.pb.go:323 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc00083f9e0, 0xc000753950, 0x34776d8, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1279 +0xccf
google.golang.org/grpc.(*Server).handleStream(0xc0005d0380, {0x23d1088, 0xc000d20180}, 0xc00083f9e0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:1608 +0xa2a
google.golang.org/grpc.(*Server).serveStreams.func1.2()
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:923 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
/go/pkg/mod/google.golang.org/grpc@v1.41.0/server.go:921 +0x294
goroutine 922635 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc001053720)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 938435 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8dc0, 0xc001ce1cc0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 928626 [select]:
go.temporal.io/server/service/matching.(*taskWriter).taskWriterLoop(0xc0003e4a10, {0x2392228, 0xc00235f340})
/temporal/service/matching/taskWriter.go:219 +0x12a
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 632 [select]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop(0xc00074a240, {0x2392228, 0xc001162800})
/temporal/common/namespace/registry.go:355 +0x187
go.temporal.io/server/internal/goro.Go.func1()
/temporal/internal/goro/goro.go:56 +0x6c
created by go.temporal.io/server/internal/goro.Go
/temporal/internal/goro/goro.go:52 +0x112
goroutine 633 [chan receive]:
go.temporal.io/server/common/namespace.(*registry).refreshLoop.func1()
/temporal/common/namespace/registry.go:345 +0x4d
created by go.temporal.io/server/common/namespace.(*registry).refreshLoop
/temporal/common/namespace/registry.go:344 +0x106
goroutine 925226 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0007f2c60)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:323 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/transport.go:338
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000c27ad0, {0x1f736c0, 0xc0022247e0}, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:961 +0xbb
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x6e)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:826 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc000799300, 0xc0004335c0, 0xc000433590)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:684 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000799300, {0x1f736c0, 0xc0022247e0})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/stream.go:825 +0x11f
google.golang.org/grpc.invoke({0x2392298, 0xc0022f3e00}, {0x2076920, 0x8}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x1c75901, {0xc0011657a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:73 +0xd7
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0xbfbf4f, 0x20a9218, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x7f5645814108, {0xc0011657a0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
github.com/grpc-ecosystem/go-grpc-middleware/retry.UnaryClientInterceptor.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x20, 0xc00222cb70, ...)
/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.3.0/retry/retry.go:36 +0x14e
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0xc00144d2b0, {0xc00221f2c0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/retry.NewRetryOptionsInterceptor.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x0, 0xc00222cb40, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/retry/interceptor.go:157 +0x6d0
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x0, {0xc0007e4540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal/common/metrics.NewGRPCMetricsInterceptor.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x1e0cfc0, 0xc00222cab0, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/common/metrics/interceptor.go:47 +0x1d2
google.golang.org/grpc.getChainUnaryInvoker.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x1e0cfc0, {0xc0007e4540, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:379 +0x154
go.temporal.io/sdk/internal.errorInterceptor({0x2392298, 0xc0022f3e00}, {0x2076920, 0x203000}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x203000, 0xc00222ca80, ...)
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/grpc_dialer.go:174 +0x7e
google.golang.org/grpc.chainUnaryClientInterceptors.func1({0x2392298, 0xc0022f3e00}, {0x2076920, 0x46}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, 0x7f5645814108, 0x20a9218, ...)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/clientconn.go:367 +0x128
google.golang.org/grpc.(*ClientConn).Invoke(0xc000721650, {0x2392298, 0xc0022f3e00}, {0x2076920, 0x100781d38}, {0x1f33320, 0xc00221f280}, {0x1f736c0, 0xc0022247e0}, {0x0, ...})
/go/pkg/mod/google.golang.org/grpc@v1.41.0/call.go:35 +0x21f
go.temporal.io/api/workflowservice/v1.(*workflowServiceClient).PollWorkflowTaskQueue(0xc00000f888, {0x2392298, 0xc0022f3e00}, 0x0, {0x0, 0x0, 0x0})
/go/pkg/mod/go.temporal.io/api@v1.5.1-0.20211018190919-a5f4a169cd08/workflowservice/v1/service.pb.go:379 +0xc9
go.temporal.io/sdk/internal.(*workflowTaskPoller).poll(0xc001161860, {0x2392298, 0xc0022f3e00})
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:652 +0x148
go.temporal.io/sdk/internal.(*basePoller).doPoll.func1()
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:207 +0x4e
created by go.temporal.io/sdk/internal.(*basePoller).doPoll
/go/pkg/mod/go.temporal.io/sdk@v1.10.0/internal/internal_task_pollers.go:206 +0x1ef
goroutine 743152 [select]:
go.temporal.io/server/service/matching.(*taskReader).getTasksPump(0xc0017f87e0, {0x2392228, 0xc0017f3e00})
/temporal/service/matching/taskReader.go:140 +0x13b
go.temporal.io/server/internal/goro.(*Group).Go.func1()
/temporal/internal/goro/group.go:57 +0x6a
created by go.temporal.io/server/internal/goro.(*Group).Go
/temporal/internal/goro/group.go:55 +0xbe
goroutine 938382 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8580, 0xc00194e330)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938535 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001fca180)
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:1078 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
/go/pkg/mod/google.golang.org/grpc@v1.41.0/internal/transport/http2_server.go:330 +0x1837
goroutine 938206 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1()
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:621 +0xb0
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/go/pkg/mod/github.com/go-sql-driver/mysql@v1.5.0/connection.go:618 +0x105
goroutine 929011 [select]:
go.temporal.io/server/service/matching.(*liveness).eventLoop(0xc0011b8ff0)
/temporal/service/matching/liveness.go:101 +0xc6
created by go.temporal.io/server/service/matching.(*liveness).Start
/temporal/service/matching/liveness.go:80 +0x70
goroutine 938342 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de80b0, 0x34c4b38)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 938387 [select]:
go.temporal.io/server/service/history.(*taskProcessor).taskWorker(0xc001de8580, 0x0)
/temporal/service/history/taskProcessor.go:164 +0xe7
created by go.temporal.io/server/service/history.(*taskProcessor).start
/temporal/service/history/taskProcessor.go:145 +0x2c
goroutine 935295 [select]:
go.temporal.io/server/service/matching.(*TaskMatcher).poll(0xc0008a6060, {0x2392298, 0xc002419e60}, 0x0)
/temporal/service/matching/matcher.go:365 +0x337
go.temporal.io/server/service/matching.(*TaskMatcher).Poll(...)
/temporal/service/matching/matcher.go:287
go.temporal.io/server/service/matching.(*taskQueueManagerImpl).GetTask(0xc0006e70e0, {0x23922d0, 0xc0023153e0}, 0x1bd3d80)
/temporal/service/matching/taskQueueManager.go:363 +0x448
go.temporal.io/server/service/matching.(*matchingEngineImpl).getTask(0x23922d0, {0x23922d0, 0xc0023153e0}, 0xc0013e2db0, 0x1bd58c0, 0x13e2dc0)
/temporal/service/matching/matchingEngine.go:698 +0x5d
go.temporal.io/server/service/matching.(*matchingEngineImpl).PollActivityTaskQueue(0xc00074e0e0, 0xc002315380, 0xc0013dbd80)
/temporal/service/matching/matchingEngine.go:451 +0x3cf
go.temporal.io/server/service/matching.(*Handler).PollActivityTaskQueue(0xc000084240, {0x23922d0, 0xc0023150e0}, 0xc0013dbd80)
/temporal/service/matching/handler.go:218 +0x21f
go.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler.func1({0x23922d0, 0xc0023150e0}, {0x1f353a0, 0xc0013dbd80})
/temporal/api/matchingservice/v1/service.pb.go:321 +0x78
go.temporal.io/server/common/rpc/interceptor.(*RateLimitInterceptor).Intercept(0xc0006e43c0, {0x23922d0, 0xc0023150e0}, {0x1f353a0, 0xc0013dbd80}, 0x237bc38, 0xc001ad6ac8)